[api,lineage] Ensure node-local traffic

Since 0.37, many requests to the k8s API now go through a mutating
webhook (lineage-controller-webhook). Since the lineage webhook makes
multiple requests to the k8s API and, indirectly, to the Cozystack API
server, each request for, e.g., creating a secret now causes a lot of
chatter between the webhook, the k8s API, and the Cozystack API. When
this happens cross-node or, worse yet, cross-zone, this can blow up the
latency for simple requests.

This patch changes the Cozystack API to a DaemonSet targetting
controlplane nodes, configures its service for an `Local` internal
traffic policy and adds environment variables indicating that the k8s
API server is to be found at <hostIP>:6443, **not only for the Cozystack
API, but also for the lineage-controller-webhook.** This is a valid
configuration in most scenarios, including the default installation
method on top of Talos Linux in Cozystack, however, if this is not valid
in your environment, you must now set the values
`.lineageControllerWebhook.localK8sAPIEndpoint.enabled` and
`.cozystackAPI.localK8sAPIEndpoint.enabled` to `false` in the respective
system Helm releases.

```release-note
[api,lineage] Configure all chatter between the Lineage webhook, the
Cozystack API server and the Kubernetes API server to be confined to a
single controlplane node, improving k8s API latency.
```

Signed-off-by: Timofei Larkin <lllamnyp@gmail.com>

packages/system/cozystack-api/templates/apiservice.yaml

@@ -1,9 +1,10 @@
 apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/cozystack-api"
   name: v1alpha1.apps.cozystack.io
 spec:
-  insecureSkipTLSVerify: true
   group: apps.cozystack.io
   groupPriorityMinimum: 1000
   versionPriority: 15
@@ -15,9 +16,10 @@ spec:
 apiVersion: apiregistration.k8s.io/v1
 kind: APIService
 metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/cozystack-api"
   name: v1alpha1.core.cozystack.io
 spec:
-  insecureSkipTLSVerify: true
   group: core.cozystack.io
   groupPriorityMinimum: 1000
   versionPriority: 15

packages/system/cozystack-api/templates/certmanager.yaml

@@ -0,0 +1,45 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: cozystack-api-selfsigned
+  namespace: {{ .Release.Namespace }}
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: cozystack-api-ca
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: cozystack-api-ca
+  duration: 43800h  # 5 years
+  commonName: cozystack-api-ca
+  issuerRef:
+    name: cozystack-api-selfsigned
+  isCA: true
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: cozystack-api-ca
+  namespace: {{ .Release.Namespace }}
+spec:
+  ca:
+    secretName: cozystack-api-ca
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: cozystack-api
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: cozystack-api-cert
+  duration: 8760h
+  renewBefore: 720h
+  issuerRef:
+    name: cozystack-api-ca
+  commonName: cozystack-api
+  dnsNames:
+    - cozystack-api
+    - cozystack-api.{{ .Release.Namespace }}.svc

packages/system/cozystack-api/templates/deployment.yaml

@@ -1,12 +1,18 @@
 apiVersion: apps/v1
+{{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }}
+kind: DaemonSet
+{{- else }}
 kind: Deployment
+{{- end }}
 metadata:
   name: cozystack-api
   namespace: cozy-system
   labels:
     app: cozystack-api
 spec:
-  replicas: 2
+  {{- if not .Values.cozystackAPI.localK8sAPIEndpoint.enabled }}
+  replicas: {{ .Values.cozystackAPI.replicas }}
+  {{- end }}
   selector:
     matchLabels:
       app: cozystack-api
@@ -16,6 +22,35 @@ spec:
         app: cozystack-api
     spec:
       serviceAccountName: cozystack-api
+      {{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }}
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      {{- end }}
       containers:
       - name: cozystack-api
+        args:
+        - --tls-cert-file=/tmp/cozystack-api-certs/tls.crt
+        - --tls-private-key-file=/tmp/cozystack-api-certs/tls.key
+        {{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }}
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: status.hostIP
+        - name: KUBERNETES_SERVICE_PORT
+          value: "6443"
+        {{- end }}
         image: "{{ .Values.cozystackAPI.image }}"
+        ports:
+        - containerPort: 443
+          name: https
+        volumeMounts:
+          - name: cozystack-api-certs
+            mountPath: /tmp/cozystack-api-certs
+            readOnly: true
+      volumes:
+        - name: cozystack-api-certs
+          secret:
+            secretName: cozystack-api-cert
+            defaultMode: 0400

packages/system/cozystack-api/templates/service.yaml

@@ -4,9 +4,12 @@ metadata:
   name: cozystack-api
   namespace: cozy-system
 spec:
+  {{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }}
+  internalTrafficPolicy: Local
+  {{- end }}
   ports:
   - port: 443
     protocol: TCP
-    targetPort: 443
+    targetPort: https
   selector:
     app: cozystack-api

packages/system/cozystack-api/values.yaml

@@ -1,2 +1,5 @@
 cozystackAPI:
   image: ghcr.io/cozystack/cozystack/cozystack-api:v0.37.5@sha256:1166252a96f14f4bcf4369577151aa3c372dabf171da9a8dbd28bab81889ac87
+  localK8sAPIEndpoint:
+    enabled: true
+  replicas: 2

packages/system/lineage-controller-webhook/templates/daemonset.yaml

@@ -26,6 +26,16 @@ spec:
       containers:
       - name: lineage-controller-webhook
         image: "{{ .Values.lineageControllerWebhook.image }}"
+        {{- if .Values.lineageControllerWebhook.localK8sAPIEndpoint.enabled }}
+        env:
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: status.hostIP
+        - name: KUBERNETES_SERVICE_PORT
+          value: "6443"
+        {{- end }}
         args:
         {{- if .Values.lineageControllerWebhook.debug }}
         - --zap-log-level=debug

packages/system/lineage-controller-webhook/values.yaml

@@ -1,3 +1,5 @@
 lineageControllerWebhook:
   image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v0.37.5@sha256:df3bd847e9a733a9e9e4b4ebb6a48a95405c68c663837a72f613ace2904242bd
   debug: false
+  localK8sAPIEndpoint:
+    enabled: true