From cd3ea570daa8bb78895b69cfe25c235e4ee7bf41 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Tue, 28 Oct 2025 18:28:03 +0300 Subject: [PATCH] [api,lineage] Ensure node-local traffic Since 0.37, many requests to the k8s API now go through a mutating webhook (lineage-controller-webhook). Since the lineage webhook makes multiple requests to the k8s API and, indirectly, to the Cozystack API server, each request for, e.g., creating a secret now causes a lot of chatter between the webhook, the k8s API, and the Cozystack API. When this happens cross-node or, worse yet, cross-zone, this can blow up the latency for simple requests. This patch changes the Cozystack API to a DaemonSet targetting controlplane nodes, configures its service for an `Local` internal traffic policy and adds environment variables indicating that the k8s API server is to be found at :6443, **not only for the Cozystack API, but also for the lineage-controller-webhook.** This is a valid configuration in most scenarios, including the default installation method on top of Talos Linux in Cozystack, however, if this is not valid in your environment, you must now set the values `.lineageControllerWebhook.localK8sAPIEndpoint.enabled` and `.cozystackAPI.localK8sAPIEndpoint.enabled` to `false` in the respective system Helm releases. ```release-note [api,lineage] Configure all chatter between the Lineage webhook, the Cozystack API server and the Kubernetes API server to be confined to a single controlplane node, improving k8s API latency. ``` Signed-off-by: Timofei Larkin --- .../cozystack-api/templates/apiservice.yaml | 6 ++- .../cozystack-api/templates/certmanager.yaml | 45 +++++++++++++++++++ .../cozystack-api/templates/deployment.yaml | 37 ++++++++++++++- .../cozystack-api/templates/service.yaml | 5 ++- packages/system/cozystack-api/values.yaml | 3 ++ .../templates/daemonset.yaml | 10 +++++ .../lineage-controller-webhook/values.yaml | 2 + 7 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 packages/system/cozystack-api/templates/certmanager.yaml diff --git a/packages/system/cozystack-api/templates/apiservice.yaml b/packages/system/cozystack-api/templates/apiservice.yaml index d0ab1185e..3cd3665b7 100644 --- a/packages/system/cozystack-api/templates/apiservice.yaml +++ b/packages/system/cozystack-api/templates/apiservice.yaml @@ -1,9 +1,10 @@ apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: + annotations: + cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/cozystack-api" name: v1alpha1.apps.cozystack.io spec: - insecureSkipTLSVerify: true group: apps.cozystack.io groupPriorityMinimum: 1000 versionPriority: 15 @@ -15,9 +16,10 @@ spec: apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: + annotations: + cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/cozystack-api" name: v1alpha1.core.cozystack.io spec: - insecureSkipTLSVerify: true group: core.cozystack.io groupPriorityMinimum: 1000 versionPriority: 15 diff --git a/packages/system/cozystack-api/templates/certmanager.yaml b/packages/system/cozystack-api/templates/certmanager.yaml new file mode 100644 index 000000000..def27bd1e --- /dev/null +++ b/packages/system/cozystack-api/templates/certmanager.yaml @@ -0,0 +1,45 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: cozystack-api-selfsigned + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cozystack-api-ca + namespace: {{ .Release.Namespace }} +spec: + secretName: cozystack-api-ca + duration: 43800h # 5 years + commonName: cozystack-api-ca + issuerRef: + name: cozystack-api-selfsigned + isCA: true +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: cozystack-api-ca + namespace: {{ .Release.Namespace }} +spec: + ca: + secretName: cozystack-api-ca +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cozystack-api + namespace: {{ .Release.Namespace }} +spec: + secretName: cozystack-api-cert + duration: 8760h + renewBefore: 720h + issuerRef: + name: cozystack-api-ca + commonName: cozystack-api + dnsNames: + - cozystack-api + - cozystack-api.{{ .Release.Namespace }}.svc diff --git a/packages/system/cozystack-api/templates/deployment.yaml b/packages/system/cozystack-api/templates/deployment.yaml index 46779d2be..1a63a0e0d 100644 --- a/packages/system/cozystack-api/templates/deployment.yaml +++ b/packages/system/cozystack-api/templates/deployment.yaml @@ -1,12 +1,18 @@ apiVersion: apps/v1 +{{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} +kind: DaemonSet +{{- else }} kind: Deployment +{{- end }} metadata: name: cozystack-api namespace: cozy-system labels: app: cozystack-api spec: - replicas: 2 + {{- if not .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} + replicas: {{ .Values.cozystackAPI.replicas }} + {{- end }} selector: matchLabels: app: cozystack-api @@ -16,6 +22,35 @@ spec: app: cozystack-api spec: serviceAccountName: cozystack-api + {{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} + nodeSelector: + node-role.kubernetes.io/control-plane: "" + {{- end }} containers: - name: cozystack-api + args: + - --tls-cert-file=/tmp/cozystack-api-certs/tls.crt + - --tls-private-key-file=/tmp/cozystack-api-certs/tls.key + {{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} + env: + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + {{- end }} image: "{{ .Values.cozystackAPI.image }}" + ports: + - containerPort: 443 + name: https + volumeMounts: + - name: cozystack-api-certs + mountPath: /tmp/cozystack-api-certs + readOnly: true + volumes: + - name: cozystack-api-certs + secret: + secretName: cozystack-api-cert + defaultMode: 0400 diff --git a/packages/system/cozystack-api/templates/service.yaml b/packages/system/cozystack-api/templates/service.yaml index 2dcd618bf..abe67abc6 100644 --- a/packages/system/cozystack-api/templates/service.yaml +++ b/packages/system/cozystack-api/templates/service.yaml @@ -4,9 +4,12 @@ metadata: name: cozystack-api namespace: cozy-system spec: + {{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} + internalTrafficPolicy: Local + {{- end }} ports: - port: 443 protocol: TCP - targetPort: 443 + targetPort: https selector: app: cozystack-api diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index 2928457db..59cf0f98a 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,2 +1,5 @@ cozystackAPI: image: ghcr.io/cozystack/cozystack/cozystack-api:v0.37.5@sha256:1166252a96f14f4bcf4369577151aa3c372dabf171da9a8dbd28bab81889ac87 + localK8sAPIEndpoint: + enabled: true + replicas: 2 diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/daemonset.yaml index 177bcd8b8..22074e1dc 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/daemonset.yaml @@ -26,6 +26,16 @@ spec: containers: - name: lineage-controller-webhook image: "{{ .Values.lineageControllerWebhook.image }}" + {{- if .Values.lineageControllerWebhook.localK8sAPIEndpoint.enabled }} + env: + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + {{- end }} args: {{- if .Values.lineageControllerWebhook.debug }} - --zap-log-level=debug diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 53570a7cd..d04d71c6f 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,3 +1,5 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v0.37.5@sha256:df3bd847e9a733a9e9e4b4ebb6a48a95405c68c663837a72f613ace2904242bd debug: false + localK8sAPIEndpoint: + enabled: true