From 3e0217bbbaf2327d530c5d1856c498ffeb87eef6 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 2 Feb 2026 20:13:39 +0100 Subject: [PATCH] fix(cluster-autoscaler): add RBAC rules for leader election leases Add coordination.k8s.io/leases permissions to Role via additionalRules. This fixes leader election failures in the cluster-autoscaler. Also add documentation for Hetzner Cloud setup with Talos Linux. Co-Authored-By: Claude Signed-off-by: Andrei Kvapil --- .../system/cluster-autoscaler/docs/hetzner.md | 237 ++++++++++++++++++ .../cluster-autoscaler/values-azure.yaml | 12 +- .../cluster-autoscaler/values-hetzner.yaml | 12 +- 3 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 packages/system/cluster-autoscaler/docs/hetzner.md diff --git a/packages/system/cluster-autoscaler/docs/hetzner.md b/packages/system/cluster-autoscaler/docs/hetzner.md new file mode 100644 index 000000000..2de9c49ae --- /dev/null +++ b/packages/system/cluster-autoscaler/docs/hetzner.md @@ -0,0 +1,237 @@ +# Cluster Autoscaler for Hetzner Cloud + +This guide explains how to configure cluster-autoscaler for automatic node scaling in Hetzner Cloud with Talos Linux. + +## Prerequisites + +- Hetzner Cloud account with API token +- `hcloud` CLI installed +- Existing Talos Kubernetes cluster +- Talos worker machine config + +## Step 1: Create Talos Image in Hetzner Cloud + +Hetzner doesn't support direct image uploads, so we need to create a snapshot via a temporary server. + +### 1.1 Configure hcloud CLI + +```bash +export HCLOUD_TOKEN="" +``` + +### 1.2 Create temporary server in rescue mode + +```bash +# Create server (without starting) +hcloud server create \ + --name talos-image-builder \ + --type cpx22 \ + --image ubuntu-24.04 \ + --location fsn1 \ + --ssh-key \ + --start-after-create=false + +# Enable rescue mode and start +hcloud server enable-rescue --type linux64 --ssh-key talos-image-builder +hcloud server poweron talos-image-builder +``` + +### 1.3 Get server IP and write Talos image + +```bash +# Get server IP +SERVER_IP=$(hcloud server ip talos-image-builder) + +# SSH into rescue mode and write image +ssh root@$SERVER_IP + +# Inside rescue mode: +wget -O- "https://factory.talos.dev/image///hcloud-amd64.raw.xz" \ + | xz -d \ + | dd of=/dev/sda bs=4M status=progress +sync +exit +``` + +Get your schematic ID from https://factory.talos.dev with required extensions: +- `siderolabs/qemu-guest-agent` (required for Hetzner) +- Other extensions as needed (zfs, drbd, etc.) + +### 1.4 Create snapshot and cleanup + +```bash +# Power off and create snapshot +hcloud server poweroff talos-image-builder +hcloud server create-image --type snapshot --description "Talos v1.11.6" talos-image-builder + +# Get snapshot ID (save this for later) +hcloud image list --type snapshot + +# Delete temporary server +hcloud server delete talos-image-builder +``` + +## Step 2: Create Kubernetes Secrets + +### 2.1 Create namespace (if not exists) + +```bash +kubectl create namespace cozy-cluster-autoscaler-hetzner +``` + +### 2.2 Create secret with Hetzner API token + +```bash +kubectl -n cozy-cluster-autoscaler-hetzner create secret generic hetzner-credentials \ + --from-literal=token= +``` + +### 2.3 Create secret with Talos machine config + +The machine config must be base64-encoded for the `HCLOUD_CLOUD_INIT` environment variable. + +```bash +# Encode your worker.yaml +cat worker.yaml | base64 -w0 > worker.b64 + +# Create secret +kubectl -n cozy-cluster-autoscaler-hetzner create secret generic talos-config \ + --from-file=cloud-init=worker.b64 +``` + +## Step 3: Configure Cluster Autoscaler + +Create or update your values file for the cluster-autoscaler-hetzner package: + +```yaml +cluster-autoscaler: + cloudProvider: hetzner + + autoscalingGroups: + - name: workers-fsn1 + minSize: 0 + maxSize: 10 + instanceType: CPX21 # 3 vCPU, 4GB RAM + region: FSN1 + + extraEnv: + HCLOUD_IMAGE: "" + HCLOUD_NETWORK: "" # Optional: private network + HCLOUD_FIREWALL: "" # Optional: firewall + HCLOUD_SSH_KEY: "" # Optional: SSH key + HCLOUD_PUBLIC_IPV4: "true" + HCLOUD_PUBLIC_IPV6: "false" + + extraEnvSecrets: + HCLOUD_TOKEN: + name: hetzner-credentials + key: token + HCLOUD_CLOUD_INIT: + name: talos-config + key: cloud-init +``` + +## Step 4: Deploy + +### Via Cozystack Package + +Update the Package resource with your configuration: + +```yaml +apiVersion: cozystack.io/v1alpha1 +kind: Package +metadata: + name: cozystack.cluster-autoscaler-hetzner +spec: + variant: default + components: + cluster-autoscaler-hetzner: + values: + cluster-autoscaler: + autoscalingGroups: + - name: workers-fsn1 + minSize: 0 + maxSize: 10 + instanceType: cpx22 + region: FSN1 + extraEnv: + HCLOUD_IMAGE: "" + HCLOUD_SSH_KEY: "" + HCLOUD_PUBLIC_IPV4: "true" + HCLOUD_PUBLIC_IPV6: "false" + extraEnvSecrets: + HCLOUD_TOKEN: + name: hetzner-credentials + key: token + HCLOUD_CLOUD_INIT: + name: talos-config + key: cloud-init +``` + +Apply with: +```bash +kubectl apply -f package.yaml +``` + +### Via Helm (direct) + +```bash +helm upgrade --install cluster-autoscaler-hetzner \ + ./packages/system/cluster-autoscaler \ + -n cozy-cluster-autoscaler-hetzner \ + -f values-hetzner.yaml \ + -f my-values.yaml +``` + +## Step 5: Verify + +```bash +# Check autoscaler logs +kubectl -n cozy-cluster-autoscaler-hetzner logs -l app.kubernetes.io/name=cluster-autoscaler -f + +# Check autoscaler status +kubectl -n cozy-cluster-autoscaler-hetzner get configmap cluster-autoscaler-status -o yaml + +# Test scale-up by creating pending pods +kubectl run test-pending --image=nginx --requests='cpu=2,memory=4Gi' +``` + +## Hetzner Server Types + +| Type | vCPU | RAM | Good for | +|------|------|-----|----------| +| cpx22 | 2 | 4GB | Small workloads | +| cpx32 | 4 | 8GB | General purpose | +| cpx42 | 8 | 16GB | Medium workloads | +| cpx52 | 16 | 32GB | Large workloads | +| ccx13 | 2 dedicated | 8GB | CPU-intensive | +| ccx23 | 4 dedicated | 16GB | CPU-intensive | +| ccx33 | 8 dedicated | 32GB | CPU-intensive | +| cax11 | 2 ARM | 4GB | ARM workloads | +| cax21 | 4 ARM | 8GB | ARM workloads | + +> **Note**: Some older server types (cpx11, cpx21, etc.) may be unavailable in certain regions. + +## Hetzner Regions + +- `FSN1` - Falkenstein, Germany +- `NBG1` - Nuremberg, Germany +- `HEL1` - Helsinki, Finland +- `ASH` - Ashburn, USA +- `HIL` - Hillsboro, USA + +## Troubleshooting + +### Nodes not joining cluster + +1. Check that machine config is correct and base64-encoded +2. Verify network connectivity (private network, firewall rules) +3. Check Talos image has `qemu-guest-agent` extension + +### Scale-down not working + +Talos caches absent nodes for up to 30 minutes. Wait or restart autoscaler. + +### Image not found + +Verify snapshot ID exists: `hcloud image list --type snapshot` diff --git a/packages/system/cluster-autoscaler/values-azure.yaml b/packages/system/cluster-autoscaler/values-azure.yaml index 7de2a0f97..7720481a1 100644 --- a/packages/system/cluster-autoscaler/values-azure.yaml +++ b/packages/system/cluster-autoscaler/values-azure.yaml @@ -1,4 +1,12 @@ cluster-autoscaler: cloudProvider: azure - extraArgs: - leader-elect-resource-name: azure-cluster-autoscaler + rbac: + additionalRules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update diff --git a/packages/system/cluster-autoscaler/values-hetzner.yaml b/packages/system/cluster-autoscaler/values-hetzner.yaml index 4d07736d0..8e19fc53c 100644 --- a/packages/system/cluster-autoscaler/values-hetzner.yaml +++ b/packages/system/cluster-autoscaler/values-hetzner.yaml @@ -1,4 +1,12 @@ cluster-autoscaler: cloudProvider: hetzner - extraArgs: - leader-elect-resource-name: hetzner-cluster-autoscaler + rbac: + additionalRules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update