From eb3508a7dda9fdee4d01d985df06282842bfd5bc Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Tue, 20 Oct 2026 15:59:02 +0300 Subject: [PATCH] feat(cozystack-controller): grant the GPU controller its writes The binder writes finalizers and status on GPU classes, claims and nodes, creates and deletes the template claims of a VMI's claimTemplate entries, updates a gated launcher pod to pin and release it, deletes a pinned launcher pod its node refuses, and deletes the unscheduled VMI of a claim that must leave its node once that VMI's pod is pinned. RBAC cannot narrow the pod rights to launcher pods. It reports on a VMI, through events.k8s.io, why its pod stays gated. The settings ConfigMap is the only ConfigMap the controller writes, so its rights live in a Role in cozy-system rather than the ClusterRole: get, update and patch are scoped to that object by name, and create, which RBAC cannot scope by name, is scoped by the namespace. With --metrics-secure the metrics endpoint authenticates and authorizes each scrape through TokenReview and SubjectAccessReview, so the controller needs create on both for the GPU metrics to be scraped. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../cozystack-controller/templates/rbac.yaml | 69 ++++++++++ .../cozystack-controller/tests/rbac_test.yaml | 129 +++++++++++++++++- 2 files changed, 197 insertions(+), 1 deletion(-) diff --git a/packages/system/cozystack-controller/templates/rbac.yaml b/packages/system/cozystack-controller/templates/rbac.yaml index e40ab9080..28b84d055 100644 --- a/packages/system/cozystack-controller/templates/rbac.yaml +++ b/packages/system/cozystack-controller/templates/rbac.yaml @@ -67,9 +67,78 @@ rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# The GPU binder adds finalizers and places claims. The create right is for +# the template claims made for a VMI's claimTemplate entries, which the +# controller deletes with their VMI. +- apiGroups: ["gpu.cozystack.io"] + resources: ["gpuclasses", "gpuclaims"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: ["gpu.cozystack.io"] + resources: ["gpuclaims"] + verbs: ["create", "delete"] +- apiGroups: ["gpu.cozystack.io"] + resources: ["gpuclasses/status", "gpuclaims/status", "gpunodes/status"] + verbs: ["get", "update", "patch"] +# The GPU controller releases a gated virt-launcher pod by pinning it to the +# node of its claims and removing the gate in one update, and deletes a +# pinned launcher pod its node refuses, since a pin cannot be undone. The +# grant is cluster-wide, like the Secret writes above: tenant namespaces +# appear at runtime and nothing binds controller roles per tenant, and RBAC +# cannot narrow it to launcher pods. The reads come from the rule below. +- apiGroups: [""] + resources: ["pods"] + verbs: ["update", "delete"] +# The GPU controller reports on a VMI why its launcher pod stays gated. +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] +# The GPU controller deletes the unscheduled VMI of a claim that has to +# leave its node once the VMI's pod is pinned, so the claim becomes +# unattached and can move; a VMI whose pod is still gated is left alone. +- apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["delete"] +# --metrics-secure authenticates and authorizes each scrape through the API +# server. +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] # Leader election (--leader-elect) is granted by a namespaced Role in cozy-system # (see leader-election-role.yaml): the Lease lives in the controller's own # namespace, so the write does not need to reach every namespace. - apiGroups: ['*'] resources: ['*'] verbs: ["get", "list", "watch"] +--- +# The GPU controller writes one ConfigMap, the GPU settings in cozy-system. +# create cannot be scoped by name, so it is scoped by namespace instead. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cozystack-controller-gpu-settings + namespace: cozy-system +rules: +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cozystack-gpu-settings"] + verbs: ["get", "update", "patch"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cozystack-controller-gpu-settings + namespace: cozy-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cozystack-controller-gpu-settings +subjects: +- kind: ServiceAccount + name: cozystack-controller + namespace: cozy-system diff --git a/packages/system/cozystack-controller/tests/rbac_test.yaml b/packages/system/cozystack-controller/tests/rbac_test.yaml index 891e28870..61465ac72 100644 --- a/packages/system/cozystack-controller/tests/rbac_test.yaml +++ b/packages/system/cozystack-controller/tests/rbac_test.yaml @@ -1,4 +1,4 @@ -suite: cozystack-controller ClusterRole grants the shared Secret write +suite: cozystack-controller ClusterRole grants templates: - templates/rbac.yaml @@ -36,3 +36,130 @@ tests: apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + + - it: lets the GPU binder place claims and write class, claim and node status + documentSelector: + path: kind + value: ClusterRole + asserts: + - contains: + path: rules + content: + apiGroups: ["gpu.cozystack.io"] + resources: ["gpuclasses", "gpuclaims"] + verbs: ["get", "list", "watch", "update"] + - contains: + path: rules + content: + apiGroups: ["gpu.cozystack.io"] + resources: ["gpuclaims"] + verbs: ["create", "delete"] + - contains: + path: rules + content: + apiGroups: ["gpu.cozystack.io"] + resources: ["gpuclasses/status", "gpuclaims/status", "gpunodes/status"] + verbs: ["get", "update", "patch"] + + - it: lets the GPU controller release gated launcher pods and delete refused ones + documentSelector: + path: kind + value: ClusterRole + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["pods"] + verbs: ["update", "delete"] + + - it: lets the GPU controller report on a VMI why its pod waits + documentSelector: + path: kind + value: ClusterRole + asserts: + - contains: + path: rules + content: + apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "patch"] + + - it: lets the GPU controller delete only VMIs among KubeVirt objects + documentSelector: + path: kind + value: ClusterRole + asserts: + - contains: + path: rules + content: + apiGroups: ["kubevirt.io"] + resources: ["virtualmachineinstances"] + verbs: ["delete"] + + - it: lets the secure metrics endpoint authenticate and authorize scrapes + documentSelector: + path: kind + value: ClusterRole + asserts: + - contains: + path: rules + content: + apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] + - contains: + path: rules + content: + apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["create"] + + - it: grants ConfigMap writes nowhere cluster-wide + documentSelector: + path: kind + value: ClusterRole + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps"] + any: true + + - it: scopes ConfigMap writes to the GPU settings object in cozy-system + documentSelector: + path: kind + value: Role + asserts: + - equal: + path: metadata.namespace + value: cozy-system + - contains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cozystack-gpu-settings"] + verbs: ["get", "update", "patch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps"] + verbs: ["create"] + + - it: binds the GPU settings Role to the controller's ServiceAccount + documentSelector: + path: kind + value: RoleBinding + asserts: + - equal: + path: roleRef.name + value: cozystack-controller-gpu-settings + - contains: + path: subjects + content: + kind: ServiceAccount + name: cozystack-controller + namespace: cozy-system