fix(cozystack-basics): refuse a direct delete of a tenant namespace

A tenant namespace is meant to go away with its Tenant. The tenant
chart's pre-delete hook deletes the applications first and the tenant
modules such as etcd after them, and helm-controller deletes the
namespace at the end. A direct delete hands the namespace to the
namespace controller instead. It deletes the pods first and then
everything else together, so the tenant etcd and its etcd-ca-tls Secret
can be gone before the TenantControlPlanes that use them. Kamaji then
cannot reach the datastore to clean up, the TenantControlPlane finalizer
never clears and the namespace stays Terminating.

The policy refuses DELETE of a namespace labelled
tenant.cozystack.io/tenant-root, which the tenant chart puts on every
tenant namespace below tenant-root. tenant-root carries no such label
and stays with the no-delete guardrail. Two callers are let through.
helm-controller and its shards run as the flux ServiceAccount in
cozy-fluxcd and delete the namespace during a Tenant uninstall. The
garbage collector runs in kube-system and removes a child namespace
through its ownerReference once the parent namespace is gone. The
namespace controller only finalizes a namespace and never sends a
DELETE for it, so it is not affected. Everyone else is refused,
system:masters included, as with the no-delete guardrail. The message
names the Tenant to delete and the label to remove when the Tenant is
already gone.

The policy test helpers take the template path now, so the new test
compiles the shipped template with the apiserver CEL compiler like the
external TLS policy test does.

Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
This commit is contained in:
Yan Bondarenko committed 2026-10-21 04:30:33 +03:00
1 parent 1f92bcd20e
commit 6bb0fda706
7 files changed
+331 -11

No files matched your search

+37
View File
@@ -561,3 +561,40 @@ data:
version: "${version}"
EOF
}
@test "Tenant namespace delete policy refuses a direct delete and keeps the Tenant uninstall" {
if ! kubectl api-resources --api-group=admissionregistration.k8s.io \
2>/dev/null | grep -qw validatingadmissionpolicies; then
skip "ValidatingAdmissionPolicy API not available on this cluster"
fi
kubectl get validatingadmissionpolicy cozystack-tenant-namespace-delete-policy
kubectl get validatingadmissionpolicybinding cozystack-tenant-namespace-delete-policy
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Tenant
metadata:
name: deleteprobe
namespace: tenant-root
spec: {}
EOF
timeout 60 sh -ec 'until kubectl get hr/tenant-deleteprobe -n tenant-root >/dev/null 2>&1; do sleep 2; done'
kubectl wait hr/tenant-deleteprobe -n tenant-root --timeout=1m --for=condition=ready
local selected output rc still_there
selected=$(kubectl get namespaces --selector tenant.cozystack.io/tenant-root --field-selector metadata.name=tenant-deleteprobe -o name)
[ "$selected" = "namespace/tenant-deleteprobe" ]
output=$(kubectl delete namespace tenant-deleteprobe --dry-run=server 2>&1) && rc=0 || rc=$?
echo "kubectl delete exit=$rc, output=$output"
[ "$rc" -ne 0 ]
echo "$output" | grep -qF "ValidatingAdmissionPolicy 'cozystack-tenant-namespace-delete-policy'"
echo "$output" | grep -qF 'kubectl delete tenants.apps.cozystack.io deleteprobe --namespace tenant-root.'
kubectl delete tenants.apps.cozystack.io deleteprobe -n tenant-root --timeout=5m
kubectl wait namespace/tenant-deleteprobe --for=delete --timeout=5m
if ! still_there=$(kubectl get namespace tenant-deleteprobe --ignore-not-found -o name); then
echo "FAIL: reading the tenant-deleteprobe namespace did not answer"; false
fi
if [ -n "$still_there" ]; then echo "FAIL: deleting the Tenant must remove its namespace"; false; fi
}
@@ -0,0 +1,59 @@
{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }}
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
name: cozystack-tenant-namespace-delete-policy
labels:
internal.cozystack.io/managed-by-cozystack: ""
spec:
failurePolicy: Fail
matchConstraints:
resourceRules:
- apiGroups: [""]
apiVersions: ["v1"]
operations: ["DELETE"]
resources: ["namespaces"]
variables:
- name: platformCaller
expression: >-
has(request.userInfo.groups) && request.userInfo.groups.exists(g,
g == "system:serviceaccounts:cozy-fluxcd" ||
g == "system:serviceaccounts:kube-system")
- name: name
expression: oldObject.metadata.name
- name: cut
expression: variables.name.lastIndexOf("-")
- name: tenant
expression: variables.name.substring(variables.cut + 1)
- name: parentNamespace
expression: >-
variables.cut > 0 && variables.name.substring(0, variables.cut) != "tenant"
? variables.name.substring(0, variables.cut)
: "tenant-root"
validations:
- expression: variables.platformCaller
messageExpression: >-
"Namespace " + variables.name + " belongs to the Tenant " + variables.tenant +
" in " + variables.parentNamespace + ". Delete the Tenant instead with kubectl delete tenants.apps.cozystack.io " +
variables.tenant + " --namespace " + variables.parentNamespace +
". Deleting the namespace directly can remove the tenant etcd before the Kubernetes clusters that use it and leave the namespace stuck in Terminating." +
" If the Tenant is already gone, remove the label first with kubectl label namespace " + variables.name +
" tenant.cozystack.io/tenant-root-"
reason: Forbidden
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding
metadata:
name: cozystack-tenant-namespace-delete-policy
labels:
internal.cozystack.io/managed-by-cozystack: ""
spec:
policyName: cozystack-tenant-namespace-delete-policy
validationActions: [Deny]
matchResources:
objectSelector:
matchExpressions:
- key: tenant.cozystack.io/tenant-root
operator: Exists
{{- end }}
@@ -1,5 +1,6 @@
suite: cozystack-basics VAP policies are gated on the ValidatingAdmissionPolicy API
templates:
- templates/tenant-namespace-delete-policy.yaml
- templates/route-hostname-policy.yaml
- templates/gateway-hostname-policy.yaml
- templates/ingress-hostname-policy.yaml
@@ -20,6 +21,9 @@ capabilities:
tests:
- it: none of the VAP policies render when the API is unavailable
asserts:
- hasDocuments:
count: 0
template: templates/tenant-namespace-delete-policy.yaml
- hasDocuments:
count: 0
template: templates/route-hostname-policy.yaml
@@ -0,0 +1,51 @@
suite: tenant namespace delete policy renders with its binding
templates:
- templates/tenant-namespace-delete-policy.yaml
release:
name: cozystack-basics
namespace: cozy-system
capabilities:
apiVersions:
- admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy
tests:
- it: renders the policy and a Deny binding to it
asserts:
- hasDocuments:
count: 2
- documentIndex: 0
equal:
path: kind
value: ValidatingAdmissionPolicy
- documentIndex: 0
equal:
path: metadata.name
value: cozystack-tenant-namespace-delete-policy
- documentIndex: 1
equal:
path: spec.policyName
value: cozystack-tenant-namespace-delete-policy
- documentIndex: 1
equal:
path: spec.validationActions
value: [Deny]
- it: matches only DELETE of core namespaces
asserts:
- documentIndex: 0
equal:
path: spec.matchConstraints.resourceRules
value:
- apiGroups: [""]
apiVersions: ["v1"]
operations: ["DELETE"]
resources: ["namespaces"]
- it: binds to the namespaces the tenant chart labels as descendants of tenant-root
asserts:
- documentIndex: 1
equal:
path: spec.matchResources.objectSelector
value:
matchExpressions:
- key: tenant.cozystack.io/tenant-root
operator: Exists
@@ -79,3 +79,9 @@ tests:
equal:
path: spec.chartRef.name
value: cozystack-tenant-application-default-tenant
- it: tenant-root namespace carries no tenant.cozystack.io/tenant-root label, which the tenant namespace delete policy selects on
asserts:
- documentIndex: 0
notExists:
path: metadata.labels["tenant.cozystack.io/tenant-root"]
@@ -41,14 +41,14 @@ import (
// expressions a cluster actually receives rather than a copy of them.
const externalTLSPolicyPath = "../../../../packages/system/cozystack-basics/templates/database-external-tls-policy.yaml"
// loadExternalTLSPolicy returns the policy and its binding from the chart
// template. The template's only directives are the capability guard lines
// around the manifests; any other directive would mean the test is no longer
// looking at what renders, so it fails instead of guessing.
func loadExternalTLSPolicy(t *testing.T) (*admissionregistrationv1.ValidatingAdmissionPolicy, *admissionregistrationv1.ValidatingAdmissionPolicyBinding) {
// loadPolicy returns the policy and its binding from a chart template. The
// template's only directives are the capability guard lines around the
// manifests; any other directive would mean the test is no longer looking at
// what renders, so it fails instead of guessing.
func loadPolicy(t *testing.T, path string) (*admissionregistrationv1.ValidatingAdmissionPolicy, *admissionregistrationv1.ValidatingAdmissionPolicyBinding) {
t.Helper()
raw, err := os.ReadFile(externalTLSPolicyPath)
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read policy template: %v", err)
}
@@ -92,11 +92,11 @@ func loadExternalTLSPolicy(t *testing.T) (*admissionregistrationv1.ValidatingAdm
return policy, binding
}
// compileExternalTLSPolicy mirrors compilePolicy in
// compilePolicy mirrors compilePolicy in
// k8s.io/apiserver/pkg/admission/plugin/policy/validating, which is
// unexported: the same composition environment, the same variable, match
// condition, validation and message compilers.
func compileExternalTLSPolicy(t *testing.T, policy *admissionregistrationv1.ValidatingAdmissionPolicy) validating.Validator {
func compilePolicy(t *testing.T, policy *admissionregistrationv1.ValidatingAdmissionPolicy) validating.Validator {
t.Helper()
compiler, err := plugincel.NewCompositedCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion()))
@@ -139,7 +139,7 @@ func compileExternalTLSPolicy(t *testing.T, policy *admissionregistrationv1.Vali
}
func TestDatabaseExternalTLSPolicy_Binding(t *testing.T) {
policy, binding := loadExternalTLSPolicy(t)
policy, binding := loadPolicy(t, externalTLSPolicyPath)
if binding.Spec.PolicyName != policy.Name {
t.Errorf("binding names policy %q, want %q", binding.Spec.PolicyName, policy.Name)
@@ -171,8 +171,8 @@ func TestDatabaseExternalTLSPolicy_Binding(t *testing.T) {
}
func TestDatabaseExternalTLSPolicy_Decisions(t *testing.T) {
policy, _ := loadExternalTLSPolicy(t)
validator := compileExternalTLSPolicy(t, policy)
policy, _ := loadPolicy(t, externalTLSPolicyPath)
validator := compilePolicy(t, policy)
tlsOn := map[string]any{"enabled": true}
tlsOff := map[string]any{"enabled": false}
@@ -0,0 +1,163 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package application
import (
"context"
"slices"
"strings"
"testing"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/admission/plugin/policy/validating"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"k8s.io/apiserver/pkg/authentication/user"
)
const tenantNamespaceDeletePolicyPath = "../../../../packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml"
func TestTenantNamespaceDeletePolicy_Binding(t *testing.T) {
policy, binding := loadPolicy(t, tenantNamespaceDeletePolicyPath)
if binding.Spec.PolicyName != policy.Name {
t.Errorf("binding names policy %q, want %q", binding.Spec.PolicyName, policy.Name)
}
if !slices.Equal(binding.Spec.ValidationActions, []admissionregistrationv1.ValidationAction{admissionregistrationv1.Deny}) {
t.Errorf("binding validationActions = %v, want [Deny]", binding.Spec.ValidationActions)
}
if policy.Spec.FailurePolicy == nil || *policy.Spec.FailurePolicy != admissionregistrationv1.Fail {
t.Errorf("failurePolicy must be Fail")
}
rules := policy.Spec.MatchConstraints.ResourceRules
if len(rules) != 1 {
t.Fatalf("want one resource rule, got %d", len(rules))
}
rule := rules[0]
if !slices.Equal(rule.APIGroups, []string{""}) || !slices.Equal(rule.APIVersions, []string{"v1"}) {
t.Errorf("rule must match core v1 only, got apiGroups %v apiVersions %v", rule.APIGroups, rule.APIVersions)
}
if !slices.Equal(rule.Operations, []admissionregistrationv1.OperationType{admissionregistrationv1.Delete}) {
t.Errorf("operations = %v, want [DELETE]", rule.Operations)
}
if !slices.Equal(rule.Resources, []string{"namespaces"}) {
t.Errorf("resources = %v, want [namespaces]", rule.Resources)
}
if binding.Spec.MatchResources == nil || binding.Spec.MatchResources.ObjectSelector == nil {
t.Fatal("binding must select namespaces by label")
}
selector, err := metav1.LabelSelectorAsSelector(binding.Spec.MatchResources.ObjectSelector)
if err != nil {
t.Fatalf("binding objectSelector: %v", err)
}
selected := []struct {
name string
labels labels.Set
want bool
}{
{name: "tenant namespace under tenant-root", labels: labels.Set{"tenant.cozystack.io/tenant-root": "", "tenant.cozystack.io/tenant-foo": "", "namespace.cozystack.io/host": "foo.example.org"}, want: true},
{name: "nested tenant namespace", labels: labels.Set{"tenant.cozystack.io/tenant-root": "", "tenant.cozystack.io/tenant-foo": "", "tenant.cozystack.io/tenant-foo-bar": ""}, want: true},
{name: "tenant-root namespace", labels: labels.Set{"platform.cozystack.io/no-delete": "true", "namespace.cozystack.io/host": "example.org", "namespace.cozystack.io/gateway": "tenant-root"}},
{name: "namespace named tenant-* without tenant labels", labels: labels.Set{"namespace.cozystack.io/host": "case-probe.example.org"}},
{name: "unlabelled namespace", labels: labels.Set{}},
}
for _, tc := range selected {
if got := selector.Matches(tc.labels); got != tc.want {
t.Errorf("%s: selected = %v, want %v", tc.name, got, tc.want)
}
}
}
func TestTenantNamespaceDeletePolicy_Decisions(t *testing.T) {
policy, _ := loadPolicy(t, tenantNamespaceDeletePolicyPath)
validator := compilePolicy(t, policy)
fluxGroups := []string{"system:serviceaccounts", "system:serviceaccounts:cozy-fluxcd", "system:authenticated"}
kubeSystemGroups := []string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"}
adminGroups := []string{"system:masters", "system:authenticated"}
cases := []struct {
name string
namespace string
user string
groups []string
denied bool
tenant string
parent string
}{
{name: "helm-controller uninstalling a Tenant", namespace: "tenant-foo", user: "system:serviceaccount:cozy-fluxcd:flux", groups: fluxGroups},
{name: "helm-controller uninstalling a nested Tenant", namespace: "tenant-foo-bar", user: "system:serviceaccount:cozy-fluxcd:flux", groups: fluxGroups},
{name: "garbage collector following a deleted parent namespace", namespace: "tenant-foo-bar", user: "system:serviceaccount:kube-system:generic-garbage-collector", groups: kubeSystemGroups},
{name: "cluster admin deleting a tenant namespace", namespace: "tenant-foo", user: "admin", groups: adminGroups, denied: true, tenant: "foo", parent: "tenant-root"},
{name: "cluster admin deleting a nested tenant namespace", namespace: "tenant-foo-bar", user: "admin", groups: adminGroups, denied: true, tenant: "bar", parent: "tenant-foo"},
{name: "cluster admin deleting a third-level tenant namespace", namespace: "tenant-foo-bar-baz", user: "admin", groups: adminGroups, denied: true, tenant: "baz", parent: "tenant-foo-bar"},
{name: "cluster admin deleting a labelled namespace without a dash", namespace: "standalone", user: "admin", groups: adminGroups, denied: true, tenant: "standalone", parent: "tenant-root"},
{name: "OIDC user bound to cluster-admin", namespace: "tenant-foo", user: "cluster-admin-user", groups: []string{"cozystack-cluster-admin", "system:authenticated"}, denied: true, tenant: "foo", parent: "tenant-root"},
{name: "cozy-system service account", namespace: "tenant-foo", user: "system:serviceaccount:cozy-system:cozystack-api", groups: []string{"system:serviceaccounts", "system:serviceaccounts:cozy-system", "system:authenticated"}, denied: true, tenant: "foo", parent: "tenant-root"},
{name: "tenant service account", namespace: "tenant-foo-bar", user: "system:serviceaccount:tenant-foo:default", groups: []string{"system:serviceaccounts", "system:serviceaccounts:tenant-foo", "system:authenticated"}, denied: true, tenant: "bar", parent: "tenant-foo"},
{name: "user without groups", namespace: "tenant-foo", user: "no-groups", denied: true, tenant: "foo", parent: "tenant-root"},
}
gvk := schema.GroupVersionKind{Version: "v1", Kind: "Namespace"}
gvr := schema.GroupVersionResource{Version: "v1", Resource: "namespaces"}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
oldObj := &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "v1",
"kind": "Namespace",
"metadata": map[string]any{
"name": tc.namespace,
"labels": map[string]any{"tenant.cozystack.io/tenant-root": ""},
},
}}
var newObj runtime.Object
attrs := admission.NewAttributesRecord(newObj, oldObj, gvk, tc.namespace, tc.namespace, gvr, "", admission.Delete, &metav1.DeleteOptions{}, false, &user.DefaultInfo{Name: tc.user, Groups: tc.groups})
versioned := &admission.VersionedAttributes{Attributes: attrs, VersionedObject: admission.NewLazyObject(newObj), VersionedOldObject: admission.NewLazyObject(oldObj), VersionedKind: gvk}
result := validator.Validate(context.Background(), gvr, versioned, nil, nil, celconfig.RuntimeCELCostBudget, nil)
var denials []validating.PolicyDecision
for _, d := range result.Decisions {
if d.Evaluation == validating.EvalError {
t.Fatalf("evaluation error: %s", d.Message)
}
if d.Action == validating.ActionDeny {
denials = append(denials, d)
}
}
if got := len(denials) > 0; got != tc.denied {
t.Fatalf("denied = %v, want %v (decisions %+v)", got, tc.denied, result.Decisions)
}
for _, d := range denials {
for _, want := range []string{
"kubectl delete tenants.apps.cozystack.io " + tc.tenant + " --namespace " + tc.parent + ".",
"kubectl label namespace " + tc.namespace + " tenant.cozystack.io/tenant-root-",
} {
if !strings.Contains(d.Message, want) {
t.Errorf("denial message must carry %q, got %q", want, d.Message)
}
}
}
})
}
}