mirror of
https://github.com/cozystack/cozystack.git
synced 2026-10-29 16:17:20 +09:00
Replace ingress-nginx as the default ingress layer for cozystack-native
services with Cilium Gateway API, materialised per-tenant via a new
TenantGateway CRD reconciled by cozystack-controller.
The extra/gateway chart renders one gateway.cozystack.io/v1alpha1
TenantGateway per tenant; the controller owns the resulting Gateway,
per-tenant Issuer, Certificate(s), and the http->https redirect
HTTPRoute. This avoids the helm-vs-controller race on
Gateway.spec.listeners when route-driven listeners are added.
publishing.certificates.solver selects HTTP-01 (default; per-app HTTPS
listener + per-app Certificate) or DNS-01 (opt-in; single wildcard
Certificate per apex; providers: cloudflare, route53, digitalocean,
rfc2136). HTTP-01 keeps zero-config per-app onboarding; DNS-01 saves
Let's Encrypt budget on apex-shared clusters.
Security model -- seven independent layers, all admission-time gates
fail-closed:
1. Listener allowedRoutes namespace whitelist via the
kube-apiserver-written kubernetes.io/metadata.name label;
plain-HTTP listener is strictly narrower than HTTPS, and HTTPS /
passthrough listeners restrict allowedRoutes.kinds to
HTTPRoute / TLSRoute respectively.
2. cozystack-gateway-hostname-policy VAP on Gateway.
3. cozystack-gateway-attached-namespaces-policy VAP on Package --
blocks tenant-* in gateway.attachedNamespaces.
4. cozystack-tenant-host-policy VAP on Tenant -- blocks spec.host
writes from non-trusted callers.
5. cozystack-namespace-host-label-policy VAP on Namespace -- blocks
writes to namespace.cozystack.io/host from non-trusted callers.
6. Render-time fail in cozystack-basics for tenant-* in
gateway.attachedNamespaces.
7. cozystack-route-hostname-policy VAP on HTTPRoute / TLSRoute,
scoped to tenant-* namespaces.
Within-apex cross-namespace hostname conflict is resolved at reconcile
time by a HostnameConflict condition (cozy-* namespace wins).
Five reconcile paths refuse to rewrite pre-existing objects with the
controller-derived name but no OwnerReference back to the
TenantGateway: Gateway, redirect HTTPRoute, per-tenant Issuer, wildcard
Certificate, per-listener Certificate.
Derived-apex tenants (tenants whose host is a subdomain of a parent
tenant's) auto-enable Gateway when the platform-level gateway.enabled
is on.
Bonus fix: pkg/registry/apps/application/rest.go was ignoring the
createValidation and deleteValidation callbacks that genericapiserver
hands to every storage method, silently bypassing admission for Create
and Delete on every apps.cozystack.io resource (Tenant, MariaDB,
Postgres, Kubernetes, ...). Aligned with the existing updateValidation
pattern.
Always-on changes: cilium.envoy.enabled + cilium.gatewayAPI.enabled
flip to true (extra cilium-envoy DaemonSet, ~100 MB RAM/node idle),
and the cozystack-api admission-chain fix. Everything else is gated
behind gateway.enabled=false and tenant.spec.gateway=false; existing
clusters see no behavioural change until an operator opts in.
Tests: 60+ Go cases in internal/controller/tenantgateway/, admission-
chain regression tests in pkg/registry/apps/application/, helm-unittest
across 7 charts (extra/gateway, core/platform, apps/tenant,
system/cozystack-basics, system/cert-manager-issuers,
system/cozystack-api, system/dashboard), and 14 e2e bats scenarios in
hack/e2e-apps/gateway.bats. New make test-controllers target wired
into pull-requests.yaml.
Deps: sigs.k8s.io/gateway-api v1.4.1, cert-manager v1.17.4.
Assisted-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>
232 lines
7.4 KiB
Go
232 lines
7.4 KiB
Go
package controller
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
|
)
|
|
|
|
func newAppDefScheme(t *testing.T) *runtime.Scheme {
|
|
t.Helper()
|
|
scheme := runtime.NewScheme()
|
|
if err := cozyv1alpha1.AddToScheme(scheme); err != nil {
|
|
t.Fatalf("add cozyv1alpha1: %v", err)
|
|
}
|
|
if err := appsv1.AddToScheme(scheme); err != nil {
|
|
t.Fatalf("add appsv1: %v", err)
|
|
}
|
|
return scheme
|
|
}
|
|
|
|
// TestApplicationDefinition_NoEventsIsNoop pins the early-exit path:
|
|
// when no event has been recorded yet (lastEvent zero), Reconcile must
|
|
// return an empty Result without touching the deployment or computing
|
|
// hashes.
|
|
func TestApplicationDefinition_NoEventsIsNoop(t *testing.T) {
|
|
scheme := newAppDefScheme(t)
|
|
fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
|
|
|
|
r := &ApplicationDefinitionReconciler{
|
|
Client: fakeClient,
|
|
Scheme: scheme,
|
|
Debounce: 5 * time.Second,
|
|
}
|
|
|
|
res, err := r.Reconcile(context.TODO(), reconcile.Request{})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 {
|
|
t.Fatalf("expected empty result, got %+v", res)
|
|
}
|
|
}
|
|
|
|
// TestApplicationDefinition_WithinDebounceRequeues pins the debounce-window
|
|
// path: an event was just recorded, within the debounce window. Reconcile
|
|
// must return a RequeueAfter equal to the remaining wait, no work done.
|
|
func TestApplicationDefinition_WithinDebounceRequeues(t *testing.T) {
|
|
scheme := newAppDefScheme(t)
|
|
fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
|
|
|
|
r := &ApplicationDefinitionReconciler{
|
|
Client: fakeClient,
|
|
Scheme: scheme,
|
|
Debounce: 5 * time.Second,
|
|
}
|
|
r.lastEvent = time.Now()
|
|
|
|
res, err := r.Reconcile(context.TODO(), reconcile.Request{})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if res.RequeueAfter <= 0 {
|
|
t.Fatalf("expected positive RequeueAfter, got %+v", res)
|
|
}
|
|
if res.RequeueAfter > 5*time.Second {
|
|
t.Fatalf("RequeueAfter exceeds debounce window: %v", res.RequeueAfter)
|
|
}
|
|
}
|
|
|
|
// TestApplicationDefinition_AlreadyHandledIsNoop pins the lastHandled
|
|
// gate: lastHandled >= lastEvent means this event has already been
|
|
// processed; Reconcile is a no-op.
|
|
func TestApplicationDefinition_AlreadyHandledIsNoop(t *testing.T) {
|
|
scheme := newAppDefScheme(t)
|
|
fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
|
|
|
|
now := time.Now().Add(-10 * time.Second)
|
|
r := &ApplicationDefinitionReconciler{
|
|
Client: fakeClient,
|
|
Scheme: scheme,
|
|
Debounce: 5 * time.Second,
|
|
lastEvent: now,
|
|
lastHandled: now,
|
|
}
|
|
|
|
res, err := r.Reconcile(context.TODO(), reconcile.Request{})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 {
|
|
t.Fatalf("expected empty result, got %+v", res)
|
|
}
|
|
}
|
|
|
|
// TestApplicationDefinition_MissingDeploymentIgnored pins the
|
|
// IgnoreNotFound path: when the cozystack-api Deployment is absent
|
|
// (initial install in progress, etc.), Reconcile must not return an
|
|
// error.
|
|
func TestApplicationDefinition_MissingDeploymentIgnored(t *testing.T) {
|
|
scheme := newAppDefScheme(t)
|
|
fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
|
|
|
|
r := &ApplicationDefinitionReconciler{
|
|
Client: fakeClient,
|
|
Scheme: scheme,
|
|
Debounce: 5 * time.Second,
|
|
lastEvent: time.Now().Add(-10 * time.Second),
|
|
}
|
|
|
|
res, err := r.Reconcile(context.TODO(), reconcile.Request{})
|
|
if err != nil {
|
|
t.Fatalf("expected nil error on missing deployment, got %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 {
|
|
t.Fatalf("expected empty result, got %+v", res)
|
|
}
|
|
}
|
|
|
|
// TestApplicationDefinition_HashChangePatchesDeployment pins the happy
|
|
// path: an event has elapsed past debounce, the deployment exists with
|
|
// either no hash annotation or a stale hash, Reconcile must patch the
|
|
// pod template's cozystack.io/config-hash annotation.
|
|
func TestApplicationDefinition_HashChangePatchesDeployment(t *testing.T) {
|
|
scheme := newAppDefScheme(t)
|
|
|
|
dep := &appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "cozystack-api",
|
|
Namespace: "cozy-system",
|
|
},
|
|
}
|
|
appDef := &cozyv1alpha1.ApplicationDefinition{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "harbor"},
|
|
}
|
|
|
|
fakeClient := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithObjects(dep, appDef).
|
|
Build()
|
|
|
|
r := &ApplicationDefinitionReconciler{
|
|
Client: fakeClient,
|
|
Scheme: scheme,
|
|
Debounce: 5 * time.Second,
|
|
lastEvent: time.Now().Add(-10 * time.Second),
|
|
}
|
|
|
|
if _, err := r.Reconcile(context.TODO(), reconcile.Request{}); err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
|
|
got := &appsv1.Deployment{}
|
|
if err := fakeClient.Get(context.TODO(), types.NamespacedName{Name: "cozystack-api", Namespace: "cozy-system"}, got); err != nil {
|
|
t.Fatalf("get deployment: %v", err)
|
|
}
|
|
hash, ok := got.Spec.Template.Annotations["cozystack.io/config-hash"]
|
|
if !ok || hash == "" {
|
|
t.Fatalf("expected config-hash annotation set, got %q (present=%v)", hash, ok)
|
|
}
|
|
}
|
|
|
|
// TestApplicationDefinition_HashUnchangedSkipsPatch pins the no-op
|
|
// path: same hash already on the deployment means no patch is issued.
|
|
// We seed the deployment with the hash that the reconciler will compute
|
|
// and verify the lastHandled marker advances without changing the
|
|
// annotation.
|
|
func TestApplicationDefinition_HashUnchangedSkipsPatch(t *testing.T) {
|
|
scheme := newAppDefScheme(t)
|
|
|
|
appDef := &cozyv1alpha1.ApplicationDefinition{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "harbor"},
|
|
}
|
|
|
|
// First, run the reconciler against a deployment with empty annotation
|
|
// to capture the hash it computes.
|
|
dep := &appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "cozystack-api",
|
|
Namespace: "cozy-system",
|
|
},
|
|
}
|
|
fakeClient := fake.NewClientBuilder().
|
|
WithScheme(scheme).
|
|
WithObjects(dep, appDef).
|
|
Build()
|
|
r := &ApplicationDefinitionReconciler{
|
|
Client: fakeClient,
|
|
Scheme: scheme,
|
|
Debounce: 5 * time.Second,
|
|
lastEvent: time.Now().Add(-10 * time.Second),
|
|
}
|
|
if _, err := r.Reconcile(context.TODO(), reconcile.Request{}); err != nil {
|
|
t.Fatalf("first reconcile: %v", err)
|
|
}
|
|
|
|
got := &appsv1.Deployment{}
|
|
if err := fakeClient.Get(context.TODO(), types.NamespacedName{Name: "cozystack-api", Namespace: "cozy-system"}, got); err != nil {
|
|
t.Fatalf("get deployment: %v", err)
|
|
}
|
|
expectedHash := got.Spec.Template.Annotations["cozystack.io/config-hash"]
|
|
|
|
// Now drive a second reconcile cycle with a fresh event but the same
|
|
// state — hash will match, no patch should be needed. We can only
|
|
// verify the no-op via lastHandled advancing; the patch itself is a
|
|
// no-op so the annotation stays identical.
|
|
r.lastEvent = time.Now().Add(-10 * time.Second)
|
|
r.lastHandled = time.Time{}
|
|
if _, err := r.Reconcile(context.TODO(), reconcile.Request{}); err != nil {
|
|
t.Fatalf("second reconcile: %v", err)
|
|
}
|
|
|
|
again := &appsv1.Deployment{}
|
|
if err := fakeClient.Get(context.TODO(), types.NamespacedName{Name: "cozystack-api", Namespace: "cozy-system"}, again); err != nil {
|
|
t.Fatalf("get deployment after second reconcile: %v", err)
|
|
}
|
|
if again.Spec.Template.Annotations["cozystack.io/config-hash"] != expectedHash {
|
|
t.Fatalf("hash mutated when it should have been stable: was %q now %q",
|
|
expectedHash, again.Spec.Template.Annotations["cozystack.io/config-hash"])
|
|
}
|
|
if r.lastHandled.IsZero() {
|
|
t.Fatalf("expected lastHandled advanced after no-op reconcile")
|
|
}
|
|
}
|