package lineagecontrollerwebhook import ( "context" "encoding/json" "errors" "fmt" "maps" "strings" "github.com/cozystack/cozystack/pkg/lineage" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utiljson "k8s.io/apimachinery/pkg/util/json" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" schedulerapi "github.com/cozystack/cozystack-scheduler/pkg/apis/v1alpha1" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" ) var ( NoAncestors = errors.New("no managed apps found in lineage") AncestryAmbiguous = errors.New("object ancestry is ambiguous") ) const ( ManagedObjectKey = "internal.cozystack.io/managed-by-cozystack" ManagerGroupKey = "apps.cozystack.io/application.group" ManagerKindKey = "apps.cozystack.io/application.kind" ManagerNameKey = "apps.cozystack.io/application.name" ) // getResourceSelectors returns the appropriate ApplicationDefinitionResources for a given GroupKind func (h *LineageControllerWebhook) getResourceSelectors(gk schema.GroupKind, crd *cozyv1alpha1.ApplicationDefinition) *cozyv1alpha1.ApplicationDefinitionResources { switch { case gk.Group == "" && gk.Kind == "Secret": return &crd.Spec.Secrets case gk.Group == "" && gk.Kind == "Service": return &crd.Spec.Services case gk.Group == "networking.k8s.io" && gk.Kind == "Ingress": return &crd.Spec.Ingresses default: return nil } } // SetupWithManager registers the handler with the webhook server. func (h *LineageControllerWebhook) SetupWithManagerAsWebhook(mgr ctrl.Manager) error { cfg := rest.CopyConfig(mgr.GetConfig()) var err error h.dynClient, err = dynamic.NewForConfig(cfg) if err != nil { return err } httpClient, err := rest.HTTPClientFor(cfg) if err != nil { return err } h.mapper, err = apiutil.NewDynamicRESTMapper(cfg, httpClient) if err != nil { return err } h.ownerCache = lineage.NewObjectCache(ownerCacheTTL) h.initConfig() // Register HTTP path -> handler. mgr.GetWebhookServer().Register("/mutate-lineage", &admission.Webhook{Handler: h}) return nil } // InjectDecoder lets controller-runtime give us a decoder for AdmissionReview requests. func (h *LineageControllerWebhook) InjectDecoder(d admission.Decoder) error { h.decoder = d return nil } // Handle is called for each AdmissionReview that matches the webhook config. func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Request) admission.Response { logger := log.FromContext(ctx).WithValues( "gvk", req.Kind.String(), "namespace", req.Namespace, "name", req.Name, "operation", req.Operation, ) warn := make(admission.Warnings, 0) obj := &unstructured.Unstructured{} if err := h.decodeUnstructured(req, obj); err != nil { return admission.Errored(400, fmt.Errorf("decode object: %w", err)) } // The objectSelector skips objects carrying the managed label, but on UPDATE // it matches if either the old or the new object does. Pods go on, since // every Pod admission also needs applySchedulingClass. if obj.GetKind() != "Pod" && hasAllLineageLabels(obj) { logger.V(1).Info("object already carries lineage labels, skipping owner walk") return admission.Allowed("lineage labels already present") } owner, err := h.getOwner(ctx, obj) switch { case err != nil && errors.Is(err, AncestryAmbiguous): warn = append(warn, "object ancestry ambiguous, using first ancestor found") case err != nil && errors.Is(err, NoAncestors): // not a problem, mark object as unmanaged case err != nil: logger.Error(err, "error computing lineage labels") return admission.Errored(500, fmt.Errorf("error computing lineage labels: %w", err)) } labels, err := h.computeLabels(ctx, obj, owner) if err != nil { logger.Error(err, "error computing lineage labels") return admission.Errored(500, fmt.Errorf("error computing lineage labels: %w", err)) } h.applyLabels(obj, labels) if err := h.applySchedulingClass(ctx, obj, owner, req.Namespace); err != nil { logger.Error(err, "error applying scheduling class") return admission.Errored(500, fmt.Errorf("error applying scheduling class: %w", err)) } mutated, err := json.Marshal(obj) if err != nil { return admission.Errored(500, fmt.Errorf("marshal mutated pod: %w", err)) } logger.V(1).Info("mutated pod", "namespace", obj.GetNamespace(), "name", obj.GetName()) return admission.PatchResponseFromRaw(req.Object.Raw, mutated).WithWarnings(warn...) } func (h *LineageControllerWebhook) getOwner(ctx context.Context, o *unstructured.Unstructured) (*unstructured.Unstructured, error) { owners := lineage.WalkOwnershipGraphWithCache(ctx, h.dynClient, h.mapper, h, h.ownerCache, o) if len(owners) == 0 { return nil, NoAncestors } obj, err := owners[0].GetUnstructuredCached(ctx, h.dynClient, h.mapper, h.ownerCache) if err != nil { return nil, err } if len(owners) > 1 { err = AncestryAmbiguous } return obj, err } func (h *LineageControllerWebhook) computeLabels(ctx context.Context, obj *unstructured.Unstructured, owner *unstructured.Unstructured) (map[string]string, error) { if owner == nil { return nil, nil } gv, err := schema.ParseGroupVersion(owner.GetAPIVersion()) if err != nil { // should never happen, we got an APIVersion right from the API return nil, fmt.Errorf("could not parse APIVersion %s to a group and version: %w", owner.GetAPIVersion(), err) } labels := map[string]string{ // truncate apigroup to first 63 chars ManagedObjectKey: "true", ManagerGroupKey: func(s string) string { if len(s) < 63 { return s } s = s[:63] for b := s[62]; !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')); s = s[:len(s)-1] { b = s[len(s)-1] } return s }(gv.Group), ManagerKindKey: owner.GetKind(), ManagerNameKey: owner.GetName(), } templateLabels := map[string]string{ "kind": strings.ToLower(owner.GetKind()), "name": owner.GetName(), "namespace": obj.GetNamespace(), } cfg := h.config.Load().(*runtimeConfig) crd := cfg.appCRDMap[appRef{gv.Group, owner.GetKind()}] resourceSelectors := h.getResourceSelectors(obj.GroupVersionKind().GroupKind(), crd) labels[corev1alpha1.TenantResourceLabelKey] = func(b bool) string { if b { return corev1alpha1.TenantResourceLabelValue } return "false" }(matchResourceToExcludeInclude(ctx, obj.GetName(), templateLabels, obj.GetLabels(), resourceSelectors)) return labels, nil } func (h *LineageControllerWebhook) applyLabels(o *unstructured.Unstructured, labels map[string]string) { existing := o.GetLabels() if existing == nil { existing = make(map[string]string) } maps.Copy(existing, labels) o.SetLabels(existing) } // hasAllLineageLabels needs every key computeLabels writes, not just // ManagedObjectKey: an UPDATE can carry a partial set written by another client, // and the tenantsecret registry filters on TenantResourceLabelKey. func hasAllLineageLabels(o *unstructured.Unstructured) bool { labels := o.GetLabels() if labels == nil { return false } if labels[ManagedObjectKey] != "true" { return false } for _, k := range []string{ManagerGroupKey, ManagerKindKey, ManagerNameKey, corev1alpha1.TenantResourceLabelKey} { if _, ok := labels[k]; !ok { return false } } return true } // applySchedulingClass injects schedulerName, scheduling-class annotation, and // scheduling-class label into Pods whose namespace carries the // scheduler.cozystack.io/scheduling-class label. // // The label (in addition to the annotation) lets SchedulingClass authors write // podAffinity rules whose labelSelector matches pods across applications that // share the same class. // // If the referenced SchedulingClass CR does not exist (e.g. the scheduler // package is not installed), the injection is silently skipped so that pods // are not left Pending. func (h *LineageControllerWebhook) applySchedulingClass(ctx context.Context, obj, owner *unstructured.Unstructured, namespace string) error { if obj.GetKind() != "Pod" { return nil } // Determine scheduling class: owner Application field takes priority, // then fall back to namespace label. var schedulingClass string if owner != nil { var app appsv1alpha1.Application if err := runtime.DefaultUnstructuredConverter.FromUnstructured(owner.Object, &app); err == nil { schedulingClass = app.SchedulingClass() } } if schedulingClass == "" { ns := &corev1.Namespace{} if err := h.Get(ctx, client.ObjectKey{Name: namespace}, ns); err != nil { return fmt.Errorf("getting namespace %s: %w", namespace, err) } schedulingClass = ns.Labels[schedulerapi.SchedulingClassLabel] } if schedulingClass == "" { return nil } // Verify that the referenced SchedulingClass CR exists. // If the CRD is not installed or the CR is missing, skip injection // so that pods are not stuck Pending on a non-existent scheduler. _, err := h.dynClient.Resource(schedulingClassGVR).Get(ctx, schedulingClass, metav1.GetOptions{}) if err != nil { logger := log.FromContext(ctx) logger.Info("SchedulingClass not found, skipping scheduler injection", "schedulingClass", schedulingClass, "namespace", namespace) return nil } if err := unstructured.SetNestedField(obj.Object, schedulerapi.SchedulerName, "spec", "schedulerName"); err != nil { return fmt.Errorf("setting schedulerName: %w", err) } annotations := obj.GetAnnotations() if annotations == nil { annotations = make(map[string]string) } annotations[schedulerapi.SchedulingClassAnnotation] = schedulingClass obj.SetAnnotations(annotations) podLabels := obj.GetLabels() if podLabels == nil { podLabels = make(map[string]string) } podLabels[schedulerapi.SchedulingClassLabel] = schedulingClass obj.SetLabels(podLabels) return nil } var schedulingClassGVR = schema.GroupVersionResource{ Group: schedulerapi.Group, Version: schedulerapi.Version, Resource: schedulerapi.Resource, } func (h *LineageControllerWebhook) decodeUnstructured(req admission.Request, out *unstructured.Unstructured) error { if h.decoder != nil { if err := h.decoder.Decode(req, out); err == nil { return nil } if req.Kind.Group != "" || req.Kind.Kind != "" || req.Kind.Version != "" { out.SetGroupVersionKind(schema.GroupVersionKind{ Group: req.Kind.Group, Version: req.Kind.Version, Kind: req.Kind.Kind, }) if err := h.decoder.Decode(req, out); err == nil { return nil } } } if len(req.Object.Raw) == 0 { return errors.New("empty admission object") } // encoding/json would turn every number into float64, and an int64 above // 2^53 would come back altered in the patch computed against req.Object.Raw. return utiljson.Unmarshal(req.Object.Raw, &out.Object) }