Files
cozystack/pkg/apiserver/apiserver.go
T
Aleksei Sviridkin 3dde6503ec refactor(api): keep each app's kind without the apimachinery fork
Every apps kind is registered on the one Application Go type, and the
stock Scheme.ConvertToVersion picks the first kind registered for a
type, so without help a Redis comes back from the API as whichever app
kind was registered first. So far the fix lived in a fork of
k8s.io/apimachinery that patches ConvertToVersion. Upstream has not
taken the change, and the fork has to be rebased onto every Kubernetes
minor before the libraries can move.

The apps group now encodes through a serializer that picks the kind the
object carries, along the lines upstream suggested instead of changing
Scheme. The replace directive goes, and with it the Renovate rule that
kept the fork's stale release tags from being proposed as updates. The
serializer relies on the storage stamping the kind on every object it
returns, which it does. A test drives create, get, list, update, watch,
apply and table through the real handler chain for two kinds and fails
without the serializer on upstream apimachinery.

One path converts through the Scheme rather than the serializer and
goes back to the first registered kind: the row objects of a Table
requested with includeObject=Object, which kubectl asks for only with
--sort-by. Admission webhooks and ValidatingAdmissionPolicy get the
request kind set explicitly.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
2026-09-26 16:08:24 +03:00

285 lines
9.6 KiB
Go

/*
Copyright 2024 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package apiserver
import (
"context"
"fmt"
"time"
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apiserver/pkg/registry/rest"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/client-go/dynamic"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"github.com/cozystack/cozystack/pkg/apis/apps"
appsinstall "github.com/cozystack/cozystack/pkg/apis/apps/install"
"github.com/cozystack/cozystack/pkg/apis/core"
coreinstall "github.com/cozystack/cozystack/pkg/apis/core/install"
"github.com/cozystack/cozystack/pkg/apis/sdn"
sdninstall "github.com/cozystack/cozystack/pkg/apis/sdn/install"
"github.com/cozystack/cozystack/pkg/config"
cozyregistry "github.com/cozystack/cozystack/pkg/registry"
applicationstorage "github.com/cozystack/cozystack/pkg/registry/apps/application"
optionstorage "github.com/cozystack/cozystack/pkg/registry/core/option"
tapstorage "github.com/cozystack/cozystack/pkg/registry/core/tap"
tenantmodulestorage "github.com/cozystack/cozystack/pkg/registry/core/tenantmodule"
tenantnamespacestorage "github.com/cozystack/cozystack/pkg/registry/core/tenantnamespace"
tenantsecretstorage "github.com/cozystack/cozystack/pkg/registry/core/tenantsecret"
securitygroupstorage "github.com/cozystack/cozystack/pkg/registry/sdn/securitygroup"
)
var (
// Scheme defines methods for serializing and deserializing API objects.
Scheme = runtime.NewScheme()
mgrScheme = runtime.NewScheme()
// Codecs provides methods for retrieving codecs and serializers for specific
// versions and content types.
Codecs = serializer.NewCodecFactory(Scheme)
CozyComponentName = "cozy"
syncPeriod = 5 * time.Minute
)
func init() {
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{
Development: true,
// any other zap.Options tweaks
})))
klog.SetLogger(ctrl.Log.WithName("klog"))
appsinstall.Install(Scheme)
coreinstall.Install(Scheme)
sdninstall.Install(Scheme)
// Register HelmRelease types.
if err := helmv2.AddToScheme(mgrScheme); err != nil {
panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err))
}
// Register the in-tree CiliumNetworkPolicy mirror backing SecurityGroup.
if err := securitygroupstorage.AddToScheme(mgrScheme); err != nil {
panic(fmt.Errorf("failed to add CiliumNetworkPolicy mirror to scheme: %w", err))
}
if err := corev1.AddToScheme(mgrScheme); err != nil {
panic(fmt.Errorf("Failed to add core types to scheme: %w", err))
}
if err := rbacv1.AddToScheme(mgrScheme); err != nil {
panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err))
}
// Register Cozystack types for WorkloadMonitor queries.
if err := cozyv1alpha1.AddToScheme(mgrScheme); err != nil {
panic(fmt.Errorf("failed to add Cozystack types to scheme: %w", err))
}
// Add unversioned types.
metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
// Add unversioned types.
unversioned := schema.GroupVersion{Group: "", Version: "v1"}
Scheme.AddUnversionedTypes(unversioned,
&metav1.Status{},
&metav1.APIVersions{},
&metav1.APIGroupList{},
&metav1.APIGroup{},
&metav1.APIResourceList{},
)
}
// Config defines the configuration for the apiserver.
type Config struct {
GenericConfig *genericapiserver.RecommendedConfig
ResourceConfig *config.ResourceConfig
}
// CozyServer holds the state for the Kubernetes master/api server.
type CozyServer struct {
GenericAPIServer *genericapiserver.GenericAPIServer
}
type completedConfig struct {
GenericConfig genericapiserver.CompletedConfig
ResourceConfig *config.ResourceConfig
}
// CompletedConfig embeds a private pointer that cannot be created outside of this package.
type CompletedConfig struct {
*completedConfig
}
// Complete fills in any fields that are not set but are required for valid operation.
func (cfg *Config) Complete() CompletedConfig {
c := completedConfig{
cfg.GenericConfig.Complete(),
cfg.ResourceConfig,
}
return CompletedConfig{&c}
}
// New returns a new instance of CozyServer from the given configuration.
func (c completedConfig) New() (*CozyServer, error) {
genericServer, err := c.GenericConfig.New("cozy-apiserver", genericapiserver.NewEmptyDelegate())
if err != nil {
return nil, err
}
s := &CozyServer{
GenericAPIServer: genericServer,
}
// Create a dynamic client for HelmRelease using InClusterConfig.
cfg, err := ctrl.GetConfig()
if err != nil {
return nil, fmt.Errorf("failed to get kubeconfig: %w", err)
}
mgr, err := ctrl.NewManager(cfg, ctrl.Options{
Scheme: mgrScheme,
Cache: cache.Options{SyncPeriod: &syncPeriod},
})
if err != nil {
return nil, fmt.Errorf("failed to build manager: %w", err)
}
if err := mgr.GetFieldIndexer().IndexField(
context.Background(),
&corev1.Service{},
"spec.type",
func(rawObj client.Object) []string {
svc := rawObj.(*corev1.Service)
return []string{string(svc.Spec.Type)}
}); err != nil {
return nil, fmt.Errorf("failed to index service spec.type field: %w", err)
}
ctx := ctrl.SetupSignalHandler()
if err = mustGetInformers(ctx, mgr,
&helmv2.HelmRelease{},
&corev1.Secret{},
&corev1.Namespace{},
&corev1.Service{},
&rbacv1.RoleBinding{},
&cozyv1alpha1.WorkloadMonitor{},
); err != nil {
return nil, fmt.Errorf("failed to get informers: %w", err)
}
go func() {
if err := mgr.Start(ctx); err != nil {
panic(fmt.Errorf("manager start failed: %w", err))
}
}()
if ok := mgr.GetCache().WaitForCacheSync(ctx); !ok {
return nil, fmt.Errorf("cache sync failed")
}
cli := mgr.GetClient()
watchCli, err := client.NewWithWatch(cfg, client.Options{Scheme: mgrScheme})
if err != nil {
return nil, fmt.Errorf("failed to build watch client: %w", err)
}
// Dynamic client for the Option resource's providers — direct (uncached)
// reads of arbitrary kinds (Nodes, KubeVirt, instancetypes, …) without
// registering an informer for each.
dyn, err := dynamic.NewForConfig(cfg)
if err != nil {
return nil, fmt.Errorf("failed to build dynamic client: %w", err)
}
// --- static, cluster-scoped resource for core group ---
coreV1alpha1Storage := map[string]rest.Storage{}
coreV1alpha1Storage["tenantnamespaces"] = cozyregistry.RESTInPeace(
tenantnamespacestorage.NewREST(cli, watchCli),
)
coreV1alpha1Storage["tenantsecrets"] = cozyregistry.RESTInPeace(
tenantsecretstorage.NewREST(cli, watchCli),
)
coreV1alpha1Storage["tenantmodules"] = cozyregistry.RESTInPeace(
tenantmodulestorage.NewREST(cli, watchCli),
)
coreV1alpha1Storage["options"] = cozyregistry.RESTInPeace(
optionstorage.NewRESTWithParams(
optionstorage.DefaultProviders(dyn),
optionstorage.DefaultParamProviders(dyn),
),
)
coreV1alpha1Storage["taps"] = cozyregistry.RESTInPeace(
tapstorage.NewREST(dyn),
)
coreApiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(core.GroupName, Scheme, metav1.ParameterCodec, Codecs)
coreApiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = coreV1alpha1Storage
if err := s.GenericAPIServer.InstallAPIGroup(&coreApiGroupInfo); err != nil {
return nil, err
}
// --- static, namespaced resource for sdn group ---
sdnV1alpha1Storage := map[string]rest.Storage{}
sdnV1alpha1Storage["securitygroups"] = cozyregistry.RESTInPeace(
securitygroupstorage.NewREST(cli, watchCli),
)
sdnApiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(sdn.GroupName, Scheme, metav1.ParameterCodec, Codecs)
sdnApiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = sdnV1alpha1Storage
if err := s.GenericAPIServer.InstallAPIGroup(&sdnApiGroupInfo); err != nil {
return nil, err
}
// --- dynamically-configured, per-tenant resources ---
appsV1alpha1Storage := map[string]rest.Storage{}
for _, resConfig := range c.ResourceConfig.Resources {
storage := applicationstorage.NewREST(cli, watchCli, &resConfig)
appsV1alpha1Storage[resConfig.Application.Plural] = cozyregistry.RESTInPeace(storage)
}
if err := InstallAppsAPIGroup(s.GenericAPIServer, appsV1alpha1Storage); err != nil {
return nil, err
}
return s, nil
}
// InstallAppsAPIGroup registers the apps.cozystack.io API group on the given
// server using the provided storage map (plural name → rest.Storage).
func InstallAppsAPIGroup(server *genericapiserver.GenericAPIServer, storage map[string]rest.Storage) error {
info := genericapiserver.NewDefaultAPIGroupInfo(apps.GroupName, Scheme, metav1.ParameterCodec, Codecs)
info.NegotiatedSerializer = kindPreservingSerializer{Codecs}
info.VersionedResourcesStorageMap["v1alpha1"] = storage
return server.InstallAPIGroup(&info)
}
func mustGetInformers(ctx context.Context, mgr ctrl.Manager, types ...client.Object) error {
for i := range types {
if _, err := mgr.GetCache().GetInformer(ctx, types[i]); err != nil {
return fmt.Errorf("failed to get informer for %T: %w", types[i], err)
}
}
return nil
}