mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 20:23:39 +09:00
Merge branch 'main' into feat/observability-proxy-ingress
Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
129 files changed
+6402
-4363
No files matched your search
Generated
+3
@@ -74,6 +74,9 @@ type ConfigSpec struct {
|
||||
}
|
||||
|
||||
type Disk struct {
|
||||
// Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both.
|
||||
// +kubebuilder:validation:Minimum=1
|
||||
BootOrder *int `json:"bootOrder,omitempty"`
|
||||
// Disk bus type (e.g. "sata").
|
||||
Bus string `json:"bus,omitempty"`
|
||||
// Disk name.
|
||||
|
||||
+8
-1
@@ -61,7 +61,9 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
|
||||
if in.Disks != nil {
|
||||
in, out := &in.Disks, &out.Disks
|
||||
*out = make([]Disk, len(*in))
|
||||
copy(*out, *in)
|
||||
for i := range *in {
|
||||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
if in.Networks != nil {
|
||||
in, out := &in.Networks, &out.Networks
|
||||
@@ -100,6 +102,11 @@ func (in *ConfigSpec) DeepCopy() *ConfigSpec {
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *Disk) DeepCopyInto(out *Disk) {
|
||||
*out = *in
|
||||
if in.BootOrder != nil {
|
||||
in, out := &in.BootOrder, &out.BootOrder
|
||||
*out = new(int)
|
||||
**out = **in
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Disk.
|
||||
|
||||
@@ -14,9 +14,10 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Command observability-proxy serves tenant logs from the VictoriaLogs
|
||||
// stores behind a Kubernetes TokenReview and SubjectAccessReview, with the
|
||||
// tenant's namespace scope forced into every upstream request.
|
||||
// Command observability-proxy serves tenant logs and metrics from the
|
||||
// VictoriaLogs and VictoriaMetrics stores behind a Kubernetes TokenReview
|
||||
// and SubjectAccessReview, with the tenant's namespace scope forced into
|
||||
// every upstream request.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -37,6 +38,7 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"k8s.io/apimachinery/pkg/util/validation"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
@@ -56,6 +58,7 @@ type options struct {
|
||||
tailIdleTimeout time.Duration
|
||||
shutdownTimeout time.Duration
|
||||
logs observabilityproxy.LogsLimits
|
||||
metrics observabilityproxy.MetricsConfig
|
||||
limiter observabilityproxy.LimiterConfig
|
||||
}
|
||||
|
||||
@@ -81,6 +84,14 @@ func parseFlags(args []string) (options, error) {
|
||||
fs.IntVar(&o.logs.DefaultLimit, "logs-default-limit", 1000, "entries returned when a request sets no limit")
|
||||
fs.IntVar(&o.logs.MaxLimit, "logs-max-limit", 10000, "largest limit a request may set")
|
||||
fs.DurationVar(&o.logs.MaxTailResume, "logs-max-tail-resume", 15*time.Minute, "how far back a tail may resume with since or start_offset")
|
||||
// Store names and retention follow metricsStorages in
|
||||
// packages/system/monitoring/values.yaml.
|
||||
fs.StringVar(&o.metrics.ShortTermStore, "metrics-shortterm-store", "shortterm", "VMCluster name of the short-term metrics store of the root tenant")
|
||||
fs.StringVar(&o.metrics.LongTermStore, "metrics-longterm-store", "longterm", "VMCluster name of the long-term metrics store of the root tenant")
|
||||
fs.DurationVar(&o.metrics.ShortTermRetention, "metrics-shortterm-retention", 72*time.Hour, "retention of the short-term store; older windows are read from the long-term one")
|
||||
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
|
||||
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
|
||||
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
|
||||
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
|
||||
fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables")
|
||||
fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user")
|
||||
@@ -94,6 +105,13 @@ func parseFlags(args []string) (options, error) {
|
||||
if fs.NArg() > 0 {
|
||||
return o, fmt.Errorf("unexpected argument %q", fs.Arg(0))
|
||||
}
|
||||
o.metrics.RootTenant = o.rootTenant
|
||||
for _, store := range []string{o.metrics.ShortTermStore, o.metrics.LongTermStore} {
|
||||
// The name becomes part of a Service host name.
|
||||
if errs := validation.IsDNS1123Label("vmselect-" + store); store == "" || len(errs) > 0 {
|
||||
return o, fmt.Errorf("invalid metrics store name %q", store)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case o.rootTenant == "":
|
||||
return o, errors.New("--root-tenant must not be empty")
|
||||
@@ -110,8 +128,17 @@ func parseFlags(args []string) (options, error) {
|
||||
return o, errors.New("--logs-max-range must be at least 3s")
|
||||
case o.logs.DefaultLimit <= 0 || o.logs.MaxLimit < o.logs.DefaultLimit:
|
||||
return o, errors.New("--logs-default-limit must be positive and not above --logs-max-limit")
|
||||
case o.metrics.ShortTermRetention <= 0 || o.metrics.Limits.MaxRange <= 0 || o.metrics.Limits.MaxTimeout <= 0:
|
||||
return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive")
|
||||
case o.metrics.Limits.MaxPoints < 2:
|
||||
return o, errors.New("--metrics-max-points must be at least 2")
|
||||
case o.logs.MaxTailResume < 0:
|
||||
return o, errors.New("--logs-max-tail-resume must not be negative")
|
||||
// The proxy's deadline starts before the request reaches vmselect, so
|
||||
// at an equal or longer timeout it cuts a query vmselect is still
|
||||
// allowed to run and answers 504 for a store that is within its time.
|
||||
case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout >= o.requestTimeout:
|
||||
return o, errors.New("--metrics-max-timeout must be below --request-timeout")
|
||||
case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0:
|
||||
return o, errors.New("timeouts and the auth cache TTL must not be negative")
|
||||
// NaN passes every comparison and +Inf never empties a bucket; either
|
||||
@@ -167,6 +194,7 @@ func run(o options) error {
|
||||
proxy := observabilityproxy.NewServer(observabilityproxy.Config{
|
||||
ClusterDomain: o.clusterDomain,
|
||||
Logs: o.logs,
|
||||
Metrics: o.metrics,
|
||||
Limiter: o.limiter,
|
||||
RequestTimeout: o.requestTimeout,
|
||||
TailIdleTimeout: o.tailIdleTimeout,
|
||||
|
||||
@@ -32,6 +32,17 @@ func TestParseFlagsDefaults(t *testing.T) {
|
||||
if o.authCacheTTL != 10*time.Second || o.logs.MaxTailResume != 15*time.Minute || o.logs.MaxRange != 7*24*time.Hour {
|
||||
t.Errorf("limit defaults changed: %+v", o)
|
||||
}
|
||||
// Retention and store names of packages/system/monitoring/values.yaml.
|
||||
m := o.metrics
|
||||
if m.ShortTermStore != "shortterm" || m.LongTermStore != "longterm" || m.ShortTermRetention != 72*time.Hour {
|
||||
t.Errorf("metrics store defaults changed: %+v", m)
|
||||
}
|
||||
if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second {
|
||||
t.Errorf("metrics limit defaults changed: %+v", m.Limits)
|
||||
}
|
||||
if m.RootTenant != o.rootTenant {
|
||||
t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFlagsOverrides(t *testing.T) {
|
||||
@@ -48,6 +59,21 @@ func TestParseFlagsOverrides(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFlagsMetricsOverrides(t *testing.T) {
|
||||
o, err := parseFlags([]string{
|
||||
"--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow",
|
||||
"--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := o.metrics
|
||||
if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour ||
|
||||
m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second {
|
||||
t.Errorf("overrides not applied: %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFlagsInvalid(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--root-tenant="},
|
||||
@@ -69,7 +95,17 @@ func TestParseFlagsInvalid(t *testing.T) {
|
||||
{"--max-concurrent-per-user=-1"},
|
||||
{"--max-tails-per-user=-1"},
|
||||
{"--max-tails=-1"},
|
||||
{"--metrics-shortterm-store="},
|
||||
{"--metrics-longterm-store=Long.Term"},
|
||||
{"--metrics-shortterm-retention=0s"},
|
||||
{"--metrics-max-range=0s"},
|
||||
{"--metrics-max-points=1"},
|
||||
{"--metrics-max-timeout=0s"},
|
||||
{"--unknown"},
|
||||
// The proxy would cut the request while vmselect is still within
|
||||
// the timeout it was given.
|
||||
{"--request-timeout=10s", "--metrics-max-timeout=30s"},
|
||||
{"--request-timeout=30s", "--metrics-max-timeout=30s"},
|
||||
{"unexpected", "--root-tenant=tenant-x"},
|
||||
} {
|
||||
if _, err := parseFlags(args); err == nil {
|
||||
@@ -77,3 +113,16 @@ func TestParseFlagsInvalid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The vmselect timeout must stay strictly below a request timeout, and is
|
||||
// free when the request has none.
|
||||
func TestParseFlagsTimeoutPairs(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--request-timeout=31s", "--metrics-max-timeout=30s"},
|
||||
{"--request-timeout=0", "--metrics-max-timeout=10m"},
|
||||
} {
|
||||
if _, err := parseFlags(args); err != nil {
|
||||
t.Errorf("parseFlags(%v) refused: %v", args, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
<!--
|
||||
https://github.com/cozystack/cozystack/releases/tag/v1.7.1
|
||||
-->
|
||||
|
||||
# Cozystack v1.7.1
|
||||
|
||||
Cozystack v1.7.1 is a patch release on the v1.7 line. It fixes the COSI driver bug that could leave freshly granted S3 keys rejected for good, makes marketplace taps digest-pinned, makes MariaDB, PostgreSQL and ClickHouse backups report their real outcome, and repairs several tenant-facing issues: tenant cluster metrics, SecurityGroups on Kubernetes workers, Redis with a single replica, the Proxmox cloud-controller-manager and the tcp-balancer. Read the Migration and Upgrades section before upgrading.
|
||||
|
||||
## Fixes
|
||||
|
||||
* **SeaweedFS COSI driver v0.4.0**: bucket users granted together could get S3 keys that never work (`403 InvalidAccessKeyId`) because the v0.3.1 driver rewrote a legacy IAM file the S3 gateways were migrating. The driver is bumped to v0.4.0, which writes each grant to a per-identity IAM file. Keys already lost on running clusters stay broken: remove the user from the Bucket, wait until its BucketAccess is gone, and add it back ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5284, backport #5290).
|
||||
* **Marketplace taps are pinned to a digest**: a tapped repository no longer follows a mutable tag, so moving the tag upstream cannot change what runs in the cluster. `tap connect` accepts `oci://...@sha256:...` references and an optional keyless cosign verify policy, anchored to an exact identity and preserved across re-connects; with the verify flags the CLI verifies the signature and pins the verified digest at tap time. Community index entries can record a digest too ([**@IvanHunters**](https://github.com/IvanHunters) in #5082, backport #5246).
|
||||
* **Tenant cluster metrics reach the tenant's Monitoring**: the vmagent of tenant Kubernetes clusters now writes to the tenant's vminsert and labels series with the owning tenant instead of `cozy-monitoring`; before, no tenant cluster metric reached the tenant's Monitoring ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5024, backport #5246).
|
||||
* **SecurityGroups apply to Kubernetes worker nodes**: the virt-launcher pods of Kubernetes worker nodes are now resolved to their Kubernetes application and marked as managed by Cozystack, so a SecurityGroup attached to a Kubernetes cluster applies to its workers, not only to its control plane ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5124, backport #5246).
|
||||
* **Membership-only SecurityGroups**: a SecurityGroup with attachments and no rules no longer leaves an invalid CiliumNetworkPolicy behind; its backing policy is accepted by Cilium and still allows or denies nothing ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5133, backport #5246).
|
||||
* **Proxmox CCM and CSI images**: the Proxmox cloud-controller-manager runs v0.16.1 and the Proxmox CCM and CSI driver images are pinned by digest. The v0.15.0 tag was re-pushed upstream to a build that panics on start, which left every new Proxmox worker with the uninitialized taint ([**@themoriarti**](https://github.com/themoriarti) in #4917, backport #5246).
|
||||
* **Autoscaler capacity of KubernetesNodes pools**: on the kubevirt substrate the cluster-autoscaler capacity annotation is planned on the cores the VM actually gets, so a leading-zero `resources.cpu` such as `"010"` (decoded as YAML, eight cores) no longer yields a wrong capacity. No pool rolls ([**@themoriarti**](https://github.com/themoriarti) in #4974, backport #5246).
|
||||
* **Tenant quota admission counts extended resources**: admission now reads the parent pool's usage of extended resources and hugepages under the `requests.*` key the controller counts, so a child tenant can no longer get a share the parent has already used ([**@lexfrei**](https://github.com/lexfrei) in #5184, backport #5246).
|
||||
* **Postgres backup jobs fail as soon as the CloudNativePG Backup fails**: a Postgres `BackupJob` now fails with the recorded error instead of after 30 minutes, and reports a `WaitingForCNPGBackup` condition with the Backup's phase while it waits ([**@scooby87**](https://github.com/scooby87) in #5211, backport #5246).
|
||||
* **MariaDB backups and restores report real failures**: a `BackupJob` or `RestoreJob` whose operator Job failed now ends Failed instead of Succeeded, and a failed backup no longer creates a Ready Backup without an archive ([**@scooby87**](https://github.com/scooby87) in #5207, backport #5246).
|
||||
* **ClickHouse restores use the right archive**: a `RestoreJob` restores the archive its Backup was taken as, not the newest one. New Backups record the archive name; Backups taken before this release are restored best effort from the newest archive stamped no later than their `takenAt`, with a warning in the restore pod log. A ClickHouse `BackupJob` created under a name whose Backup still exists now fails instead of reporting that Backup as its result ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5208, backport #5246).
|
||||
* **Backup S3 endpoint changes are followed**: the backup controller re-renders the Velero BSL, the default backup strategies and the platform bucket's credentials Secret when their S3 endpoint is stale, so after the platform bucket's BucketAccess is re-granted for a new `publishing.host`, backups move to the new endpoint without forcing releases by hand ([**@scooby87**](https://github.com/scooby87) in #5230, backport #5246).
|
||||
* **Backup clients run on a dedicated image**: the ClickHouse (Altinity) and RabbitMQ backup strategy Pods now run a small `backup-http-client` image instead of the 349 MB `platform-migrations` image, so a platform migration change no longer re-digests the backup clients. Air-gapped mirrors need the new `ghcr.io/cozystack/cozystack/backup-http-client` repository ([**@androndo**](https://github.com/androndo) in #4756, backport #5246).
|
||||
* **PostgreSQL system databases**: `databases` refuses `postgres`, `template0` and `template1`, and the init job never runs `DROP DATABASE` on them, which used to fail the init job on every upgrade and leave the HelmRelease in `UpgradeFailed` ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5155, backport #5246).
|
||||
* **PostgreSQL init script is stable**: the declared databases render in a stable order, so an upgrade that leaves the databases unchanged no longer rewrites the init-script Secret ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5091, backport #5246).
|
||||
* **Redis with a single replica**: an instance with `replicas: 1` runs one Sentinel instead of three, without a Sentinel PodDisruptionBudget, so node drains are no longer blocked by it; instances with two replicas or more are unchanged ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5151, backport #5246).
|
||||
* **MariaDB operator on a custom cluster domain**: the operator uses the platform's `networking.clusterDomain` instead of a hard-coded `cozy.local`, so replication and User/Database/Grant reconciliation work on clusters with a non-default domain ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5123, backport #5246).
|
||||
* **tcp-balancer no longer OOMs**: HAProxy takes its fd and thread caps from the replica's own memory and CPU limits instead of the node, so a `t1.nano` balancer is no longer OOMKilled on nodes with many CPUs ([**@yankawai**](https://github.com/yankawai) in #5005, backport #5246).
|
||||
* **etcd alert noise**: `etcdHighNumberOfFailedGRPCRequests` counts only the gRPC codes the etcd mixin counts, so the Canceled watch streams of a stopping Kubernetes API server no longer fire it on every member ([**@yankawai**](https://github.com/yankawai) in #4985, backport #5246).
|
||||
* **Cilium IPAM hook on larger clusters**: the hook passes the CiliumNode list to `jq` on stdin and no longer fails with "Argument list too long" on clusters with more than a few nodes, which blocked the cilium upgrade and every package depending on networking ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5212, backport #5246).
|
||||
* **OIDC bootstrap Job with many users**: the Job no longer fails to start for a tenant cluster with about 240 or more `oidc.users`; the bindings now come from a ConfigMap the Job mounts ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5213, backport #5246).
|
||||
* **Dashboard tenant list links to the tenant page**: the tenant name in the tenant list opens its detail page, where Delete lives, and top-level tenants get their Edit button back ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4004, backport #5246).
|
||||
* **Dead `registries` platform value removed**: the platform chart no longer has the `registries` value or the `patch-containerd` Secret, which nothing read. Mirrors for tenant workers are the per-pool `talos.registryMirrors` of `KubernetesNodes` ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5190, backport #5246).
|
||||
|
||||
## Security
|
||||
|
||||
* **Marketplace digest pinning and signature verification**: see the marketplace entry under Fixes (#5082).
|
||||
|
||||
## Development, Testing, and CI/CD
|
||||
|
||||
* **Chainsaw asserts survive transient API errors**: an e2e assert keeps polling through an error such as an etcd leader change instead of failing the suite early ([**@lexfrei**](https://github.com/lexfrei) in #4867, backport #5246).
|
||||
* **Proxmox e2e hygiene**: the Proxmox tenant kubeconfig is kept per run and removed afterwards, the HelmRelease guard reads the `KubernetesNodes` pool release, and the leak allowlist is validated against every suite layout ([**@themoriarti**](https://github.com/themoriarti) in #4971, #4970 and #4972, backports #5246).
|
||||
* **Frozen-line builds start with `GITHUB_TOKEN`**: the initial packages build dispatched when a release line is frozen now runs ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4888, backport #5246).
|
||||
* **Unit tests after the release image restamp**: two unit tests that read fields the release build rewrites stay green on `release-1.7`; this is a release-branch-only fix ([**@themoriarti**](https://github.com/themoriarti) in #5179).
|
||||
|
||||
## Documentation
|
||||
|
||||
* **[website] Documentation**: the v1.7 docs gain the guides that lived only in the cozystack repository, the pending v1.7 docs PRs, a DRBD note for the replicated storage class, a refreshed Chainsaw e2e guide, restored legacy URLs and a grouping of older doc versions ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#763, cozystack/website#782, cozystack/website#675, cozystack/website#611, cozystack/website#775 and cozystack/website#767).
|
||||
* **[website] Blog and demo**: a full-stack observability post for v1.7, the Cozystack 1.7 announcement, a three-board Raspberry Pi 5 post, and a collapsible navbar with a closable hint on the live demo page ([**@tym83**](https://github.com/tym83) in cozystack/website#790, cozystack/website#801 and cozystack/website#802; [**@kvaps**](https://github.com/kvaps) in cozystack/website#789).
|
||||
* **[website] Pages deploy**: the deploy workflow drops the Dart Sass install ([**@kvaps**](https://github.com/kvaps) in cozystack/website#798).
|
||||
* **[ansible-cozystack] Installer v1.7.0**: tracks the Cozystack installer image at v1.7.0, released in ansible-cozystack v1.7.0 ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#89).
|
||||
|
||||
## Migration and Upgrades
|
||||
|
||||
Platform migrations do not change in this release: `migrations.targetVersion` stays at 61.
|
||||
|
||||
* **Remove system databases from PostgreSQL `databases` before upgrading (#5155).** A release that declares `postgres`, `template0` or `template1` under `databases` is refused at render; remove the entry first. A system database a previous version took over is handed back to the superuser on the next upgrade: the event trigger is removed, ownership and the `public` schema are restored, and its `<db>_admin` and `<db>_readonly` roles are dropped.
|
||||
* **Tenant quota updates can be rejected after upgrading (#5184).** Admission now counts the parent's usage of extended resources and hugepages, so an update to an existing tenant can be rejected when its quota for these resources exceeds the parent's remaining budget, including an update that leaves the quota unchanged.
|
||||
* **Air-gapped mirrors need `backup-http-client` (#4756).** Mirror `ghcr.io/cozystack/cozystack/backup-http-client` before upgrading, or ClickHouse and RabbitMQ backup Pods cannot pull their image.
|
||||
* **tcp-balancer pods roll once (#5005).** The new HAProxy `global` section changes the rendered config; `maxconn` follows the memory limit at about 16 connections per MiB (2035 on `t1.nano`).
|
||||
* **Redis scaled to one replica loses its Sentinel PDB (#5151).** A post-upgrade hook deletes the Sentinel PodDisruptionBudget the operator created earlier for an instance with `replicas: 1`; scaling an instance between 1 and 2 replicas scales its Sentinel Deployment between 1 and 3.
|
||||
* **Backups created by failed MariaDB runs are not cleaned up (#5207).** Backups made by failed runs before this fix are not removed; restoring from one can fail or select another dump from the shared prefix.
|
||||
* **Keys lost to the COSI grant bug stay lost (#5284).** Re-add the affected user to the Bucket as described in the Fixes entry.
|
||||
* **`registries` in the platform Package is ignored (#5190).** A Package that still sets it keeps installing; the `patch-containerd` Secret in `cozy-system` is deleted by the release.
|
||||
* **SecurityGroups now reach Kubernetes workers (#5124).** A SecurityGroup attached to a `Kubernetes` application starts applying to its worker nodes after the upgrade, so review attached groups before upgrading.
|
||||
|
||||
## Contributors
|
||||
|
||||
We'd like to thank all contributors who made this release possible:
|
||||
|
||||
* [**@androndo**](https://github.com/androndo)
|
||||
* [**@IvanHunters**](https://github.com/IvanHunters)
|
||||
* [**@kvaps**](https://github.com/kvaps)
|
||||
* [**@lexfrei**](https://github.com/lexfrei)
|
||||
* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri)
|
||||
* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil)
|
||||
* [**@scooby87**](https://github.com/scooby87)
|
||||
* [**@themoriarti**](https://github.com/themoriarti)
|
||||
* [**@tym83**](https://github.com/tym83)
|
||||
* [**@yankawai**](https://github.com/yankawai)
|
||||
|
||||
**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.7.0...v1.7.1
|
||||
@@ -9,6 +9,10 @@ load test_helper
|
||||
# scope: they are re-vendored by `make update`, so a bad severity there is
|
||||
# fixed upstream or in patches/, not by editing the label.
|
||||
ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown "
|
||||
# The alerts that exist only to be dropped, the one place `none` is right.
|
||||
# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the
|
||||
# platform config does not have, so delivering it at any severity is noise.
|
||||
ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor "
|
||||
|
||||
helm_actions_stripped() {
|
||||
awk '
|
||||
@@ -104,6 +108,14 @@ alert_severity_contract() {
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$severity" = "none" ]; then
|
||||
for blackholed in $ALERT_SEVERITY_BLACKHOLED; do
|
||||
if [ "$blackholed" = "$alert" ]; then
|
||||
accepted=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ "$accepted" -eq 0 ]; then
|
||||
echo "$file: $alert -> '$severity'" >&2
|
||||
bad=$((bad + 1))
|
||||
@@ -128,6 +140,83 @@ alert_severity_contract() {
|
||||
alert_severity_contract
|
||||
}
|
||||
|
||||
@test "InfoInhibitor is labelled for the blackhole route" {
|
||||
rows=$(mktemp) || return 1
|
||||
if ! alert_severity_rows > "$rows"; then
|
||||
rm -f "$rows"
|
||||
return 1
|
||||
fi
|
||||
tab=$(printf '\t')
|
||||
found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true)
|
||||
delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true)
|
||||
rm -f "$rows"
|
||||
[ "$found" -gt 0 ]
|
||||
[ "$delivered" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "severity none passes only on an alert that exists to be dropped" {
|
||||
fixture=$(mktemp -d)
|
||||
mkdir -p "$fixture/packages/system/example/alerts"
|
||||
printf '%s\n' \
|
||||
'apiVersion: operator.victoriametrics.com/v1beta1' \
|
||||
'kind: VMRule' \
|
||||
'spec:' \
|
||||
' groups:' \
|
||||
' - name: general' \
|
||||
' rules:' \
|
||||
' - alert: InfoInhibitor' \
|
||||
' labels:' \
|
||||
' severity: none' \
|
||||
> "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
|
||||
git -C "$fixture" init -q
|
||||
git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
|
||||
|
||||
inhibitor_succeeded=0
|
||||
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$?
|
||||
|
||||
printf '%s\n' \
|
||||
'apiVersion: operator.victoriametrics.com/v1beta1' \
|
||||
'kind: VMRule' \
|
||||
'spec:' \
|
||||
' groups:' \
|
||||
' - name: silent' \
|
||||
' rules:' \
|
||||
' - alert: SilentAlert' \
|
||||
' labels:' \
|
||||
' severity: none' \
|
||||
> "$fixture/packages/system/example/alerts/01-silent.yaml"
|
||||
git -C "$fixture" add packages/system/example/alerts/01-silent.yaml
|
||||
|
||||
silent_succeeded=0
|
||||
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$?
|
||||
rm -rf "$fixture"
|
||||
[ "$inhibitor_succeeded" -eq 0 ]
|
||||
[ "$silent_succeeded" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "an alert that exists to be dropped still fails on a refused severity" {
|
||||
fixture=$(mktemp -d)
|
||||
mkdir -p "$fixture/packages/system/example/alerts"
|
||||
printf '%s\n' \
|
||||
'apiVersion: operator.victoriametrics.com/v1beta1' \
|
||||
'kind: VMRule' \
|
||||
'spec:' \
|
||||
' groups:' \
|
||||
' - name: general' \
|
||||
' rules:' \
|
||||
' - alert: InfoInhibitor' \
|
||||
' labels:' \
|
||||
' severity: info' \
|
||||
> "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
|
||||
git -C "$fixture" init -q
|
||||
git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
|
||||
|
||||
contract_succeeded=0
|
||||
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
|
||||
rm -rf "$fixture"
|
||||
[ "$contract_succeeded" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "a parser failure cannot leave the contract green" {
|
||||
fixture=$(mktemp -d)
|
||||
mkdir -p "$fixture/packages/system/example/alerts"
|
||||
|
||||
@@ -2718,9 +2718,12 @@ func buildBarmanPlugin(objectStoreName, serverName string) cnpgtypes.PluginConfi
|
||||
// high-water mark on a 38 MB database is 254 MiB, so 128Mi cannot hold it. A
|
||||
// tenant that leaves resourceQuotas empty has no LimitRange, and there the
|
||||
// sidecar had no requests and no limit at all. 256Mi holds the measured working
|
||||
// set and 1Gi is four times the measurement. No CPU limit is set, matching the
|
||||
// entityOperator precedent in packages/apps/kafka: a throttled sidecar stalls
|
||||
// WAL archiving instead of failing it.
|
||||
// set and 1Gi is four times the measurement. The CPU limit is explicit for the
|
||||
// same reason: left unset, that LimitRange gives the sidecar its 250m default,
|
||||
// so gzip compression of full backups and WAL segments runs on a quarter of a
|
||||
// core (a 1.6 GB database took 30 minutes to back up). One core keeps the
|
||||
// sidecar from starving the instance while removing that ceiling; it counts
|
||||
// against the tenant's limits.cpu budget, one core per instance pod.
|
||||
//
|
||||
// Checksum: since botocore ~1.36 the default (when_supported) attaches a
|
||||
// flexible checksum to every PutObject, which non-AWS S3-compatible backends
|
||||
@@ -2740,6 +2743,7 @@ func barmanSidecarConfiguration() *cnpgtypes.InstanceSidecarConfiguration {
|
||||
corev1.ResourceMemory: resource.MustParse("256Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
|
||||
@@ -2156,8 +2156,11 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) {
|
||||
sc.Resources.Requests.Cpu().MilliValue() != 100 {
|
||||
t.Errorf("ObjectStore leaves Barman subject to undersized tenant resource defaults: %+v", sc.Resources)
|
||||
}
|
||||
if _, ok := sc.Resources.Limits[corev1.ResourceCPU]; ok {
|
||||
t.Errorf("ObjectStore caps sidecar CPU, which throttles WAL archiving instead of failing it: %+v", sc.Resources.Limits)
|
||||
// Without an explicit CPU limit the tenant LimitRange caps the sidecar at
|
||||
// its 250m default, and every backup and WAL upload compresses on a
|
||||
// quarter of a core.
|
||||
if sc.Resources.Limits.Cpu().MilliValue() != 1000 {
|
||||
t.Errorf("ObjectStore leaves the sidecar CPU limit to the tenant LimitRange default: %+v", sc.Resources.Limits)
|
||||
}
|
||||
// The ObjectStore must be owner-referenced to the Cluster so Kubernetes GC
|
||||
// removes it when the Cluster is deleted (no orphan in the platform flow,
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// instantDefaultStep is the step vmselect uses for an instant query without
|
||||
// one (defaultStep in app/vmselect/prometheus); it is also the implicit
|
||||
// rollup window of a bare selector.
|
||||
const instantDefaultStep = 5 * time.Minute
|
||||
|
||||
// silenceInterval is how far before a rollup or subquery window vmselect
|
||||
// starts reading (maxSilenceInterval in app/vmselect/promql: the value of
|
||||
// -search.minStalenessInterval, 5m when unset; cozystack does not set it).
|
||||
const silenceInterval = 5 * time.Minute
|
||||
|
||||
// latencyOffset is how far vmselect moves an instant query evaluated within
|
||||
// it of the current time back (-search.latencyOffset, 30s by default and
|
||||
// not set by cozystack); the proxy forwards neither nocache nor
|
||||
// latency_offset.
|
||||
const latencyOffset = 30 * time.Second
|
||||
|
||||
// queryDurationPart is the subset of the MetricsQL duration grammar
|
||||
// (scanSingleDuration in metricsql) the scan understands. Step multiples
|
||||
// (5i, $__interval), negative parts and upper-case units are left out on
|
||||
// purpose: anything else makes the bound unknown.
|
||||
var (
|
||||
queryDurationPart = regexp.MustCompile(`(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`)
|
||||
queryDuration = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y))+$`)
|
||||
querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`)
|
||||
)
|
||||
|
||||
var queryDurationUnits = map[string]time.Duration{
|
||||
"ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour,
|
||||
"d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour,
|
||||
}
|
||||
|
||||
// queryStart is the earliest instant a query evaluated from start may read;
|
||||
// zero when the bound is unknown.
|
||||
func queryStart(query string, start time.Time, step time.Duration) time.Time {
|
||||
lookback, ok := queryLookback(query, step)
|
||||
if !ok {
|
||||
return time.Time{}
|
||||
}
|
||||
return start.Add(-lookback)
|
||||
}
|
||||
|
||||
// queryLookback is an upper bound on how far before its evaluation window
|
||||
// a query reads, from a lexical scan that never parses the query. Every
|
||||
// [...], offset and "(" counts, including ones inside quoted label values:
|
||||
// an extra hit only makes the bound larger or unknown, never smaller.
|
||||
// Windows and offsets are summed rather than maxed, which covers nesting;
|
||||
// WITH is refused because a template applies one window several times.
|
||||
// vmselect starts a rollup max(window, step) plus the silence interval
|
||||
// before the first point, a subquery window+step+silence before it and
|
||||
// aligns both down to a step. A rollup function or offset over anything
|
||||
// but a selector is an implicit subquery (getRollupExprArg), so every "("
|
||||
// is counted as one. Each step term uses the largest step seen.
|
||||
func queryLookback(query string, step time.Duration) (time.Duration, bool) {
|
||||
if strings.Contains(query, "@") {
|
||||
return 0, false
|
||||
}
|
||||
var sum float64
|
||||
subqueries := 0
|
||||
maxStep := float64(step)
|
||||
for rest := query; ; {
|
||||
open := strings.IndexByte(rest, '[')
|
||||
if open < 0 {
|
||||
break
|
||||
}
|
||||
closing := strings.IndexByte(rest[open:], ']')
|
||||
if closing < 0 {
|
||||
return 0, false
|
||||
}
|
||||
window, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":")
|
||||
rest = rest[open+closing+1:]
|
||||
if isSubquery {
|
||||
subqueries++
|
||||
if strings.TrimSpace(window) == "" {
|
||||
window = "0s"
|
||||
}
|
||||
if strings.TrimSpace(subStep) != "" {
|
||||
d, ok := parseQueryDuration(subStep)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
maxStep = max(maxStep, d)
|
||||
}
|
||||
}
|
||||
d, ok := parseQueryDuration(window)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
sum += d
|
||||
}
|
||||
lower := strings.ToLower(query)
|
||||
for i := 0; ; {
|
||||
k := strings.Index(lower[i:], "offset")
|
||||
if k < 0 {
|
||||
break
|
||||
}
|
||||
k += i
|
||||
i = k + len("offset")
|
||||
if !isKeyword(lower, k, i) {
|
||||
continue
|
||||
}
|
||||
arg := strings.TrimLeft(lower[i:], " \t\n\r")
|
||||
negative := strings.HasPrefix(arg, "-")
|
||||
if negative {
|
||||
arg = strings.TrimLeft(arg[1:], " \t\n\r")
|
||||
}
|
||||
end := strings.IndexFunc(arg, func(r rune) bool { return !inDuration(r) })
|
||||
if end < 0 {
|
||||
end = len(arg)
|
||||
}
|
||||
d, ok := parseQueryDuration(arg[:end])
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
if !negative {
|
||||
sum += d
|
||||
}
|
||||
}
|
||||
if hasKeyword(lower, "with") {
|
||||
return 0, false
|
||||
}
|
||||
n := float64(subqueries + strings.Count(query, "("))
|
||||
sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval)
|
||||
if sum > float64(maxMetricsDuration) {
|
||||
return 0, false
|
||||
}
|
||||
return time.Duration(sum), true
|
||||
}
|
||||
|
||||
// isKeyword reports whether lower[start:end] stands on its own rather than
|
||||
// inside an identifier.
|
||||
func isKeyword(lower string, start, end int) bool {
|
||||
return (start == 0 || !continuesIdent(lower[start-1], false)) &&
|
||||
(end == len(lower) || !continuesIdent(lower[end], true))
|
||||
}
|
||||
|
||||
func inDuration(r rune) bool {
|
||||
return r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.' || r == '_'
|
||||
}
|
||||
|
||||
func hasKeyword(lower, word string) bool {
|
||||
for i := 0; ; {
|
||||
k := strings.Index(lower[i:], word)
|
||||
if k < 0 {
|
||||
return false
|
||||
}
|
||||
k += i
|
||||
i = k + len(word)
|
||||
if isKeyword(lower, k, i) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// continuesIdent reports whether c, next to "offset", makes it part of an
|
||||
// identifier rather than the keyword. Digits and dots only continue an
|
||||
// identifier after it, so before it they still count as a keyword hit.
|
||||
func continuesIdent(c byte, after bool) bool {
|
||||
return c >= 'a' && c <= 'z' || c == '_' || c == ':' || c >= 0x80 ||
|
||||
after && (c >= '0' && c <= '9' || c == '.' || c == '\\')
|
||||
}
|
||||
|
||||
// parseQueryDuration returns nanoseconds as a float so that a sum of huge
|
||||
// durations cannot wrap; the caller compares it against the bound.
|
||||
func parseQueryDuration(s string) (float64, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if querySeconds.MatchString(s) {
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
return f * float64(time.Second), err == nil
|
||||
}
|
||||
if !queryDuration.MatchString(s) {
|
||||
return 0, false
|
||||
}
|
||||
var d float64
|
||||
for _, m := range queryDurationPart.FindAllStringSubmatch(s, -1) {
|
||||
f, err := strconv.ParseFloat(m[1], 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
d += f * float64(queryDurationUnits[m[2]])
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MetricsLimits bound the window, resolution and timeout a metrics request
|
||||
// asks vmselect for. Windows and offsets inside the expression are not
|
||||
// limited here: they cannot read past the store's retention, vmselect caps
|
||||
// the series and samples one query may touch, and MaxTimeout bounds the
|
||||
// work of rollups over long overlapping windows.
|
||||
type MetricsLimits struct {
|
||||
MaxRange time.Duration
|
||||
MaxPoints int
|
||||
MaxTimeout time.Duration
|
||||
}
|
||||
|
||||
// MetricsConfig names the vmselect stores of the root tenant and the
|
||||
// retention that decides between them.
|
||||
type MetricsConfig struct {
|
||||
RootTenant string
|
||||
ShortTermStore string
|
||||
LongTermStore string
|
||||
ShortTermRetention time.Duration
|
||||
Limits MetricsLimits
|
||||
// UpstreamURL maps a store name to its Prometheus API base URL, without
|
||||
// a trailing slash; nil means the in-cluster vmselect Service.
|
||||
UpstreamURL func(store string) string
|
||||
}
|
||||
|
||||
// store picks the short-term store when everything from start on is still
|
||||
// within its retention, so a live graph never waits on the downsampled one.
|
||||
func (c MetricsConfig) store(start, now time.Time) string {
|
||||
if now.Sub(start) <= c.ShortTermRetention {
|
||||
return c.ShortTermStore
|
||||
}
|
||||
return c.LongTermStore
|
||||
}
|
||||
|
||||
// maxMetricsMatchers bounds match[]: vmselect runs one index search per
|
||||
// matcher and has no cap of its own.
|
||||
const maxMetricsMatchers = 20
|
||||
|
||||
// labelsDefaultWindow is vmselect's own default for the label and series
|
||||
// APIs (defaultStep in app/vmselect/prometheus); the proxy sends it
|
||||
// explicitly so the store choice and the range check see the same window.
|
||||
const labelsDefaultWindow = 5 * time.Minute
|
||||
|
||||
var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
|
||||
|
||||
type metricsRequest struct {
|
||||
path string
|
||||
params url.Values
|
||||
// start is the earliest instant the request reads, for the store choice;
|
||||
// zero when that is unknown, which picks the long-term store.
|
||||
start time.Time
|
||||
}
|
||||
|
||||
// namespaceSelector quotes ns the way MetricsQL unquotes a label value
|
||||
// (strconv.Unquote in metricsql's extractStringValue).
|
||||
func namespaceSelector(ns string) string {
|
||||
return "{namespace=" + strconv.Quote(ns) + "}"
|
||||
}
|
||||
|
||||
// BuildMetricsParams rebuilds the upstream parameters from an allowlist and
|
||||
// adds the namespace scope as the only extra_filters[] value. vmselect ORs
|
||||
// repeated extra_filters, so a client value would widen the scope; it ANDs
|
||||
// the single one into every selector of the query and every match[], which
|
||||
// is what lets the query pass through unparsed.
|
||||
func BuildMetricsParams(endpoint string, in url.Values, namespace string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) {
|
||||
req := metricsRequest{path: endpoint, params: url.Values{}}
|
||||
out := req.params
|
||||
out.Set("extra_filters[]", namespaceSelector(namespace))
|
||||
timeout := lim.MaxTimeout
|
||||
if v := in.Get("timeout"); v != "" {
|
||||
d, herr := parseMetricsDuration("timeout", v)
|
||||
if herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
timeout = min(d, lim.MaxTimeout)
|
||||
}
|
||||
out.Set("timeout", formatSeconds(timeout))
|
||||
|
||||
switch endpoint {
|
||||
case "api/v1/query":
|
||||
if herr := copyQuery(out, in); herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
at := now
|
||||
if v := in.Get("time"); v != "" {
|
||||
t, herr := parseMetricsTime("time", v)
|
||||
if herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
at = t
|
||||
out.Set("time", formatTime(t))
|
||||
}
|
||||
step := instantDefaultStep
|
||||
if v := in.Get("step"); v != "" {
|
||||
var herr *HTTPError
|
||||
if step, herr = parseStep(v, lim); herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
out.Set("step", formatSeconds(step))
|
||||
}
|
||||
req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step)
|
||||
return req, nil
|
||||
|
||||
case "api/v1/query_range":
|
||||
if herr := copyQuery(out, in); herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
for _, k := range []string{"start", "end", "step"} {
|
||||
if in.Get(k) == "" {
|
||||
return req, errorf(http.StatusBadRequest, "%s is required", k)
|
||||
}
|
||||
}
|
||||
start, end, herr := metricsWindow(in, now, lim)
|
||||
if herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
step, herr := parseStep(in.Get("step"), lim)
|
||||
if herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
if points := end.Sub(start)/step + 1; points > time.Duration(lim.MaxPoints) {
|
||||
return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints)
|
||||
}
|
||||
req.start = queryStart(out.Get("query"), start, step)
|
||||
out.Set("start", formatTime(start))
|
||||
out.Set("end", formatTime(end))
|
||||
out.Set("step", formatSeconds(step))
|
||||
return req, nil
|
||||
}
|
||||
|
||||
name, isLabelValues := strings.CutPrefix(endpoint, "api/v1/label/")
|
||||
if isLabelValues {
|
||||
var ok bool
|
||||
if name, ok = strings.CutSuffix(name, "/values"); !ok || strings.Contains(name, "/") {
|
||||
return req, errorf(http.StatusNotFound, "not found")
|
||||
}
|
||||
if !labelName.MatchString(name) {
|
||||
return req, errorf(http.StatusBadRequest, "invalid label name %q", name)
|
||||
}
|
||||
} else if endpoint != "api/v1/series" && endpoint != "api/v1/labels" {
|
||||
return req, errorf(http.StatusNotFound, "not found")
|
||||
}
|
||||
start, end, herr := metricsWindow(in, now, lim)
|
||||
if herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
req.start = start
|
||||
out.Set("start", formatTime(start))
|
||||
out.Set("end", formatTime(end))
|
||||
|
||||
matchers := in["match[]"]
|
||||
if len(matchers) > maxMetricsMatchers {
|
||||
return req, errorf(http.StatusUnprocessableEntity, "at most %d match[] are allowed", maxMetricsMatchers)
|
||||
}
|
||||
for _, m := range matchers {
|
||||
if m == "" {
|
||||
return req, errorf(http.StatusBadRequest, "empty match[]")
|
||||
}
|
||||
out.Add("match[]", m)
|
||||
}
|
||||
if len(matchers) == 0 {
|
||||
if endpoint == "api/v1/series" {
|
||||
return req, errorf(http.StatusBadRequest, "match[] is required")
|
||||
}
|
||||
// vmselect started with -search.ignoreExtraFiltersAtLabelsAPI drops
|
||||
// extra_filters[] on the label APIs unless a match[] is present
|
||||
// (getCommonParamsInternal), so the scope is repeated here.
|
||||
out.Set("match[]", namespaceSelector(namespace))
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func copyQuery(out, in url.Values) *HTTPError {
|
||||
q := in.Get("query")
|
||||
if q == "" {
|
||||
return errorf(http.StatusBadRequest, "query is required")
|
||||
}
|
||||
out.Set("query", q)
|
||||
return nil
|
||||
}
|
||||
|
||||
// metricsWindow reads start and end with vmselect's label API defaults:
|
||||
// end is now and start is five minutes before end.
|
||||
func metricsWindow(in url.Values, now time.Time, lim MetricsLimits) (time.Time, time.Time, *HTTPError) {
|
||||
end := now
|
||||
if v := in.Get("end"); v != "" {
|
||||
t, herr := parseMetricsTime("end", v)
|
||||
if herr != nil {
|
||||
return time.Time{}, time.Time{}, herr
|
||||
}
|
||||
end = t
|
||||
}
|
||||
start := end.Add(-labelsDefaultWindow)
|
||||
if v := in.Get("start"); v != "" {
|
||||
t, herr := parseMetricsTime("start", v)
|
||||
if herr != nil {
|
||||
return time.Time{}, time.Time{}, herr
|
||||
}
|
||||
start = t
|
||||
}
|
||||
if start.After(end) {
|
||||
return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "start is after end")
|
||||
}
|
||||
if end.Sub(start) > lim.MaxRange {
|
||||
return time.Time{}, time.Time{}, errorf(http.StatusUnprocessableEntity, "range %s exceeds the limit of %s", end.Sub(start), lim.MaxRange)
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
// parseStep refuses a step above the range limit: vmselect aligns the
|
||||
// window outward to whole steps, and an implicit rollup window is one step,
|
||||
// so a huge step reads far more than the range it was sent with.
|
||||
func parseStep(v string, lim MetricsLimits) (time.Duration, *HTTPError) {
|
||||
step, herr := parseMetricsDuration("step", v)
|
||||
if herr != nil {
|
||||
return 0, herr
|
||||
}
|
||||
if step > lim.MaxRange {
|
||||
return 0, errorf(http.StatusUnprocessableEntity, "step %s exceeds the limit of %s", step, lim.MaxRange)
|
||||
}
|
||||
return step, nil
|
||||
}
|
||||
|
||||
// maxMetricsDuration is vmselect's own bound on step and timeout
|
||||
// (maxDurationMsecs in lib/httputil); it also keeps the rounding below
|
||||
// from overflowing.
|
||||
const maxMetricsDuration = 100 * 365 * 24 * time.Hour
|
||||
|
||||
// parseMetricsDuration takes Prometheus float seconds or a duration such as
|
||||
// 1m30s. vmselect works in whole milliseconds and refuses anything shorter;
|
||||
// the value is rounded up to the millisecond so the limits are checked
|
||||
// against what vmselect will use, not against a fraction it drops.
|
||||
func parseMetricsDuration(key, v string) (time.Duration, *HTTPError) {
|
||||
d, err := parseDuration(v)
|
||||
if err != nil || d < time.Millisecond || d > maxMetricsDuration {
|
||||
return 0, errorf(http.StatusBadRequest, "invalid %s %q", key, v)
|
||||
}
|
||||
return (d + time.Millisecond - 1).Truncate(time.Millisecond), nil
|
||||
}
|
||||
|
||||
// parseMetricsTime takes the two forms the Prometheus API defines: float
|
||||
// unix seconds and RFC3339. vmselect would also guess milliseconds from the
|
||||
// digit count and accept relative times; a guessed unit silently moves the
|
||||
// window, so those are refused. The result is cut to the millisecond, as
|
||||
// formatTime sends it, so the limits see the window vmselect will use.
|
||||
func parseMetricsTime(key, v string) (time.Time, *HTTPError) {
|
||||
if sec, frac, ok := strings.Cut(v, "."); isDigits(sec) && (!ok || isDigits(frac)) {
|
||||
if len(sec) > 10 {
|
||||
return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: not unix seconds", key, v)
|
||||
}
|
||||
secs, _ := strconv.ParseInt(sec, 10, 64)
|
||||
var nanos int64
|
||||
if ok {
|
||||
nanos, _ = strconv.ParseInt((frac + "000000000")[:9], 10, 64)
|
||||
}
|
||||
return metricsTimeInRange(key, v, time.Unix(secs, nanos))
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339Nano, v)
|
||||
if err != nil {
|
||||
return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q", key, v)
|
||||
}
|
||||
return metricsTimeInRange(key, v, t)
|
||||
}
|
||||
|
||||
// maxMetricsTime is the last instant vmselect can read: it takes an
|
||||
// RFC3339 time through time.UnixNano (timeutil.ParseTimeAt).
|
||||
var maxMetricsTime = time.Unix(0, math.MaxInt64)
|
||||
|
||||
func metricsTimeInRange(key, v string, t time.Time) (time.Time, *HTTPError) {
|
||||
if t.Before(time.Unix(0, 0)) || !t.Before(maxMetricsTime) {
|
||||
return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: outside 1970..2262", key, v)
|
||||
}
|
||||
return time.UnixMilli(t.UnixMilli()).UTC(), nil
|
||||
}
|
||||
|
||||
// formatTime writes RFC3339 with milliseconds. vmselect reads a number
|
||||
// without "-" as a unix time whose unit it guesses from its size
|
||||
// (timeutil.getUnixTimestampNanoseconds): "1000000.000" is taken as
|
||||
// 1000000000 seconds, and anything past 2038 in milliseconds as
|
||||
// microseconds. A dated form never reaches that guess.
|
||||
func formatTime(t time.Time) string {
|
||||
return t.UTC().Format("2006-01-02T15:04:05.000Z07:00")
|
||||
}
|
||||
|
||||
// formatSeconds writes a whole-millisecond duration so that vmselect reads
|
||||
// back exactly those milliseconds. vmselect multiplies the float seconds
|
||||
// by 1e3 and truncates (httputil.GetDuration), so "1.001" arrives as
|
||||
// 1000ms; where the shortest form truncates low, half a millisecond more
|
||||
// lands the truncation on the intended value without reaching the next.
|
||||
func formatSeconds(d time.Duration) string {
|
||||
ms := d.Milliseconds()
|
||||
s := strconv.FormatFloat(float64(ms)/1000, 'f', -1, 64)
|
||||
if secs, _ := strconv.ParseFloat(s, 64); int64(secs*1e3) != ms {
|
||||
s = fmt.Sprintf("%d.%03d5", ms/1000, ms%1000)
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testMetricsLimits() MetricsLimits {
|
||||
return MetricsLimits{MaxRange: 30 * 24 * time.Hour, MaxPoints: 11000, MaxTimeout: 30 * time.Second}
|
||||
}
|
||||
|
||||
const wantMetricsScope = `{namespace="tenant-a"}`
|
||||
|
||||
func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) }
|
||||
|
||||
func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest {
|
||||
t.Helper()
|
||||
req, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("BuildMetricsParams(%s, %v): %v", endpoint, in, err)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func metricsStatus(endpoint string, in url.Values) int {
|
||||
_, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits())
|
||||
if err == nil {
|
||||
return http.StatusOK
|
||||
}
|
||||
return err.Code
|
||||
}
|
||||
|
||||
func TestBuildMetricsParamsAllowlist(t *testing.T) {
|
||||
hostile := url.Values{
|
||||
"extra_filters": {`{namespace=~".+"}`},
|
||||
"extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`},
|
||||
"extra_label": {"namespace=tenant-b"},
|
||||
"match": {`{namespace="tenant-b"}`},
|
||||
"nocache": {"1"},
|
||||
"limit": {"1"},
|
||||
"AccountID": {"7"},
|
||||
"latency_offset": {"1h"},
|
||||
"round_digits": {"2"},
|
||||
}
|
||||
cases := map[string]url.Values{
|
||||
"api/v1/query": {"query": {"up"}, "time": {unix(testNow)}, "step": {"30s"}},
|
||||
"api/v1/query_range": {"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}},
|
||||
"api/v1/series": {"match[]": {"up"}},
|
||||
"api/v1/labels": {"match[]": {"up"}},
|
||||
"api/v1/label/pod/values": {"match[]": {"up"}},
|
||||
}
|
||||
allowed := map[string]bool{"query": true, "start": true, "end": true, "step": true, "time": true, "match[]": true, "timeout": true, "extra_filters[]": true}
|
||||
for endpoint, in := range cases {
|
||||
t.Run(endpoint, func(t *testing.T) {
|
||||
for k, v := range hostile {
|
||||
in[k] = v
|
||||
}
|
||||
req := buildMetrics(t, endpoint, in)
|
||||
if req.path != endpoint {
|
||||
t.Errorf("path = %q", req.path)
|
||||
}
|
||||
for k := range req.params {
|
||||
if !allowed[k] {
|
||||
t.Errorf("parameter %q forwarded: %v", k, req.params)
|
||||
}
|
||||
}
|
||||
if got := req.params["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope {
|
||||
t.Errorf("extra_filters[] = %q, want exactly [%s]", got, wantMetricsScope)
|
||||
}
|
||||
for _, m := range req.params["match[]"] {
|
||||
if strings.Contains(m, "tenant-b") {
|
||||
t.Errorf("client match without brackets forwarded as match[]: %q", m)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceSelectorEscapes(t *testing.T) {
|
||||
for _, ns := range []string{"tenant-a", `a"} or {namespace=~".+`, `back\slash`, "new\nline"} {
|
||||
sel := namespaceSelector(ns)
|
||||
lit, ok := strings.CutPrefix(sel, "{namespace=")
|
||||
lit, ok2 := strings.CutSuffix(lit, "}")
|
||||
if !ok || !ok2 {
|
||||
t.Fatalf("selector %q has the wrong shape", sel)
|
||||
}
|
||||
// MetricsQL reads a double-quoted label value with strconv.Unquote
|
||||
// (metricsql extractStringValue).
|
||||
got, err := strconv.Unquote(lit)
|
||||
if err != nil || got != ns {
|
||||
t.Errorf("selector %q decodes to %q, %v; want %q", sel, got, err, ns)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricsParamsLabelEndpointsAlwaysMatch(t *testing.T) {
|
||||
for _, endpoint := range []string{"api/v1/labels", "api/v1/label/__name__/values"} {
|
||||
req := buildMetrics(t, endpoint, url.Values{})
|
||||
if got := req.params["match[]"]; len(got) != 1 || got[0] != wantMetricsScope {
|
||||
t.Errorf("%s without match[] sends %q, want the namespace selector", endpoint, got)
|
||||
}
|
||||
req = buildMetrics(t, endpoint, url.Values{"match[]": {"up", `{job="x"}`}})
|
||||
if got := req.params["match[]"]; len(got) != 2 || got[0] != "up" || got[1] != `{job="x"}` {
|
||||
t.Errorf("%s client match[] = %q, want them kept", endpoint, got)
|
||||
}
|
||||
}
|
||||
if got := metricsStatus("api/v1/series", url.Values{}); got != http.StatusBadRequest {
|
||||
t.Errorf("series without match[] = %d, want 400", got)
|
||||
}
|
||||
if got := metricsStatus("api/v1/series", url.Values{"match[]": {""}}); got != http.StatusBadRequest {
|
||||
t.Errorf("empty match[] = %d, want 400", got)
|
||||
}
|
||||
many := url.Values{}
|
||||
for i := range maxMetricsMatchers + 1 {
|
||||
many.Add("match[]", "up"+strconv.Itoa(i))
|
||||
}
|
||||
if got := metricsStatus("api/v1/labels", many); got != http.StatusUnprocessableEntity {
|
||||
t.Errorf("%d match[] = %d, want 422", maxMetricsMatchers+1, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricsParamsLimits(t *testing.T) {
|
||||
ago := func(d time.Duration) string { return unix(testNow.Add(-d)) }
|
||||
day := 24 * time.Hour
|
||||
rng := func(start, end, step string) url.Values {
|
||||
return url.Values{"query": {"up"}, "start": {start}, "end": {end}, "step": {step}}
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
in url.Values
|
||||
want int
|
||||
}{
|
||||
{"range at the limit", "api/v1/query_range", rng(ago(30*day), ago(0), "1h"), http.StatusOK},
|
||||
{"range above the limit", "api/v1/query_range", rng(ago(31*day), ago(0), "1h"), http.StatusUnprocessableEntity},
|
||||
{"points at the limit", "api/v1/query_range", rng(ago(10999*time.Minute), ago(0), "1m"), http.StatusOK},
|
||||
{"points above the limit", "api/v1/query_range", rng(ago(11000*time.Minute), ago(0), "1m"), http.StatusUnprocessableEntity},
|
||||
{"points above the limit after rounding the step up", "api/v1/query_range", rng(ago(22*time.Second), ago(0), "0.0019"), http.StatusUnprocessableEntity},
|
||||
// 10.9999s admits 11000 points, but vmselect gets both bounds cut to
|
||||
// the millisecond: 11.000s, 11001 points.
|
||||
{"points above the limit once the bounds are cut to milliseconds", "api/v1/query_range", rng("2026-11-04T11:59:49.0009Z", "2026-11-04T12:00:00.0008Z", "0.001"), http.StatusUnprocessableEntity},
|
||||
{"overflowing step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "2562047h47m16.854775807s"), http.StatusBadRequest},
|
||||
{"overflowing timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"2562047h47m16.854775807s"}}, http.StatusBadRequest},
|
||||
{"step above the range limit", "api/v1/query_range", rng(ago(time.Hour), ago(0), "31d"), http.StatusUnprocessableEntity},
|
||||
{"zero step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0"), http.StatusBadRequest},
|
||||
{"sub-millisecond step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0.0001"), http.StatusBadRequest},
|
||||
{"missing step", "api/v1/query_range", url.Values{"query": {"up"}, "start": {ago(time.Hour)}, "end": {ago(0)}}, http.StatusBadRequest},
|
||||
{"missing start", "api/v1/query_range", url.Values{"query": {"up"}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest},
|
||||
{"missing query", "api/v1/query_range", url.Values{"start": {ago(time.Hour)}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest},
|
||||
{"inverted range", "api/v1/query_range", rng(ago(0), ago(time.Hour), "1m"), http.StatusBadRequest},
|
||||
{"millisecond timestamp", "api/v1/query_range", rng(ago(time.Hour)+"000", ago(0)+"000", "1m"), http.StatusBadRequest},
|
||||
{"relative time", "api/v1/query_range", rng("-1h", "now", "1m"), http.StatusBadRequest},
|
||||
{"rfc3339", "api/v1/query_range", rng("2026-11-04T11:00:00Z", "2026-11-04T12:00:00.5Z", "15s"), http.StatusOK},
|
||||
{"instant without time", "api/v1/query", url.Values{"query": {"up"}}, http.StatusOK},
|
||||
{"instant step above the range limit", "api/v1/query", url.Values{"query": {"up"}, "step": {"31d"}}, http.StatusUnprocessableEntity},
|
||||
{"instant bad time", "api/v1/query", url.Values{"query": {"up"}, "time": {"yesterday"}}, http.StatusBadRequest},
|
||||
{"negative timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"-1s"}}, http.StatusBadRequest},
|
||||
{"series range above the limit", "api/v1/series", url.Values{"match[]": {"up"}, "start": {ago(31 * day)}}, http.StatusUnprocessableEntity},
|
||||
{"labels inverted range", "api/v1/labels", url.Values{"start": {ago(0)}, "end": {ago(time.Hour)}}, http.StatusBadRequest},
|
||||
{"invalid label name", "api/v1/label/a-b/values", url.Values{}, http.StatusBadRequest},
|
||||
{"unknown endpoint", "api/v1/status/tsdb", url.Values{}, http.StatusNotFound},
|
||||
{"export", "api/v1/export", url.Values{"match[]": {"up"}}, http.StatusNotFound},
|
||||
{"label without values", "api/v1/label/pod", url.Values{}, http.StatusNotFound},
|
||||
{"root", "", url.Values{}, http.StatusNotFound},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := metricsStatus(tt.endpoint, tt.in); got != tt.want {
|
||||
t.Fatalf("status = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
|
||||
req := buildMetrics(t, "api/v1/query_range", url.Values{
|
||||
"query": {"up"}, "start": {"2026-11-04T11:00:00.25Z"}, "end": {unix(testNow)}, "step": {"1m30s"},
|
||||
})
|
||||
want := url.Values{
|
||||
"query": {"up"}, "start": {"2026-11-04T11:00:00.250Z"}, "end": {"2026-11-04T12:00:00.000Z"}, "step": {"90"},
|
||||
"timeout": {"30"}, "extra_filters[]": {wantMetricsScope},
|
||||
}
|
||||
if req.params.Encode() != want.Encode() {
|
||||
t.Errorf("params = %v\nwant %v", req.params, want)
|
||||
}
|
||||
|
||||
// vmselect drops a sub-millisecond remainder; the points check has to
|
||||
// see the step it will use.
|
||||
for in, want := range map[string]string{"60.0001": "60.001", "0.0019": "0.002"} {
|
||||
req = buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-20 * time.Second))}, "end": {unix(testNow)}, "step": {in}})
|
||||
if got := req.params.Get("step"); got != want {
|
||||
t.Errorf("step %s forwarded as %q, want %s", in, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
for in, want := range map[string]string{"": "30", "10s": "10", "5m": "30", "1.5": "1.5"} {
|
||||
v := url.Values{"query": {"up"}}
|
||||
if in != "" {
|
||||
v.Set("timeout", in)
|
||||
}
|
||||
if got := buildMetrics(t, "api/v1/query", v).params.Get("timeout"); got != want {
|
||||
t.Errorf("timeout %q forwarded as %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
req = buildMetrics(t, "api/v1/labels", url.Values{})
|
||||
if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" {
|
||||
t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end"))
|
||||
}
|
||||
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-30*time.Second)) {
|
||||
t.Errorf("instant query without time: params %v, start %v", q.params, q.start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsStoreByRange(t *testing.T) {
|
||||
cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour}
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
in url.Values
|
||||
want string
|
||||
}{
|
||||
{"instant now", "api/v1/query", url.Values{"query": {"up"}}, "shortterm"},
|
||||
{"instant within retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-71 * time.Hour))}}, "shortterm"},
|
||||
{"instant past retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-73 * time.Hour))}}, "longterm"},
|
||||
{"live graph", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-6 * time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, "shortterm"},
|
||||
{"range fits with lookback", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-71 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"},
|
||||
{"range lookback past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"},
|
||||
{"range past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-7 * 24 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"},
|
||||
{"old window ending long ago", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-5 * 24 * time.Hour))}, "end": {unix(testNow.Add(-4 * 24 * time.Hour))}, "step": {"600"}}, "longterm"},
|
||||
{"labels default window", "api/v1/labels", url.Values{}, "shortterm"},
|
||||
{"series past retention", "api/v1/series", url.Values{"match[]": {"up"}, "start": {unix(testNow.Add(-4 * 24 * time.Hour))}}, "longterm"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := buildMetrics(t, tt.endpoint, tt.in)
|
||||
if got := cfg.store(req.start, testNow); got != tt.want {
|
||||
t.Fatalf("store = %s, want %s", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryLookback(t *testing.T) {
|
||||
tests := []struct {
|
||||
query string
|
||||
step time.Duration
|
||||
want time.Duration
|
||||
}{
|
||||
{`up`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 35*time.Minute},
|
||||
{`rate(x[300])`, time.Minute, 20 * time.Minute},
|
||||
{`rate(x[ 1h30m ])`, time.Minute, 105 * time.Minute},
|
||||
{`max_over_time(x[1h:5m])`, time.Minute, 115 * time.Minute},
|
||||
{`max_over_time(x[1h:])`, time.Minute, 83 * time.Minute},
|
||||
{`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 42*time.Hour + 35*time.Minute},
|
||||
{`rate(rate(x))`, time.Hour, 2*(3*time.Hour+5*time.Minute) + 2*time.Hour + 5*time.Minute},
|
||||
{`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute},
|
||||
{`sum(x OFFSET 2h)`, 5 * time.Minute, 155 * time.Minute},
|
||||
{`rate(x[5m] offset 1h)`, time.Minute, 80 * time.Minute},
|
||||
{`x offset -1h`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`x offset - 1h`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`node_offset_seconds + offset_total + with_x`, 5 * time.Minute, 15 * time.Minute},
|
||||
{`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute},
|
||||
{`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.query, func(t *testing.T) {
|
||||
got, ok := queryLookback(tt.query, tt.step)
|
||||
if !ok || got != tt.want {
|
||||
t.Fatalf("queryLookback = %s, %t; want %s, true", got, ok, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryLookbackUnknown(t *testing.T) {
|
||||
for _, q := range []string{
|
||||
`x @ 123`,
|
||||
`x @ start()`,
|
||||
`x{a="a@b"}`,
|
||||
`rate(x[5i])`,
|
||||
`rate(x[$__interval])`,
|
||||
`x[1h-5m]`,
|
||||
`x[7D]`,
|
||||
`x[1.d]`,
|
||||
`x[]`,
|
||||
`rate(x[7d)`,
|
||||
`x offset`,
|
||||
`x offset w`,
|
||||
`WITH (w = 7d) rate(x[w])`,
|
||||
`x{a="["}`,
|
||||
`x{a="[5m"}[7d]`,
|
||||
`x{path="/offset/1"}`,
|
||||
`x[99999999999y]`,
|
||||
`x offset 99999999999y`,
|
||||
`x offset 3_000_000`,
|
||||
`WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))`,
|
||||
`with (s(m) = m offset 2d) s(s(x))`,
|
||||
} {
|
||||
t.Run(q, func(t *testing.T) {
|
||||
if got, ok := queryLookback(q, 5*time.Minute); ok {
|
||||
t.Fatalf("queryLookback = %s, true; want unknown", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsStoreByLookback(t *testing.T) {
|
||||
cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour}
|
||||
rangeFrom := func(q string, ago time.Duration, step string) url.Values {
|
||||
return url.Values{"query": {q}, "start": {unix(testNow.Add(-ago))}, "end": {unix(testNow)}, "step": {step}}
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
in url.Values
|
||||
want string
|
||||
}{
|
||||
{"instant short window", "api/v1/query", url.Values{"query": {"rate(x[5m])"}}, "shortterm"},
|
||||
{"instant 7d window", "api/v1/query", url.Values{"query": {"increase(x[7d])"}}, "longterm"},
|
||||
{"instant offset 10d", "api/v1/query", url.Values{"query": {"x offset 10d"}}, "longterm"},
|
||||
{"instant negative offset", "api/v1/query", url.Values{"query": {"x offset -1h"}}, "shortterm"},
|
||||
{"instant at timestamp", "api/v1/query", url.Values{"query": {"x @ 123"}}, "longterm"},
|
||||
{"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"},
|
||||
{"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"},
|
||||
{"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"},
|
||||
{"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"x[71h54m59s]"}, "step": {"0.001"}}, "longterm"},
|
||||
{"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"},
|
||||
{"instant offset with underscores", "api/v1/query", url.Values{"query": {"x offset 3_000_000"}}, "longterm"},
|
||||
{"instant WITH template", "api/v1/query", url.Values{"query": {"WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))"}}, "longterm"},
|
||||
{"instant implicit subqueries", "api/v1/query", url.Values{"query": {"rate(rate(rate(rate(rate(rate(x))))))"}, "step": {"43200"}}, "longterm"},
|
||||
{"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"},
|
||||
{"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"},
|
||||
{"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"},
|
||||
{"quoted offset before offset", "api/v1/query", url.Values{"query": {`x{a="offset 1m"} offset 10d`}}, "longterm"},
|
||||
{"range short window", "api/v1/query_range", rangeFrom("rate(x[1h])", 6*time.Hour, "60"), "shortterm"},
|
||||
{"range 7d window", "api/v1/query_range", rangeFrom("increase(x[7d])", 6*time.Hour, "60"), "longterm"},
|
||||
{"range offset past retention", "api/v1/query_range", rangeFrom("x offset 3d", time.Hour, "60"), "longterm"},
|
||||
{"range lookback fits", "api/v1/query_range", rangeFrom("rate(x[5m])", 70*time.Hour, "600"), "shortterm"},
|
||||
{"range lookback crosses retention", "api/v1/query_range", rangeFrom("rate(x[5m])", 71*time.Hour+50*time.Minute, "600"), "longterm"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := buildMetrics(t, tt.endpoint, tt.in)
|
||||
if got := cfg.store(req.start, testNow); got != tt.want {
|
||||
t.Fatalf("store = %s, want %s", got, tt.want)
|
||||
}
|
||||
if q := req.params.Get("query"); q != tt.in.Get("query") {
|
||||
t.Fatalf("query forwarded as %q, want it unchanged", q)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// vmselectMillis repeats how vmselect reads a duration argument
|
||||
// (httputil.GetDuration in lib/httputil/duration.go, v1.136.22 and
|
||||
// v1.139.0): float seconds, multiplied by 1e3 and truncated.
|
||||
func vmselectMillis(t *testing.T, v string) int64 {
|
||||
t.Helper()
|
||||
secs, err := strconv.ParseFloat(v, 64)
|
||||
if err != nil {
|
||||
t.Fatalf("vmselect cannot parse %q: %v", v, err)
|
||||
}
|
||||
return int64(secs * 1e3)
|
||||
}
|
||||
|
||||
// TestFormatSecondsVmselectExact sweeps every step up to 2000s and the
|
||||
// large end of the range: vmselect must read back the milliseconds the
|
||||
// limits were checked with.
|
||||
func TestFormatSecondsVmselectExact(t *testing.T) {
|
||||
check := func(ms int64) {
|
||||
if got := vmselectMillis(t, formatSeconds(time.Duration(ms)*time.Millisecond)); got != ms {
|
||||
t.Fatalf("%dms formatted as %q reaches vmselect as %dms", ms, formatSeconds(time.Duration(ms)*time.Millisecond), got)
|
||||
}
|
||||
}
|
||||
for ms := int64(1); ms <= 2_000_000; ms++ {
|
||||
check(ms)
|
||||
}
|
||||
for ms := maxMetricsDuration.Milliseconds() - 100_000; ms <= maxMetricsDuration.Milliseconds(); ms++ {
|
||||
check(ms)
|
||||
}
|
||||
// A spread of the range in between, on a stride no power of ten divides.
|
||||
for ms := int64(2_000_001); ms < maxMetricsDuration.Milliseconds(); ms += 3_153_599_999 {
|
||||
check(ms)
|
||||
}
|
||||
// Whole seconds keep their plain form.
|
||||
if got := formatSeconds(90 * time.Second); got != "90" {
|
||||
t.Errorf("90s formatted as %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildMetricsParamsPointsAsVmselectCounts: a 1.001s step is 1001ms
|
||||
// to the proxy; vmselect must count the same 11000 points at the limit.
|
||||
func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) {
|
||||
end := testNow
|
||||
step := 1001 * time.Millisecond
|
||||
rng := func(points int64) url.Values {
|
||||
start := end.Add(-time.Duration(points-1) * step)
|
||||
return url.Values{"query": {"up"}, "start": {start.Format(time.RFC3339Nano)}, "end": {end.Format(time.RFC3339Nano)}, "step": {"1.001"}}
|
||||
}
|
||||
req := buildMetrics(t, "api/v1/query_range", rng(11000))
|
||||
stepMs := vmselectMillis(t, req.params.Get("step"))
|
||||
startMs, endMs := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end"))
|
||||
if points := (endMs-startMs)/stepMs + 1; stepMs != 1001 || points != 11000 {
|
||||
t.Fatalf("vmselect reads step %dms and %d points, want 1001ms and 11000", stepMs, points)
|
||||
}
|
||||
if got := metricsStatus("api/v1/query_range", rng(11001)); got != http.StatusUnprocessableEntity {
|
||||
t.Fatalf("11001 points answered %d, want 422", got)
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -56,6 +57,7 @@ type Config struct {
|
||||
// means the in-cluster Service of the "generic" logs storage.
|
||||
UpstreamURL func(store string) string
|
||||
Logs LogsLimits
|
||||
Metrics MetricsConfig
|
||||
Limiter LimiterConfig
|
||||
RequestTimeout time.Duration
|
||||
TailIdleTimeout time.Duration
|
||||
@@ -84,6 +86,12 @@ func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.
|
||||
return fmt.Sprintf("http://vlselect-generic.%s.svc.%s:9471", store, domain)
|
||||
}
|
||||
}
|
||||
if cfg.Metrics.UpstreamURL == nil {
|
||||
domain, root := cfg.ClusterDomain, cfg.Metrics.RootTenant
|
||||
cfg.Metrics.UpstreamURL = func(store string) string {
|
||||
return fmt.Sprintf("http://vmselect-%s.%s.svc.%s:8481/select/0/prometheus", store, root, domain)
|
||||
}
|
||||
}
|
||||
s := &Server{
|
||||
cfg: cfg,
|
||||
auth: auth,
|
||||
@@ -167,7 +175,7 @@ func writeError(w http.ResponseWriter, e *HTTPError) {
|
||||
// serve answers a root it has no handler for before authenticating, so an
|
||||
// unknown path costs no review calls.
|
||||
func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError {
|
||||
if route.Kind != NamespaceLogs {
|
||||
if route.Kind != NamespaceLogs && route.Kind != NamespaceMetrics {
|
||||
return errorf(http.StatusNotFound, "not found")
|
||||
}
|
||||
user, herr := s.authorize(r, route)
|
||||
@@ -182,6 +190,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
|
||||
w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10))
|
||||
return errorf(http.StatusTooManyRequests, "request rate limit exceeded")
|
||||
}
|
||||
if route.Kind == NamespaceMetrics {
|
||||
return s.serveMetrics(w, r, route, user)
|
||||
}
|
||||
endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/")
|
||||
if !ok {
|
||||
return errorf(http.StatusNotFound, "not found")
|
||||
@@ -226,29 +237,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
|
||||
ctx, stop = context.WithTimeout(ctx, limit)
|
||||
defer stop()
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, strings.NewReader(params.Encode()))
|
||||
if err != nil {
|
||||
return errorf(http.StatusInternalServerError, "building upstream request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("AccountID", "0")
|
||||
req.Header.Set("ProjectID", "0")
|
||||
tenant := http.Header{}
|
||||
tenant.Set("AccountID", "0")
|
||||
tenant.Set("ProjectID", "0")
|
||||
if tail {
|
||||
// vlselect holds even the headers of a gzip-encoded tail until the
|
||||
// first line; an explicit value stops the transport from asking for gzip.
|
||||
req.Header.Set("Accept-Encoding", "identity")
|
||||
tenant.Set("Accept-Encoding", "identity")
|
||||
}
|
||||
resp, err := s.client.Do(req)
|
||||
switch {
|
||||
case err == nil:
|
||||
case r.Context().Err() != nil:
|
||||
return errorf(http.StatusBadGateway, "client went away")
|
||||
case errors.Is(context.Cause(ctx), context.DeadlineExceeded):
|
||||
return errorf(http.StatusGatewayTimeout, "log store did not answer within %s", limit)
|
||||
default:
|
||||
slog.Warn("log store unreachable", "store", store, "error", err)
|
||||
return errorf(http.StatusBadGateway, "log store unreachable")
|
||||
resp, herr := s.post(ctx, r, s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, params, "log store", tenant)
|
||||
if herr != nil {
|
||||
return herr
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
@@ -280,6 +279,75 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError {
|
||||
release, ok := s.limits.acquire(user)
|
||||
if !ok {
|
||||
return tooManyConcurrent(w)
|
||||
}
|
||||
defer release()
|
||||
if herr := s.readForm(w, r); herr != nil {
|
||||
return herr
|
||||
}
|
||||
now := s.now()
|
||||
req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits)
|
||||
if herr != nil {
|
||||
return herr
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
if s.cfg.RequestTimeout > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, s.cfg.RequestTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
store := s.cfg.Metrics.store(req.start, now)
|
||||
resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil)
|
||||
if herr != nil {
|
||||
return herr
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "" {
|
||||
w.Header().Set("Content-Type", ct)
|
||||
}
|
||||
// As for a bounded logs answer: the client must take it by the request
|
||||
// deadline, and a break-off cuts the connection.
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
_ = http.NewResponseController(w).SetWriteDeadline(deadline)
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
if _, err := io.Copy(w, resp.Body); err != nil {
|
||||
slog.Debug("copying upstream response", "error", err)
|
||||
return errAborted
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// post sends params as a form with only the given headers, so nothing the
|
||||
// client sent reaches the store, and tells an unreachable store (502) from
|
||||
// one that missed its deadline or, for a tail, its idle timeout (504).
|
||||
func (s *Server) post(ctx context.Context, r *http.Request, target string, params url.Values, what string, header http.Header) (*http.Response, *HTTPError) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(params.Encode()))
|
||||
if err != nil {
|
||||
return nil, errorf(http.StatusInternalServerError, "building upstream request: %v", err)
|
||||
}
|
||||
for k, v := range header {
|
||||
req.Header[k] = v
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
resp, err := s.client.Do(req)
|
||||
switch {
|
||||
case err == nil:
|
||||
return resp, nil
|
||||
case r.Context().Err() != nil:
|
||||
return nil, errorf(http.StatusBadGateway, "client went away")
|
||||
case errors.Is(context.Cause(ctx), context.DeadlineExceeded):
|
||||
return nil, errorf(http.StatusGatewayTimeout, "%s did not answer in time", what)
|
||||
default:
|
||||
slog.Warn(what+" unreachable", "url", target, "error", err)
|
||||
return nil, errorf(http.StatusBadGateway, "%s unreachable", what)
|
||||
}
|
||||
}
|
||||
|
||||
// concurrencyRetryAfter is a hint only: a slot frees when another request
|
||||
// of the user ends, which the proxy cannot predict, and one second keeps
|
||||
// a polling client from retrying in a tight loop.
|
||||
|
||||
@@ -421,15 +421,20 @@ func stalledAPIServer(t *testing.T, stallTokens bool) kubernetes.Interface {
|
||||
// a bounded review call ends the request.
|
||||
func TestServerStalledReviewAnswersInTime(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
name, path string
|
||||
stallTokens bool
|
||||
}{{"token review", true}, {"subject access review", false}} {
|
||||
}{
|
||||
{"logs token review", logsRoot + "query", true},
|
||||
{"logs subject access review", logsRoot + "query", false},
|
||||
{"metrics token review", metricsRoot + "api/v1/query", true},
|
||||
{"metrics subject access review", metricsRoot + "api/v1/query", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
h := newHarness(t, Config{RequestTimeout: 200 * time.Millisecond}, nil)
|
||||
h.server.auth = NewAuthorizer(stalledAPIServer(t, tc.stallTokens), 10*time.Second)
|
||||
got := make(chan int, 1)
|
||||
go func() {
|
||||
resp := h.do(http.MethodPost, logsRoot+"query", "good", url.Values{"query": {"*"}}, nil)
|
||||
resp := h.do(http.MethodPost, tc.path, "good", url.Values{"query": {"up"}}, nil)
|
||||
_ = resp.Body.Close()
|
||||
got <- resp.StatusCode
|
||||
}()
|
||||
@@ -444,3 +449,56 @@ func TestServerStalledReviewAnswersInTime(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerMetricsRequestLifetime repeats the body, write and break-off
|
||||
// checks on the metrics root, which has its own handler.
|
||||
func TestServerMetricsRequestLifetime(t *testing.T) {
|
||||
query := metricsRoot + "api/v1/query?query=up"
|
||||
t.Run("slow body", func(t *testing.T) {
|
||||
h := newHarness(t, Config{
|
||||
RequestTimeout: 200 * time.Millisecond,
|
||||
Limiter: LimiterConfig{MaxConcurrentPerUser: 1},
|
||||
}, promUpstream)
|
||||
pipe := servePipe(t, h.server)
|
||||
conn := pipe.send(t, "POST "+metricsRoot+"api/v1/query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+
|
||||
"Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=")
|
||||
eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 },
|
||||
"a metrics request with an unfinished body does not hold a concurrency slot")
|
||||
if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout {
|
||||
t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode)
|
||||
}
|
||||
eventually(t, time.Second, func() bool { return slotsHeld(h) == 0 }, "the slot of an unfinished body was not released")
|
||||
})
|
||||
t.Run("stalled client", func(t *testing.T) {
|
||||
// Larger than the server's write buffer, so the handler itself
|
||||
// blocks on the client.
|
||||
big := `{"status":"success","data":"` + strings.Repeat("x", 64<<10) + `"}`
|
||||
h := newHarness(t, Config{
|
||||
RequestTimeout: 200 * time.Millisecond,
|
||||
Limiter: LimiterConfig{MaxConcurrentPerUser: 1},
|
||||
}, func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, big) })
|
||||
pipe := servePipe(t, h.server)
|
||||
pipe.send(t, "GET "+query+" HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n")
|
||||
eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, "the stalled metrics request never held its slot")
|
||||
eventually(t, 2*time.Second, func() bool { return slotsHeld(h) == 0 },
|
||||
"a metrics client that stopped reading keeps its concurrency slot")
|
||||
})
|
||||
t.Run("truncated answer", func(t *testing.T) {
|
||||
h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Length", "100")
|
||||
_, _ = io.WriteString(w, `{"status":"success",`)
|
||||
})
|
||||
// Nothing has been flushed yet when the copy fails, so the
|
||||
// connection may close before the status line.
|
||||
req, _ := http.NewRequest(http.MethodGet, h.proxy.URL+query, nil)
|
||||
req.Header.Set("Authorization", "Bearer good")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if _, err := io.ReadAll(resp.Body); err == nil {
|
||||
t.Fatal("a truncated store answer reached the client as a complete response")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
authorizationv1 "k8s.io/api/authorization/v1"
|
||||
)
|
||||
|
||||
const metricsRoot = "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/"
|
||||
|
||||
func promUpstream(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
|
||||
}
|
||||
|
||||
func TestServerMetricsRebuildsUpstreamRequest(t *testing.T) {
|
||||
h := newHarness(t, Config{}, promUpstream)
|
||||
hostile := url.Values{
|
||||
"query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"},
|
||||
"start": {unix(testNow.Add(-time.Hour))},
|
||||
"end": {unix(testNow)},
|
||||
"step": {"60"},
|
||||
"extra_filters": {`{namespace=~".+"}`},
|
||||
"extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`},
|
||||
"extra_label": {"namespace=tenant-b"},
|
||||
"nocache": {"1"},
|
||||
}
|
||||
headers := map[string]string{"AccountID": "9", "X-Forwarded-For": "1.2.3.4", "Cookie": "kc-access=secret"}
|
||||
|
||||
resp := h.do(http.MethodGet, metricsRoot+"api/v1/query_range?"+hostile.Encode(), "good", nil, headers)
|
||||
_ = resp.Body.Close()
|
||||
resp = h.do(http.MethodPost, metricsRoot+"api/v1/query_range?extra_filters[]="+url.QueryEscape(`{namespace="x"}`), "good", hostile, headers)
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != "application/json" {
|
||||
t.Fatalf("POST answered %d %s %q", resp.StatusCode, resp.Header.Get("Content-Type"), body)
|
||||
}
|
||||
|
||||
reqs := h.requests()
|
||||
if len(reqs) != 2 {
|
||||
t.Fatalf("upstream saw %d requests", len(reqs))
|
||||
}
|
||||
if reqs[0].form.Encode() != reqs[1].form.Encode() {
|
||||
t.Fatalf("GET and POST built different upstream requests:\n%v\n%v", reqs[0].form, reqs[1].form)
|
||||
}
|
||||
if n := h.kube.sars.Load(); n != 1 {
|
||||
t.Fatalf("SubjectAccessReviews = %d, want 1: POST must reuse the GET decision", n)
|
||||
}
|
||||
want := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "tenant-a"}
|
||||
if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want {
|
||||
t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want)
|
||||
}
|
||||
for _, r := range reqs {
|
||||
if r.path != "/api/v1/query_range" {
|
||||
t.Errorf("upstream path = %s", r.path)
|
||||
}
|
||||
if got := r.form["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope {
|
||||
t.Errorf("extra_filters[] = %q", got)
|
||||
}
|
||||
if r.form.Has("extra_filters") || r.form.Has("extra_label") || r.form.Has("nocache") {
|
||||
t.Errorf("client parameter forwarded: %v", r.form)
|
||||
}
|
||||
for _, k := range []string{"Authorization", "AccountID", "X-Forwarded-For", "Cookie"} {
|
||||
if r.headers.Get(k) != "" {
|
||||
t.Errorf("client header %s forwarded", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerMetricsStoreByRange(t *testing.T) {
|
||||
h := newHarness(t, Config{}, promUpstream)
|
||||
for _, path := range []string{
|
||||
"api/v1/query?query=up",
|
||||
"api/v1/query_range?query=up&step=600&start=" + unix(testNow.Add(-7*24*time.Hour)) + "&end=" + unix(testNow),
|
||||
"api/v1/label/pod/values",
|
||||
} {
|
||||
if got := h.status(http.MethodGet, metricsRoot+path, "good"); got != http.StatusOK {
|
||||
t.Fatalf("%s answered %d", path, got)
|
||||
}
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if want := []string{"shortterm", "longterm", "shortterm"}; len(h.stores) != 3 || h.stores[0] != want[0] || h.stores[1] != want[1] || h.stores[2] != want[2] {
|
||||
t.Fatalf("stores = %v, want %v", h.stores, want)
|
||||
}
|
||||
if r := h.got[2]; r.path != "/api/v1/label/pod/values" || r.form.Get("match[]") != wantMetricsScope {
|
||||
t.Fatalf("label values upstream = %s %v", r.path, r.form)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerMetricsStatusCodes(t *testing.T) {
|
||||
h := newHarness(t, Config{}, promUpstream)
|
||||
tooLong := "api/v1/query_range?query=up&step=3600&start=" + unix(testNow.Add(-31*24*time.Hour)) + "&end=" + unix(testNow)
|
||||
if got := h.status(http.MethodGet, metricsRoot+tooLong, "good"); got != http.StatusUnprocessableEntity {
|
||||
t.Errorf("range above the limit answered %d, want 422", got)
|
||||
}
|
||||
if got := h.status(http.MethodGet, metricsRoot+"api/v1/series", "good"); got != http.StatusBadRequest {
|
||||
t.Errorf("series without match[] answered %d, want 400", got)
|
||||
}
|
||||
if got := h.status(http.MethodPut, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusMethodNotAllowed {
|
||||
t.Errorf("PUT answered %d, want 405", got)
|
||||
}
|
||||
if n := len(h.requests()); n != 0 {
|
||||
t.Errorf("upstream saw %d requests for refused ones", n)
|
||||
}
|
||||
|
||||
h2 := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) {
|
||||
http.Error(w, `{"status":"error","errorType":"422"}`, http.StatusUnprocessableEntity)
|
||||
})
|
||||
if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity {
|
||||
t.Errorf("upstream 422 answered as %d", got)
|
||||
}
|
||||
h2.upstream.Close()
|
||||
if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusBadGateway {
|
||||
t.Errorf("unreachable upstream answered %d, want 502", got)
|
||||
}
|
||||
|
||||
release := make(chan struct{})
|
||||
t.Cleanup(func() { close(release) })
|
||||
h3 := newHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(_ http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case <-release:
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
})
|
||||
if got := h3.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout {
|
||||
t.Errorf("slow upstream answered %d, want 504", got)
|
||||
}
|
||||
|
||||
h4 := newHarness(t, Config{Limiter: LimiterConfig{RatePerUser: 0.001, RateBurst: 1}}, promUpstream)
|
||||
h4.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good")
|
||||
limited := h4.do(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good", nil, nil)
|
||||
_ = limited.Body.Close()
|
||||
if limited.StatusCode != http.StatusTooManyRequests || limited.Header.Get("Retry-After") != "1000" {
|
||||
t.Errorf("rate-limited request answered %d with Retry-After %q, want 429 with 1000",
|
||||
limited.StatusCode, limited.Header.Get("Retry-After"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerMetricsConcurrency(t *testing.T) {
|
||||
entered, release := make(chan struct{}, 3), make(chan struct{})
|
||||
h := newHarness(t, Config{Limiter: LimiterConfig{MaxConcurrentPerUser: 1}}, func(w http.ResponseWriter, _ *http.Request) {
|
||||
entered <- struct{}{}
|
||||
<-release
|
||||
promUpstream(w, nil)
|
||||
})
|
||||
// Registered after the harness so it runs before the servers close.
|
||||
var once sync.Once
|
||||
t.Cleanup(func() { once.Do(func() { close(release) }) })
|
||||
held := make(chan int, 2)
|
||||
hold := func(token string) {
|
||||
go func() { held <- h.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", token) }()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("request of %s did not reach the upstream", token)
|
||||
}
|
||||
}
|
||||
hold("good")
|
||||
refused := make(chan *http.Response, 1)
|
||||
go func() {
|
||||
resp := h.do(http.MethodPost, metricsRoot+"api/v1/query?query=up", "good", nil, nil)
|
||||
_ = resp.Body.Close()
|
||||
refused <- resp
|
||||
}()
|
||||
select {
|
||||
case got := <-refused:
|
||||
if got.StatusCode != http.StatusTooManyRequests || got.Header.Get("Retry-After") != "1" {
|
||||
t.Fatalf("concurrent request answered %d with Retry-After %q, want 429 with 1", got.StatusCode, got.Header.Get("Retry-After"))
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("concurrent request of the same user reached the upstream")
|
||||
}
|
||||
hold("bob-token")
|
||||
once.Do(func() { close(release) })
|
||||
for range 2 {
|
||||
if got := <-held; got != http.StatusOK {
|
||||
t.Fatalf("held request answered %d", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultMetricsUpstreamURL(t *testing.T) {
|
||||
s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil)
|
||||
if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want {
|
||||
t.Fatalf("upstream = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
@@ -52,12 +52,12 @@ type mapStores map[string]string
|
||||
func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil }
|
||||
|
||||
// newHarness wires the proxy to a fake apiserver that knows tokens "good"
|
||||
// (alice) and "bob-token" (bob), both allowed list logs in tenant-a only,
|
||||
// and to an upstream served by handler.
|
||||
// (alice) and "bob-token" (bob), both allowed list logs and metrics in
|
||||
// tenant-a only, and to an upstream served by handler.
|
||||
func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
|
||||
h := &harness{t: t}
|
||||
h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool {
|
||||
return a.Namespace == "tenant-a" && a.Resource == "logs"
|
||||
return a.Namespace == "tenant-a" && (a.Resource == "logs" || a.Resource == "metrics")
|
||||
})
|
||||
h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
@@ -80,6 +80,13 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
|
||||
h.mu.Unlock()
|
||||
return h.upstream.URL
|
||||
}
|
||||
cfg.Metrics.UpstreamURL = cfg.UpstreamURL
|
||||
if cfg.Metrics.ShortTermStore == "" {
|
||||
cfg.Metrics.ShortTermStore, cfg.Metrics.LongTermStore, cfg.Metrics.ShortTermRetention = "shortterm", "longterm", 72*time.Hour
|
||||
}
|
||||
if cfg.Metrics.Limits == (MetricsLimits{}) {
|
||||
cfg.Metrics.Limits = testMetricsLimits()
|
||||
}
|
||||
if cfg.Logs == (LogsLimits{}) {
|
||||
cfg.Logs = testLimits()
|
||||
}
|
||||
@@ -144,7 +151,9 @@ func TestServerStatusCodes(t *testing.T) {
|
||||
{"other namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/logs/select/logsql/query", "good", http.StatusForbidden},
|
||||
{"invalid namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/Tenant/logs/select/logsql/query", "good", http.StatusBadRequest},
|
||||
{"unknown root", http.MethodGet, "/api/v1/namespaces/tenant-a/pods", "good", http.StatusNotFound},
|
||||
{"namespace metrics not served yet", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query", "good", http.StatusNotFound},
|
||||
{"unknown metrics endpoint", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/status/tsdb", "good", http.StatusNotFound},
|
||||
{"metrics in another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/metrics/api/v1/query?query=up", "good", http.StatusForbidden},
|
||||
{"metrics without token", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query?query=up", "", http.StatusUnauthorized},
|
||||
{"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound},
|
||||
{"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound},
|
||||
{"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound},
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"math"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// vmselectTimeMillis repeats how vmselect v1.136.0-cluster reads a time
|
||||
// argument (httputil.GetTime -> timeutil.ParseTimeMsec -> ParseTimeAt in
|
||||
// lib/timeutil/time.go) for the forms a client or the proxy may send: a
|
||||
// value without "-" is a number whose unit vmselect guesses from its size
|
||||
// (getUnixTimestampNanoseconds), anything else with a zone is RFC3339 read
|
||||
// through time.UnixNano. Relative and partial dates are left out; the
|
||||
// proxy refuses them.
|
||||
func vmselectTimeMillis(t *testing.T, s string) int64 {
|
||||
t.Helper()
|
||||
if strings.Contains(s, "-") {
|
||||
ts, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
t.Fatalf("vmselect cannot parse %q: %v", s, err)
|
||||
}
|
||||
return int64(math.Round(float64(ts.UnixNano()) / 1e6))
|
||||
}
|
||||
intPart, frac, _ := strings.Cut(s, ".")
|
||||
n, err := strconv.ParseInt(intPart+frac, 10, 64)
|
||||
if err != nil {
|
||||
t.Fatalf("vmselect cannot parse %q: %v", s, err)
|
||||
}
|
||||
for e := len(frac); e%3 != 0; e++ {
|
||||
n *= 10
|
||||
}
|
||||
var nanos int64
|
||||
switch {
|
||||
case n < 1<<31:
|
||||
nanos = n * 1e9
|
||||
case n < 1e3*(1<<31):
|
||||
nanos = n * 1e6
|
||||
case n < 1e6*(1<<31):
|
||||
nanos = n * 1e3
|
||||
default:
|
||||
nanos = n
|
||||
}
|
||||
return int64(math.Round(float64(nanos) / 1e6))
|
||||
}
|
||||
|
||||
// TestVmselectReadsProxyTimes sends times across the range the proxy
|
||||
// accepts, in both client forms, and checks that vmselect reads the
|
||||
// forwarded value as the same instant the proxy validated. A number such
|
||||
// as "1000000.000" is seconds to the proxy but milliseconds-as-seconds
|
||||
// guesswork to vmselect, so the proxy must not forward one.
|
||||
func TestVmselectReadsProxyTimes(t *testing.T) {
|
||||
for _, client := range []string{
|
||||
"0", "1000000", "1762400", "2147483", "2147484", "1000000.5",
|
||||
"1793793600", "1793793600.123", "2147483647", "2147483648", "9223372035",
|
||||
"1970-01-01T00:00:00Z", "1970-01-12T13:46:40.001Z", "2026-11-04T12:00:00.123Z", "2262-01-01T00:00:00Z",
|
||||
} {
|
||||
want, herr := parseMetricsTime("time", client)
|
||||
if herr != nil {
|
||||
t.Fatalf("%q refused: %v", client, herr)
|
||||
}
|
||||
req := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}, "time": {client}})
|
||||
sent := req.params.Get("time")
|
||||
if got := vmselectTimeMillis(t, sent); got != want.UnixMilli() {
|
||||
t.Errorf("client time %q forwarded as %q reaches vmselect as %dms, want %dms", client, sent, got, want.UnixMilli())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestVmselectReadsSecondsWithFraction is the window that passed the proxy as
|
||||
// 8.8 days and 30 points while vmselect, reading "1000000.000" as a
|
||||
// thousand times that many seconds, saw 2001..2025 and 30000 points.
|
||||
func TestVmselectReadsSecondsWithFraction(t *testing.T) {
|
||||
in := url.Values{"query": {"up"}, "start": {"1000000"}, "end": {"1762400"}, "step": {"25414"}}
|
||||
req := buildMetrics(t, "api/v1/query_range", in)
|
||||
start, end := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end"))
|
||||
step := vmselectMillis(t, req.params.Get("step"))
|
||||
if start != 1000000*1000 || end != 1762400*1000 {
|
||||
t.Fatalf("vmselect reads %dms..%dms, want 1000000s..1762400s", start, end)
|
||||
}
|
||||
if points := (end-start)/step + 1; points != 30 {
|
||||
t.Fatalf("vmselect counts %d points, want the 30 the proxy checked", points)
|
||||
}
|
||||
labels := buildMetrics(t, "api/v1/labels", url.Values{"start": {"1000000"}, "end": {"1000300"}})
|
||||
if got := vmselectTimeMillis(t, labels.params.Get("start")); got != 1000000*1000 {
|
||||
t.Fatalf("label API start reaches vmselect as %dms, want 1000000s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMetricsTimeRange(t *testing.T) {
|
||||
// vmselect reads an RFC3339 time through time.UnixNano, which is
|
||||
// undefined from 2262-04-11 on.
|
||||
// Before 1970 vmselect clamps to 0, which would quietly answer for a
|
||||
// window other than the one asked for.
|
||||
for _, v := range []string{"9223372037", "9999999999", "2262-04-12T00:00:00Z", "9999-01-01T00:00:00Z", "1969-12-31T23:59:59Z", "1969-12-31T23:59:59.999Z"} {
|
||||
if _, herr := parseMetricsTime("time", v); herr == nil {
|
||||
t.Errorf("%q accepted, want refused", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -424,7 +424,8 @@ func (w *WrappedControllerService) ControllerUnpublishVolume(ctx context.Context
|
||||
pvc, err := w.infraClient.CoreV1().PersistentVolumeClaims(w.infraNamespace).Get(ctx, dvName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return &csi.ControllerUnpublishVolumeResponse{}, nil
|
||||
// A failed PVC creation can still leave a hotplug entry in the VM.
|
||||
return w.ControllerService.ControllerUnpublishVolume(ctx, req)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -310,15 +310,19 @@ const (
|
||||
testVolumeName = "pvc-0"
|
||||
)
|
||||
|
||||
// teardownClient models the CAPI/CAPK teardown race: the VM is gone while its
|
||||
// VMI still reports the hot-plugged volume. The VMI drops the volume once
|
||||
// RemoveVolumeFromVMI is called, or when the garbage collector deletes a
|
||||
// VMI whose controlling VM is gone.
|
||||
// teardownClient keeps hotplug state until removal or garbage collection,
|
||||
// including when the VMI outlives its VM during CAPI/CAPK teardown.
|
||||
type teardownClient struct {
|
||||
kubevirtclient.Client
|
||||
vmiOwners []metav1.OwnerReference
|
||||
vmiRemovals int
|
||||
lookups int
|
||||
vm *kubevirtv1.VirtualMachine
|
||||
vmiOwners []metav1.OwnerReference
|
||||
detached bool
|
||||
vmiGone bool
|
||||
vmiGoneAfterCheck bool
|
||||
vmiLookups int
|
||||
vmRemovals int
|
||||
vmiRemovals int
|
||||
lookups int
|
||||
}
|
||||
|
||||
func (c *teardownClient) vmiGarbageCollected() bool {
|
||||
@@ -328,22 +332,46 @@ func (c *teardownClient) vmiGarbageCollected() bool {
|
||||
|
||||
func (c *teardownClient) GetWorkloadManagingVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachine, error) {
|
||||
c.lookups++
|
||||
if c.vm != nil {
|
||||
return c.vm, nil
|
||||
}
|
||||
return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachines"}, testVMName)
|
||||
}
|
||||
|
||||
func (c *teardownClient) GetVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachineInstance, error) {
|
||||
c.lookups++
|
||||
c.vmiLookups++
|
||||
if c.vmiGone {
|
||||
return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachineinstances"}, testVMName)
|
||||
}
|
||||
vmi := &kubevirtv1.VirtualMachineInstance{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: testVMName, Namespace: testInfraNamespace, OwnerReferences: c.vmiOwners},
|
||||
}
|
||||
if c.vmiRemovals == 0 {
|
||||
vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}}
|
||||
if !c.detached && c.vmRemovals == 0 && c.vmiRemovals == 0 {
|
||||
vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, Phase: kubevirtv1.VolumePending, Reason: "FailedPvcNotFound", HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}}
|
||||
}
|
||||
return vmi, nil
|
||||
}
|
||||
|
||||
func (c *teardownClient) EnsureVolumeRemovedVM(context.Context, string, string, string) (bool, error) {
|
||||
return c.vm == nil || len(c.vm.Spec.Template.Spec.Volumes) == 0, nil
|
||||
}
|
||||
|
||||
func (c *teardownClient) EnsureVolumeRemovedVMI(context.Context, string, string, string) (bool, error) {
|
||||
return c.vmiRemovals > 0, nil
|
||||
removed := c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0
|
||||
if c.vmiGoneAfterCheck {
|
||||
c.vmiGone = true
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (c *teardownClient) RemoveVolumeFromVM(_ context.Context, namespace, name string, opts *kubevirtv1.RemoveVolumeOptions) error {
|
||||
if namespace != testInfraNamespace || name != testVMName || opts.Name != testVolumeName {
|
||||
return errors.New("unexpected hotplug removal target")
|
||||
}
|
||||
c.vmRemovals++
|
||||
c.vm.Spec.Template.Spec.Volumes = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *kubevirtv1.RemoveVolumeOptions) error {
|
||||
@@ -352,7 +380,7 @@ func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *k
|
||||
}
|
||||
|
||||
func (c *teardownClient) EnsureVolumeRemoved(context.Context, string, string, string, time.Duration) error {
|
||||
if c.vmiRemovals > 0 || c.vmiGarbageCollected() {
|
||||
if c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 || c.vmiGarbageCollected() {
|
||||
return nil
|
||||
}
|
||||
return errors.New("timed out waiting for the VMI to drop the volume")
|
||||
@@ -410,6 +438,58 @@ func TestControllerUnpublishVolumeHonoursVMIHotplugFallback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestControllerUnpublishVolumeMissingPVC(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
vmGone bool
|
||||
detached bool
|
||||
vmiGone bool
|
||||
vmiGoneAfterCheck bool
|
||||
wantVMILookups int
|
||||
wantVMCalls int
|
||||
wantVMICalls int
|
||||
}{
|
||||
{name: "pending hotplug volume", wantVMCalls: 1},
|
||||
{name: "already detached", detached: true},
|
||||
{name: "VM gone, VMI still has volume", vmGone: true, wantVMICalls: 1, wantVMILookups: 1},
|
||||
{name: "VM gone, no attached volume", vmGone: true, detached: true},
|
||||
{name: "VM and VMI gone", vmGone: true, vmiGone: true},
|
||||
{name: "VMI disappears after attachment check", vmGone: true, vmiGoneAfterCheck: true, wantVMILookups: 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
client := &teardownClient{detached: tt.detached, vmiGone: tt.vmiGone, vmiGoneAfterCheck: tt.vmiGoneAfterCheck}
|
||||
if !tt.vmGone {
|
||||
client.vm = &kubevirtv1.VirtualMachine{Spec: kubevirtv1.VirtualMachineSpec{Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{}}}
|
||||
if !tt.detached {
|
||||
client.vm.Spec.Template.Spec.Volumes = []kubevirtv1.Volume{{
|
||||
Name: testVolumeName,
|
||||
VolumeSource: kubevirtv1.VolumeSource{DataVolume: &kubevirtv1.DataVolumeSource{Name: testVolumeName, Hotpluggable: true}},
|
||||
}}
|
||||
}
|
||||
}
|
||||
w := &WrappedControllerService{
|
||||
ControllerService: service.NewControllerService(client, testInfraNamespace, nil, util.StorageClassEnforcement{AllowAll: true}, true),
|
||||
infraClient: fake.NewClientset(),
|
||||
infraNamespace: testInfraNamespace,
|
||||
}
|
||||
req := &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeName, NodeId: testInfraNamespace + "/" + testVMName}
|
||||
if _, err := w.ControllerUnpublishVolume(context.Background(), req); err != nil {
|
||||
t.Fatalf("ControllerUnpublishVolume() error = %v", err)
|
||||
}
|
||||
if client.vmRemovals != tt.wantVMCalls || client.vmiRemovals != tt.wantVMICalls {
|
||||
t.Errorf("VM/VMI removevolume calls = %d/%d, want %d/%d", client.vmRemovals, client.vmiRemovals, tt.wantVMCalls, tt.wantVMICalls)
|
||||
}
|
||||
if client.vmiLookups != tt.wantVMILookups {
|
||||
t.Errorf("GetVirtualMachine calls = %d, want %d", client.vmiLookups, tt.wantVMILookups)
|
||||
}
|
||||
if client.vm != nil && len(client.vm.Spec.Template.Spec.Volumes) != 0 {
|
||||
t.Error("hotplug volume remains in VM spec")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func nfsCNP(owners ...string) *unstructured.Unstructured {
|
||||
refs := make([]interface{}, 0, len(owners))
|
||||
for _, owner := range owners {
|
||||
|
||||
@@ -13,7 +13,7 @@ set:
|
||||
s3CredentialsSecret:
|
||||
name: postgres-backup
|
||||
tests:
|
||||
- it: sets resources for native backup without inheriting the tenant 128Mi limit
|
||||
- it: sets resources for native backup without inheriting the tenant 128Mi and 250m limits
|
||||
set:
|
||||
backup.enabled: true
|
||||
asserts:
|
||||
@@ -24,15 +24,11 @@ tests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
documentSelector:
|
||||
path: kind
|
||||
value: ObjectStore
|
||||
- notExists:
|
||||
path: spec.instanceSidecarConfiguration.resources.limits.cpu
|
||||
documentSelector:
|
||||
path: kind
|
||||
value: ObjectStore
|
||||
- equal:
|
||||
path: spec.configuration.s3Credentials.accessKeyId.name
|
||||
value: postgres-backup
|
||||
|
||||
@@ -146,6 +146,26 @@
|
||||
{{- fail (printf "source.image %q: golden image %q lives in StorageClass %q, and a clone into StorageClass %q of the same provisioner gets no host-assisted fallback from CDI. LINSTOR creates the clone next to the source data, and StorageClass %q does not allow remote volume access from every node, so the clone may never bind. Set storageClass to %q, or ask the platform administrator to set cloneStrategyOverride to copy in the kubevirt-cdi package." .Values.source.image.name $desiredSource.pvc.name $goldenSC $targetSC $targetSC $goldenSC) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- /* CDI v1.66.1 refuses to clone a Block source PVC into a smaller
|
||||
request before it creates the claim (validateClone in
|
||||
pkg/controller/datavolume/pvc-clone-controller.go), and the
|
||||
DataVolume then waits forever. It raises the request to the target
|
||||
StorageProfile's minimumSupportedPvcSize before comparing. A
|
||||
Filesystem source is not compared, as CDI sizes that clone from
|
||||
the image inside it or with a filesystem overhead the chart does
|
||||
not reproduce. */}}
|
||||
{{- $goldenSize := dig "spec" "resources" "requests" "storage" "" ($goldenPVC | default dict) | toString }}
|
||||
{{- if and .Values.storage $goldenSize (eq (dig "spec" "volumeMode" "" ($goldenPVC | default dict) | toString) "Block") }}
|
||||
{{- $goldenBytes := include "cozy-lib.resources.toFloat" $goldenSize | float64 }}
|
||||
{{- $diskBytes := include "cozy-lib.resources.toFloat" (.Values.storage | toString) | float64 }}
|
||||
{{- if and (lt $diskBytes $goldenBytes) $targetSC }}
|
||||
{{- $minimum := dig "metadata" "annotations" "cdi.kubevirt.io/minimumSupportedPvcSize" "" (lookup "cdi.kubevirt.io/v1beta1" "StorageProfile" "" $targetSC | default dict) | toString }}
|
||||
{{- $diskBytes = maxf $diskBytes (include "cozy-lib.resources.toFloat" $minimum | float64) }}
|
||||
{{- end }}
|
||||
{{- if lt $diskBytes $goldenBytes }}
|
||||
{{- fail (printf "storage %s is smaller than golden image %q (%s), so CDI refuses to clone it and the disk would stay pending forever. Set storage to %s or more." (.Values.storage | toString) $desiredSource.pvc.name $goldenSize $goldenSize) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- else if hasKey .Values.source "http" }}
|
||||
{{- $desiredSource = dict "http" (dict "url" (required "A valid .Values.source.http.url entry required!" .Values.source.http.url)) }}
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
suite: vm-disk refuses a disk smaller than the golden image it clones
|
||||
release:
|
||||
name: test-vm-disk
|
||||
namespace: tenant-test
|
||||
templates:
|
||||
- templates/dv.yaml
|
||||
tests:
|
||||
- it: refuses a new disk at the chart default size from a larger Block golden
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
kubernetesProvider:
|
||||
scheme: &scheme
|
||||
"cdi.kubevirt.io/v1beta1/DataVolume":
|
||||
gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes}
|
||||
namespaced: true
|
||||
"cdi.kubevirt.io/v1beta1/StorageProfile":
|
||||
gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: storageprofiles}
|
||||
namespaced: false
|
||||
"v1/Namespace":
|
||||
gvr: {group: "", version: v1, resource: namespaces}
|
||||
namespaced: false
|
||||
"v1/PersistentVolumeClaim":
|
||||
gvr: {group: "", version: v1, resource: persistentvolumeclaims}
|
||||
namespaced: true
|
||||
"storage.k8s.io/v1/StorageClass":
|
||||
gvr: {group: storage.k8s.io, version: v1, resource: storageclasses}
|
||||
namespaced: false
|
||||
objects:
|
||||
- &namespace
|
||||
kind: Namespace
|
||||
apiVersion: v1
|
||||
metadata: {name: tenant-test}
|
||||
- &goldenPVC
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
|
||||
spec:
|
||||
storageClassName: replicated
|
||||
volumeMode: Block
|
||||
resources: {requests: {storage: 20Gi}}
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\), so CDI refuses to clone it and the disk would stay pending forever\. Set storage to 20Gi or more\.'
|
||||
|
||||
- it: clones into a disk the size of the golden
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
storage: 20Gi
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.storage.resources.requests.storage
|
||||
value: 20Gi
|
||||
|
||||
- it: clones into a disk larger than the golden
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
storage: 25Gi
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.storage.resources.requests.storage
|
||||
value: 25Gi
|
||||
|
||||
- it: compares bytes, so 21G is smaller than a 20Gi golden
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
storage: 21G
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorPattern: 'storage 21G is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)'
|
||||
|
||||
- it: compares bytes, so 20480Mi is the size of a 20Gi golden
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
storage: 20480Mi
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.storage.resources.requests.storage
|
||||
value: 20480Mi
|
||||
|
||||
- it: clones when the target StorageProfile's minimum PVC size covers the golden
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
- kind: StorageProfile
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
metadata:
|
||||
name: replicated
|
||||
annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi}
|
||||
asserts:
|
||||
- notFailedTemplate: {}
|
||||
|
||||
- it: refuses when the target StorageProfile's minimum PVC size is still below the golden
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
- kind: StorageProfile
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
metadata:
|
||||
name: replicated
|
||||
annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 10Gi}
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)'
|
||||
|
||||
- it: reads the minimum PVC size of the default class for an empty storageClass
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
storageClass: ""
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
- kind: StorageClass
|
||||
apiVersion: storage.k8s.io/v1
|
||||
metadata:
|
||||
name: replicated
|
||||
annotations: {storageclass.kubernetes.io/is-default-class: "true"}
|
||||
provisioner: linstor.csi.linbit.com
|
||||
- kind: StorageProfile
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
metadata:
|
||||
name: replicated
|
||||
annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi}
|
||||
asserts:
|
||||
- notFailedTemplate: {}
|
||||
|
||||
- it: does not compare sizes for a Filesystem golden
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
|
||||
spec:
|
||||
storageClassName: replicated
|
||||
volumeMode: Filesystem
|
||||
resources: {requests: {storage: 20Gi}}
|
||||
asserts:
|
||||
- notFailedTemplate: {}
|
||||
|
||||
- it: renders without a cluster to look the golden up in
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.storage.resources.requests.storage
|
||||
value: 5Gi
|
||||
|
||||
- it: clones a golden whose PVC does not exist yet
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- kind: DataVolume
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
|
||||
status: {phase: ImportScheduled}
|
||||
asserts:
|
||||
- notFailedTemplate: {}
|
||||
|
||||
- it: leaves a disk without a size to CDI, which takes the golden's
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
storage: null
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
asserts:
|
||||
- notFailedTemplate: {}
|
||||
|
||||
- it: leaves an existing disk alone
|
||||
set:
|
||||
source: {image: {name: ubuntu}}
|
||||
storage: 25Gi
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
- kind: DataVolume
|
||||
apiVersion: cdi.kubevirt.io/v1beta1
|
||||
metadata:
|
||||
name: test-vm-disk
|
||||
namespace: tenant-test
|
||||
annotations:
|
||||
vm-disk.cozystack.io/source: '{"image":{"name":"ubuntu"}}'
|
||||
vm-disk.cozystack.io/storage-class: replicated
|
||||
spec:
|
||||
source: {pvc: {name: vm-default-images-ubuntu, namespace: cozy-public}}
|
||||
storage: {resources: {requests: {storage: 10Gi}}, storageClassName: replicated}
|
||||
asserts:
|
||||
- notFailedTemplate: {}
|
||||
|
||||
- it: does not compare sizes for an http source
|
||||
set:
|
||||
source: {http: {url: "https://example.com/disk.img"}}
|
||||
storage: 1Gi
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
asserts:
|
||||
- notFailedTemplate: {}
|
||||
|
||||
- it: does not compare sizes for an upload source
|
||||
set:
|
||||
source: {upload: {}}
|
||||
storage: 1Gi
|
||||
kubernetesProvider:
|
||||
scheme: *scheme
|
||||
objects:
|
||||
- *namespace
|
||||
- *goldenPVC
|
||||
asserts:
|
||||
- notFailedTemplate: {}
|
||||
@@ -48,6 +48,7 @@ virtctl ssh <user>@<vm>
|
||||
| `disks` | List of disks to attach. | `[]object` | `[]` |
|
||||
| `disks[i].name` | Disk name. | `string` | `""` |
|
||||
| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` |
|
||||
| `disks[i].bootOrder` | Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. | `*int` | `null` |
|
||||
| `networks` | Networks to attach the VM to. | `[]object` | `[]` |
|
||||
| `networks[i].name` | Network attachment name. | `string` | `""` |
|
||||
| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` |
|
||||
|
||||
@@ -30,6 +30,13 @@ spec:
|
||||
{{- if .Values.external }}
|
||||
{{- if and (eq .Values.externalMethod "WholeIP") (not .Values.externalPorts) }}
|
||||
- port: 65535
|
||||
{{- else if not .Values.externalPorts }}
|
||||
{{- /* A Service needs at least one port. cozy-proxy only admits TCP and
|
||||
UDP ports into its PortList filter, so an SCTP placeholder keeps the
|
||||
external IP while every inbound TCP and UDP packet is dropped. */}}
|
||||
- name: no-ports
|
||||
port: 65535
|
||||
protocol: SCTP
|
||||
{{- else }}
|
||||
{{- range .Values.externalPorts }}
|
||||
- name: port-{{ . }}
|
||||
|
||||
@@ -92,7 +92,16 @@ spec:
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
disks:
|
||||
{{- /* KubeVirt rejects two devices sharing a bootOrder, so a clash is
|
||||
reported here with the disk names rather than surfacing as a
|
||||
webhook denial on the VirtualMachine. */}}
|
||||
{{- $bootOrderOwners := dict }}
|
||||
{{- range $i, $disk := .Values.disks }}
|
||||
{{- $bootOrder := int ($disk.bootOrder | default (add $i 1)) }}
|
||||
{{- with get $bootOrderOwners (toString $bootOrder) }}
|
||||
{{- fail (printf "disks %q and %q both have bootOrder %d; boot orders must be unique" . $disk.name $bootOrder) }}
|
||||
{{- end }}
|
||||
{{- $_ := set $bootOrderOwners (toString $bootOrder) $disk.name }}
|
||||
- name: disk-{{ $disk.name }}
|
||||
{{- $dv := lookup "cdi.kubevirt.io/v1beta1" "DataVolume" $.Release.Namespace (printf "vm-disk-%s" $disk.name) }}
|
||||
{{- if $dv }}
|
||||
@@ -104,7 +113,7 @@ spec:
|
||||
{{- else }}
|
||||
{{ $deviceKey }}: {}
|
||||
{{- end }}
|
||||
bootOrder: {{ add $i 1 }}
|
||||
bootOrder: {{ $bootOrder }}
|
||||
{{- else }}
|
||||
{{- fail (printf "Specified disk not exists in cluster: %s" .name) }}
|
||||
{{- end }}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
suite: vm-instance disk boot order
|
||||
|
||||
# The first entry of `disks` is the one callers treat as the system disk, so
|
||||
# booting from installer or rescue media must not require moving that media to
|
||||
# the front of the list. These cases pin that an explicit bootOrder decides
|
||||
# which device the VM boots from while list order stays as given, and that
|
||||
# leaving it unset keeps the positional order existing VMs were rendered with.
|
||||
release:
|
||||
name: test-vm
|
||||
namespace: tenant-test
|
||||
templates:
|
||||
- templates/vm.yaml
|
||||
set:
|
||||
fullnameOverride: test-vm
|
||||
instanceType: ""
|
||||
instanceProfile: ""
|
||||
resources:
|
||||
cpu: 1
|
||||
sockets: 1
|
||||
memory: 1Gi
|
||||
_cluster:
|
||||
scheduling:
|
||||
dedicatedNodesForWindowsVMs: "false"
|
||||
kubernetesProvider:
|
||||
scheme:
|
||||
"cdi.kubevirt.io/v1beta1/DataVolume":
|
||||
gvr:
|
||||
group: cdi.kubevirt.io
|
||||
version: v1beta1
|
||||
resource: datavolumes
|
||||
namespaced: true
|
||||
objects:
|
||||
- apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: vm-disk-system
|
||||
namespace: tenant-test
|
||||
annotations: {}
|
||||
- apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: vm-disk-iso
|
||||
namespace: tenant-test
|
||||
annotations:
|
||||
vm-disk.cozystack.io/optical: "true"
|
||||
- apiVersion: cdi.kubevirt.io/v1beta1
|
||||
kind: DataVolume
|
||||
metadata:
|
||||
name: vm-disk-data
|
||||
namespace: tenant-test
|
||||
annotations: {}
|
||||
tests:
|
||||
- it: keeps the positional boot order when no disk sets bootOrder
|
||||
set:
|
||||
disks:
|
||||
- name: system
|
||||
- name: iso
|
||||
- name: data
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.domain.devices.disks
|
||||
value:
|
||||
- name: disk-system
|
||||
disk: {}
|
||||
bootOrder: 1
|
||||
- name: disk-iso
|
||||
cdrom: {}
|
||||
bootOrder: 2
|
||||
- name: disk-data
|
||||
disk: {}
|
||||
bootOrder: 3
|
||||
|
||||
- it: boots from a CD-ROM in second position without reordering the disks
|
||||
set:
|
||||
disks:
|
||||
- name: system
|
||||
bootOrder: 2
|
||||
- name: iso
|
||||
bus: sata
|
||||
bootOrder: 1
|
||||
- name: data
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.template.spec.domain.devices.disks
|
||||
value:
|
||||
- name: disk-system
|
||||
disk: {}
|
||||
bootOrder: 2
|
||||
- name: disk-iso
|
||||
cdrom:
|
||||
bus: sata
|
||||
bootOrder: 1
|
||||
- name: disk-data
|
||||
disk: {}
|
||||
bootOrder: 3
|
||||
- equal:
|
||||
path: spec.template.spec.volumes[0].dataVolume.name
|
||||
value: vm-disk-system
|
||||
|
||||
- it: refuses an explicit bootOrder that collides with another disk's positional one
|
||||
set:
|
||||
disks:
|
||||
- name: system
|
||||
- name: iso
|
||||
bootOrder: 1
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: 'disks "system" and "iso" both have bootOrder 1; boot orders must be unique'
|
||||
|
||||
- it: refuses two disks given the same explicit bootOrder
|
||||
set:
|
||||
disks:
|
||||
- name: system
|
||||
bootOrder: 3
|
||||
- name: iso
|
||||
bootOrder: 3
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: 'disks "system" and "iso" both have bootOrder 3; boot orders must be unique'
|
||||
|
||||
- it: rejects a bootOrder below 1 through the values schema
|
||||
set:
|
||||
disks:
|
||||
- name: system
|
||||
bootOrder: 0
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorPattern: "minimum"
|
||||
@@ -0,0 +1,70 @@
|
||||
suite: vm-instance external Service ports
|
||||
|
||||
# The API server rejects a Service without ports, so an external PortList VM
|
||||
# whose port list was emptied could not be upgraded at all. The placeholder
|
||||
# must be SCTP: cozy-proxy only admits TCP and UDP ports into its filter, so
|
||||
# any TCP or UDP placeholder would open that port on the VM.
|
||||
release:
|
||||
name: test-vm
|
||||
namespace: tenant-test
|
||||
templates:
|
||||
- templates/service.yaml
|
||||
set:
|
||||
fullnameOverride: test-vm
|
||||
external: true
|
||||
|
||||
tests:
|
||||
- it: renders an SCTP placeholder for PortList with no ports
|
||||
set:
|
||||
externalMethod: PortList
|
||||
externalPorts: []
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.type
|
||||
value: LoadBalancer
|
||||
- equal:
|
||||
path: metadata.annotations["networking.cozystack.io/wholeIP"]
|
||||
value: "false"
|
||||
- lengthEqual:
|
||||
path: spec.ports
|
||||
count: 1
|
||||
- equal:
|
||||
path: spec.ports[0]
|
||||
value:
|
||||
name: no-ports
|
||||
port: 65535
|
||||
protocol: SCTP
|
||||
|
||||
- it: renders the listed ports for PortList
|
||||
set:
|
||||
externalMethod: PortList
|
||||
externalPorts: [22, 443]
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.ports
|
||||
count: 2
|
||||
- equal:
|
||||
path: spec.ports[0]
|
||||
value:
|
||||
name: port-22
|
||||
port: 22
|
||||
targetPort: 22
|
||||
- notContains:
|
||||
path: spec.ports
|
||||
content:
|
||||
name: no-ports
|
||||
port: 65535
|
||||
protocol: SCTP
|
||||
|
||||
- it: keeps the WholeIP placeholder when no ports are listed
|
||||
set:
|
||||
externalMethod: WholeIP
|
||||
externalPorts: []
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.annotations["networking.cozystack.io/wholeIP"]
|
||||
value: "true"
|
||||
- equal:
|
||||
path: spec.ports
|
||||
value:
|
||||
- port: 65535
|
||||
@@ -69,6 +69,11 @@
|
||||
"name"
|
||||
],
|
||||
"properties": {
|
||||
"bootOrder": {
|
||||
"description": "Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both.",
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
},
|
||||
"bus": {
|
||||
"description": "Disk bus type (e.g. \"sata\").",
|
||||
"type": "string"
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
## @field {string} name - Disk name.
|
||||
## @x-cozystack-options {source: vmdisk}
|
||||
## @field {string} [bus] - Disk bus type (e.g. "sata").
|
||||
## @field {*int} [bootOrder] - Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both.
|
||||
## @minimum 1
|
||||
|
||||
## @typedef {struct} Network - Network to attach the VM to.
|
||||
## @field {string} [name] - Network attachment name.
|
||||
@@ -67,6 +69,12 @@ disks: []
|
||||
## - name: example-system
|
||||
## - name: example-data
|
||||
## bus: sata
|
||||
## Boot from an attached ISO while keeping the system disk first in the list:
|
||||
## disks:
|
||||
## - name: example-system
|
||||
## bootOrder: 2
|
||||
## - name: example-iso
|
||||
## bootOrder: 1
|
||||
|
||||
## @param {[]Network} networks - Networks to attach the VM to.
|
||||
networks: []
|
||||
|
||||
@@ -34,14 +34,14 @@ coordinates).
|
||||
|
||||
### Common parameters
|
||||
|
||||
| Name | Description | Type | Value |
|
||||
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- |
|
||||
| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` |
|
||||
| `size` | Persistent Volume size. | `quantity` | `4Gi` |
|
||||
| `storageClass` | StorageClass used to store the data. | `string` | `""` |
|
||||
| `replicas` | Number of etcd replicas. | `int` | `3` |
|
||||
| `resources` | Resource configuration for etcd. | `object` | `{}` |
|
||||
| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` |
|
||||
| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` |
|
||||
| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively. | `object` | `{}` |
|
||||
| Name | Description | Type | Value |
|
||||
| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- |
|
||||
| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` |
|
||||
| `size` | Persistent Volume size. | `quantity` | `4Gi` |
|
||||
| `storageClass` | StorageClass used to store the data. | `string` | `""` |
|
||||
| `replicas` | Number of etcd replicas. | `int` | `3` |
|
||||
| `resources` | Resource configuration for etcd. | `object` | `{}` |
|
||||
| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` |
|
||||
| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` |
|
||||
| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones. | `object` | `{}` |
|
||||
|
||||
@@ -69,7 +69,8 @@ spec:
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
nodeTaintsPolicy: Ignore
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: etcd
|
||||
@@ -78,11 +79,15 @@ spec:
|
||||
affinity:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- else }}
|
||||
# Soft anti-affinity: spread etcd members across nodes on a best-effort
|
||||
# basis so a single node failure is unlikely to take out more than one
|
||||
# member. Preferred (not required) because a hard rule would leave a member
|
||||
# Pending when replicas exceed schedulable nodes, or when node-local storage
|
||||
# pins a member to a node. Override via .Values.affinity for a hard rule.
|
||||
# Soft anti-affinity, a preference on top of the spread constraint above,
|
||||
# which by default is the hard rule that keeps two members off one node
|
||||
# while another node has none. Preferred, not required, because a required
|
||||
# rule also forbids two members on one node when every node already has
|
||||
# one, so an etcd with more members than schedulable nodes could not start
|
||||
# all of them. With node-local storage a recreated member Pod can only
|
||||
# return to its own node, and the default spread keeps it Pending there
|
||||
# while that node holds another member and some other node holds none.
|
||||
# Override via .Values.affinity for a hard rule.
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
|
||||
@@ -29,11 +29,11 @@ spec:
|
||||
drop: ["ALL"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
args:
|
||||
# The v1alpha2 operator names the headless Service after the
|
||||
# cluster ({{ $.Release.Name }}) and does not use StatefulSet
|
||||
|
||||
@@ -41,3 +41,16 @@ tests:
|
||||
documentSelector:
|
||||
path: kind
|
||||
value: EtcdCluster
|
||||
|
||||
- it: asks the tenant quota only for what the defrag client uses
|
||||
template: templates/etcd-defrag.yaml
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.jobTemplate.spec.template.spec.containers[0].resources
|
||||
value:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
@@ -8,7 +8,7 @@ templates:
|
||||
- templates/etcd-cluster.yaml
|
||||
|
||||
tests:
|
||||
- it: uses a soft per-host spread by default
|
||||
- it: uses a strict per-host spread by default
|
||||
set:
|
||||
_cluster: {}
|
||||
documentSelector:
|
||||
@@ -16,17 +16,41 @@ tests:
|
||||
value: EtcdCluster
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.topologySpreadConstraints[0].maxSkew
|
||||
value: 1
|
||||
path: spec.topologySpreadConstraints[0]
|
||||
value:
|
||||
maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: DoNotSchedule
|
||||
nodeTaintsPolicy: Ignore
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: etcd
|
||||
- lengthEqual:
|
||||
path: spec.topologySpreadConstraints
|
||||
count: 1
|
||||
|
||||
- it: renders a soft cluster-wide spread policy as written
|
||||
set:
|
||||
_cluster:
|
||||
scheduling:
|
||||
globalAppTopologySpreadConstraints: |
|
||||
topologySpreadConstraints:
|
||||
- maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
documentSelector:
|
||||
path: kind
|
||||
value: EtcdCluster
|
||||
asserts:
|
||||
- equal:
|
||||
path: spec.topologySpreadConstraints[0].topologyKey
|
||||
value: kubernetes.io/hostname
|
||||
- equal:
|
||||
path: spec.topologySpreadConstraints[0].whenUnsatisfiable
|
||||
value: ScheduleAnyway
|
||||
- equal:
|
||||
path: spec.topologySpreadConstraints[0].labelSelector.matchLabels["app.kubernetes.io/instance"]
|
||||
value: etcd
|
||||
path: spec.topologySpreadConstraints[0]
|
||||
value:
|
||||
maxSkew: 1
|
||||
topologyKey: kubernetes.io/hostname
|
||||
whenUnsatisfiable: ScheduleAnyway
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: etcd
|
||||
- lengthEqual:
|
||||
path: spec.topologySpreadConstraints
|
||||
count: 1
|
||||
|
||||
@@ -67,7 +67,7 @@
|
||||
}
|
||||
},
|
||||
"affinity": {
|
||||
"description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.",
|
||||
"description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.",
|
||||
"type": "object",
|
||||
"default": {},
|
||||
"x-kubernetes-preserve-unknown-fields": true
|
||||
|
||||
@@ -23,5 +23,5 @@ resources:
|
||||
cpu: 1000m
|
||||
memory: 512Mi
|
||||
|
||||
## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.
|
||||
## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.
|
||||
affinity: {}
|
||||
@@ -13,10 +13,12 @@ LimitRange, which is cozy-keycloak and any tenant that leaves resourceQuotas
|
||||
empty, gave the sidecar no requests and no limit at all, so nothing reserved
|
||||
memory for it and nothing bounded it. The same request and limit settle both:
|
||||
256Mi holds the measured working set and 1Gi is four times the measurement.
|
||||
No CPU limit is set, matching the entityOperator precedent in
|
||||
packages/apps/kafka: a throttled sidecar stalls WAL archiving instead of
|
||||
failing it. Inside a tenant the limit counts against the ResourceQuota
|
||||
limits.memory budget, 1Gi per instance pod.
|
||||
The CPU limit is explicit for the same reason: left unset, that LimitRange
|
||||
gives the sidecar its 250m default, so gzip compression of full backups and
|
||||
WAL segments runs on a quarter of a core (a 1.6 GB database took 30 minutes
|
||||
to back up). One core keeps the sidecar from starving the instance while
|
||||
removing that ceiling. Inside a tenant both limits count against the
|
||||
ResourceQuota budget, one core and 1Gi per instance pod.
|
||||
|
||||
Checksum: since botocore ~1.36 (early 2025) the default
|
||||
RequestChecksumCalculation is when_supported, which attaches a flexible checksum
|
||||
@@ -39,6 +41,7 @@ instanceSidecarConfiguration:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
env:
|
||||
- name: AWS_REQUEST_CHECKSUM_CALCULATION
|
||||
|
||||
@@ -32,6 +32,8 @@ Implications:
|
||||
- **RJSF** (`@rjsf/core` + `validator-ajv8`) for application config forms.
|
||||
- **Vitest + jsdom + @testing-library/react** for tests.
|
||||
- **Monaco** for YAML view, **noVNC** for the VM console tab.
|
||||
- **uPlot** for the Metrics tab charts, lazy-loaded with that tab.
|
||||
- Metrics and logs come from the observability proxy, not kube-apiserver: `lib/observability/proxy.ts` reads its native Prometheus JSON and VictoriaLogs NDJSON through `K8sClient.raw`, under the paths nginx routes to it ahead of `/apis`.
|
||||
|
||||
## Dev loop
|
||||
|
||||
@@ -79,6 +81,7 @@ the apiserver checks that user's RBAC.
|
||||
to a chunked-encoding WATCH against the same `resourceVersion`. Don't
|
||||
poll. Don't add `refetchInterval`. If you need a one-shot, pass
|
||||
`{ watch: false }`.
|
||||
The one exception is the metrics panels in `components/observability`: the observability proxy serves the Prometheus query API, which has no watch, so they refresh on a fixed interval instead. Log lines stream through the tail endpoint and are not polled.
|
||||
4. **Tenant scoping.** Most resources live in `tenant-<name>` namespaces.
|
||||
Pull the active tenant from `useTenantContext()` — never read the
|
||||
namespace from a URL param or guess it.
|
||||
|
||||
@@ -31,7 +31,8 @@
|
||||
"react": "^19.2.4",
|
||||
"react-dom": "^19.2.4",
|
||||
"react-router": "^7.6.1",
|
||||
"simple-icons": "^16.22.0"
|
||||
"simple-icons": "^16.22.0",
|
||||
"uplot": "^1.6.32"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/vite": "^4.2.2",
|
||||
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
import { describe, it, expect, vi, afterEach } from "vitest"
|
||||
import { K8sClient } from "@cozystack/k8s-client"
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
describe("K8sClient request headers", () => {
|
||||
it("lets a method's own content type replace the JSON default", async () => {
|
||||
const fetchMock = vi.fn(async () => new Response("{}"))
|
||||
vi.stubGlobal("fetch", fetchMock)
|
||||
const client = new K8sClient({ getToken: async () => "t0k" })
|
||||
|
||||
await client.patch("apps", "v1", "deployments", "web", { spec: {} }, "default", "merge")
|
||||
|
||||
const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
|
||||
const headers = new Headers(init.headers)
|
||||
expect(headers.get("Content-Type")).toBe("application/merge-patch+json")
|
||||
expect(headers.get("Authorization")).toBe("Bearer t0k")
|
||||
})
|
||||
|
||||
it("sends JSON by default", async () => {
|
||||
const fetchMock = vi.fn(async () => new Response("{}"))
|
||||
vi.stubGlobal("fetch", fetchMock)
|
||||
|
||||
await new K8sClient().getApiGroups()
|
||||
|
||||
const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
|
||||
expect(new Headers(init.headers).get("Content-Type")).toBe("application/json")
|
||||
})
|
||||
})
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
import { describe, it, expect, vi, afterEach } from "vitest"
|
||||
import { K8sClient } from "@cozystack/k8s-client"
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe("K8sClient.raw", () => {
|
||||
it("returns the response untouched, including a non-2xx one", async () => {
|
||||
const res = new Response("not found", { status: 404 })
|
||||
vi.stubGlobal("fetch", vi.fn(async () => res))
|
||||
const client = new K8sClient()
|
||||
|
||||
await expect(client.raw("/x")).resolves.toBe(res)
|
||||
})
|
||||
|
||||
it("prefixes the base URL and attaches the bearer token", async () => {
|
||||
const fetchMock = vi.fn(async () => new Response("{}"))
|
||||
vi.stubGlobal("fetch", fetchMock)
|
||||
const client = new K8sClient({ baseUrl: "/base", getToken: async () => "t0k" })
|
||||
|
||||
await client.raw("/apis/x", { headers: { Accept: "application/json" } })
|
||||
|
||||
const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
|
||||
expect(url).toBe("/base/apis/x")
|
||||
const headers = new Headers(init.headers)
|
||||
expect(headers.get("Accept")).toBe("application/json")
|
||||
expect(headers.get("Authorization")).toBe("Bearer t0k")
|
||||
})
|
||||
|
||||
it.each([
|
||||
["a Headers instance", new Headers({ Accept: "application/x-ndjson" })],
|
||||
["a tuple list", [["Accept", "application/x-ndjson"]] as [string, string][]],
|
||||
])("keeps caller headers passed as %s", async (_, given) => {
|
||||
const fetchMock = vi.fn(async () => new Response("{}"))
|
||||
vi.stubGlobal("fetch", fetchMock)
|
||||
const client = new K8sClient({ getToken: async () => "t0k" })
|
||||
|
||||
await client.raw("/x", { headers: given })
|
||||
|
||||
const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
|
||||
const headers = new Headers(init.headers)
|
||||
expect(headers.get("Accept")).toBe("application/x-ndjson")
|
||||
expect(headers.get("Authorization")).toBe("Bearer t0k")
|
||||
})
|
||||
|
||||
it("routes a 401 through the client's unauthorized handler", async () => {
|
||||
const onUnauthorized = vi.fn()
|
||||
vi.stubGlobal("fetch", vi.fn(async () => new Response("", { status: 401 })))
|
||||
const client = new K8sClient({ onUnauthorized })
|
||||
|
||||
const res = await client.raw("/x")
|
||||
|
||||
expect(res.status).toBe(401)
|
||||
expect(onUnauthorized).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("does not swallow a network failure", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn(async () => { throw new TypeError("offline") }))
|
||||
const client = new K8sClient()
|
||||
|
||||
await expect(client.raw("/x")).rejects.toThrow("offline")
|
||||
})
|
||||
})
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
import { describe, it, expect } from "vitest"
|
||||
import { render, screen } from "@testing-library/react"
|
||||
import { EmptyState } from "./EmptyState.tsx"
|
||||
|
||||
describe("EmptyState", () => {
|
||||
it("announces its message as a status", () => {
|
||||
render(<EmptyState title="No data for this range." hint="Pick a longer range." />)
|
||||
|
||||
expect(screen.getByRole("status")).toHaveTextContent("No data for this range.Pick a longer range.")
|
||||
})
|
||||
|
||||
it("marks itself busy while something is loading", () => {
|
||||
render(<EmptyState busy title="Loading…" />)
|
||||
|
||||
expect(screen.getByRole("status")).toHaveAttribute("aria-busy", "true")
|
||||
})
|
||||
})
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
import type { ReactNode } from "react"
|
||||
import { cn, Spinner } from "@cozystack/ui"
|
||||
|
||||
interface EmptyStateProps {
|
||||
title: ReactNode
|
||||
hint?: ReactNode
|
||||
action?: ReactNode
|
||||
busy?: boolean
|
||||
tone?: "light" | "dark"
|
||||
className?: string
|
||||
}
|
||||
|
||||
export function EmptyState({ title, hint, action, busy, tone = "light", className }: EmptyStateProps) {
|
||||
const dark = tone === "dark"
|
||||
return (
|
||||
<div
|
||||
role="status"
|
||||
aria-busy={busy || undefined}
|
||||
className={cn("flex h-full flex-col items-center justify-center gap-1.5 px-4 py-8 text-center", className)}
|
||||
>
|
||||
{busy && <Spinner className={cn("mb-1 size-5", dark && "text-slate-500")} />}
|
||||
<p className={cn("text-sm", dark ? "text-slate-300" : "text-slate-600")}>{title}</p>
|
||||
{hint && <p className={cn("text-xs", dark ? "text-slate-500" : "text-slate-400")}>{hint}</p>}
|
||||
{action && <div className="mt-2">{action}</div>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
import { describe, it, expect } from "vitest"
|
||||
import { fireEvent, render, screen } from "@testing-library/react"
|
||||
import { LogLines } from "./LogLines.tsx"
|
||||
|
||||
const entry = (n: number) => ({ _time: `2026-11-04T10:00:0${n}Z`, _msg: `line ${n}`, _stream_id: "s" })
|
||||
|
||||
function sized(list: HTMLElement, scrollHeight: number) {
|
||||
Object.defineProperty(list, "scrollHeight", { configurable: true, value: scrollHeight })
|
||||
Object.defineProperty(list, "clientHeight", { configurable: true, value: 100 })
|
||||
}
|
||||
|
||||
describe("LogLines", () => {
|
||||
it("keeps the console frame and shows the placeholder inside it when there are no lines", () => {
|
||||
render(<LogLines entries={[]} placeholder="Waiting for new log lines…" />)
|
||||
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
expect(frame).toHaveTextContent("Waiting for new log lines…")
|
||||
expect(screen.queryByRole("list")).toBeNull()
|
||||
})
|
||||
|
||||
it("puts the lines in the same frame once they arrive", () => {
|
||||
const { rerender } = render(<LogLines entries={[]} placeholder="Waiting for new log lines…" />)
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
|
||||
rerender(<LogLines entries={[entry(1)]} placeholder="Waiting for new log lines…" />)
|
||||
|
||||
expect(screen.getByRole("region", { name: "Log lines" })).toBe(frame)
|
||||
expect(frame).toContainElement(screen.getByRole("list"))
|
||||
expect(frame).not.toHaveTextContent("Waiting")
|
||||
})
|
||||
|
||||
it("shows a line without _time with no timestamp and no empty source", () => {
|
||||
render(<LogLines placeholder="" entries={[{ _msg: "visible message" }]} />)
|
||||
|
||||
expect(screen.getByRole("listitem")).toHaveTextContent(/^visible message$/)
|
||||
})
|
||||
|
||||
it("shows the remaining fields of a row without _msg", () => {
|
||||
render(<LogLines placeholder="" entries={[{ kubernetes_pod_name: "db-0", rows: "42" }]} />)
|
||||
|
||||
expect(screen.getByRole("listitem")).toHaveTextContent(/^db-0\/ rows=42$/)
|
||||
})
|
||||
|
||||
it("keeps following the newest line while the reader is at the bottom", () => {
|
||||
const { rerender } = render(<LogLines placeholder="" entries={[entry(1)]} />)
|
||||
const list = screen.getByRole("list")
|
||||
sized(list, 500)
|
||||
|
||||
rerender(<LogLines placeholder="" entries={[entry(1), entry(2)]} />)
|
||||
|
||||
expect(list.scrollTop).toBe(500)
|
||||
})
|
||||
|
||||
it("leaves the position alone once the reader scrolls up", () => {
|
||||
const { rerender } = render(<LogLines placeholder="" entries={[entry(1)]} />)
|
||||
const list = screen.getByRole("list")
|
||||
sized(list, 500)
|
||||
list.scrollTop = 120
|
||||
fireEvent.scroll(list)
|
||||
|
||||
rerender(<LogLines placeholder="" entries={[entry(1), entry(2)]} />)
|
||||
|
||||
expect(list.scrollTop).toBe(120)
|
||||
})
|
||||
|
||||
it("follows again once the list is cleared for a new run", () => {
|
||||
const { rerender } = render(<LogLines placeholder="" entries={[entry(1)]} />)
|
||||
const first = screen.getByRole("list")
|
||||
sized(first, 500)
|
||||
first.scrollTop = 120
|
||||
fireEvent.scroll(first)
|
||||
|
||||
rerender(<LogLines placeholder="" entries={[]} />)
|
||||
rerender(<LogLines placeholder="" entries={[entry(2)]} />)
|
||||
const second = screen.getByRole("list")
|
||||
sized(second, 300)
|
||||
rerender(<LogLines placeholder="" entries={[entry(2), entry(3)]} />)
|
||||
|
||||
expect(second.scrollTop).toBe(300)
|
||||
})
|
||||
})
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
import { useEffect, useRef, type ReactNode } from "react"
|
||||
import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts"
|
||||
import type { LogEntry } from "../../lib/observability/proxy.ts"
|
||||
|
||||
const BOTTOM_SLACK_PX = 24
|
||||
const SHOWN_APART = new Set(["_time", "_msg", "_stream_id", POD_FIELD, CONTAINER_FIELD])
|
||||
|
||||
function source(e: LogEntry): string | null {
|
||||
const pod = e[POD_FIELD]
|
||||
const container = e[CONTAINER_FIELD]
|
||||
return pod || container ? `${pod ?? ""}/${container ?? ""}` : null
|
||||
}
|
||||
|
||||
// A `fields` or `stats` pipe can leave a row without _msg; its other fields are then the content.
|
||||
function message(e: LogEntry): string {
|
||||
if (e._msg !== undefined) return e._msg
|
||||
return Object.entries(e)
|
||||
.filter(([k, v]) => v !== undefined && !SHOWN_APART.has(k))
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
export function LogLines({ entries, placeholder }: { entries: LogEntry[]; placeholder: ReactNode }) {
|
||||
const ref = useRef<HTMLOListElement>(null)
|
||||
const pinned = useRef(true)
|
||||
|
||||
useEffect(() => {
|
||||
if (entries.length === 0) pinned.current = true
|
||||
else if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight
|
||||
}, [entries])
|
||||
|
||||
return (
|
||||
<section
|
||||
aria-label="Log lines"
|
||||
className="h-[60vh] min-h-72 overflow-hidden rounded-lg bg-slate-950 font-mono text-xs leading-5 text-slate-100 shadow-xs"
|
||||
>
|
||||
{entries.length === 0 ? (
|
||||
placeholder
|
||||
) : (
|
||||
<ol
|
||||
ref={ref}
|
||||
onScroll={(e) => {
|
||||
const el = e.currentTarget
|
||||
pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX
|
||||
}}
|
||||
className="h-full overflow-auto p-3"
|
||||
>
|
||||
{entries.map((e, i) => {
|
||||
const from = source(e)
|
||||
return (
|
||||
<li key={`${e._stream_id}-${e._time}-${i}`} className="whitespace-pre-wrap break-all">
|
||||
{e._time !== undefined && <><span className="text-slate-400">{new Date(e._time).toLocaleTimeString()}</span>{" "}</>}
|
||||
{from !== null && <><span className="text-sky-300">{from}</span>{" "}</>}
|
||||
{message(e)}
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ol>
|
||||
)}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
import { describe, it, expect, vi } from "vitest"
|
||||
import { render, screen, waitFor, within } from "@testing-library/react"
|
||||
import userEvent from "@testing-library/user-event"
|
||||
import { K8sClient, K8sProvider } from "@cozystack/k8s-client"
|
||||
import { renderWithK8sProvider } from "../../test-utils/render.tsx"
|
||||
import { LogsView } from "./LogsView.tsx"
|
||||
|
||||
const app = { namespace: "tenant-a", plural: "postgreses", name: "db" }
|
||||
const ROOT = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/"
|
||||
|
||||
const line = (time: string, msg: string, pod = "db-0") =>
|
||||
JSON.stringify({ _time: time, _msg: msg, _stream_id: "s", kubernetes_pod_name: pod, kubernetes_container_name: "postgres" })
|
||||
|
||||
function clientWith(handler: (endpoint: string, params: URLSearchParams) => Response | Promise<Response>) {
|
||||
const client = new K8sClient()
|
||||
const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => {
|
||||
const url = new URL(path, "http://x")
|
||||
return handler(url.pathname, url.searchParams)
|
||||
})
|
||||
return { client, raw }
|
||||
}
|
||||
|
||||
const values = (...v: string[]) => Response.json({ values: v.map((value) => ({ value, hits: 1 })) })
|
||||
|
||||
function defaultHandler(endpoint: string, params?: URLSearchParams): Response | Promise<Response> {
|
||||
if (endpoint.endsWith("stream_field_values")) {
|
||||
return params?.get("field") === "kubernetes_pod_name" ? values("db-0", "db-1") : values("postgres")
|
||||
}
|
||||
if (endpoint.endsWith("query")) return new Response(`${line("2026-11-04T10:00:01Z", "ready to accept connections")}\n`)
|
||||
return new Promise(() => {})
|
||||
}
|
||||
|
||||
// The provider's own QueryClient keeps the previous result while a key changes; it is
|
||||
// shared across tests, so each test below uses namespaces no other test reads.
|
||||
function renderWithProductionDefaults(client: K8sClient, namespace: string) {
|
||||
const view = (ns: string) => (
|
||||
<K8sProvider client={client}>
|
||||
<LogsView scope={{ namespace: ns }} />
|
||||
</K8sProvider>
|
||||
)
|
||||
const result = render(view(namespace))
|
||||
return { switchTo: (ns: string) => result.rerender(view(ns)) }
|
||||
}
|
||||
|
||||
describe("LogsView with the production query defaults", () => {
|
||||
it("drops the previous tenant's lines and pods while the next tenant loads", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.includes("/tenant-switch-b/") ? new Promise(() => {}) : defaultHandler(endpoint, params),
|
||||
)
|
||||
const { switchTo } = renderWithProductionDefaults(client, "tenant-switch-a")
|
||||
expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
|
||||
await screen.findByRole("option", { name: "db-1" })
|
||||
await userEvent.keyboard("{Escape}")
|
||||
|
||||
switchTo("tenant-switch-b")
|
||||
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
expect(within(frame).getByText("Loading log lines…")).toBeInTheDocument()
|
||||
expect(screen.queryByText("ready to accept connections")).toBeNull()
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
|
||||
expect((await screen.findAllByRole("option")).map((o) => o.textContent)).toEqual(["All"])
|
||||
})
|
||||
|
||||
it("starts the next tenant with no filter, pod or container of the previous one", async () => {
|
||||
const { client, raw } = clientWith(defaultHandler)
|
||||
const { switchTo } = renderWithProductionDefaults(client, "tenant-reset-a")
|
||||
await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}")
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
|
||||
await userEvent.click(await screen.findByRole("option", { name: "db-1" }))
|
||||
await waitFor(() =>
|
||||
expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("query") === '{kubernetes_pod_name="db-1"} (error)')).toBe(true),
|
||||
)
|
||||
|
||||
switchTo("tenant-reset-b")
|
||||
|
||||
await waitFor(() => expect(raw.mock.calls.some(([p]) => p.includes("/tenant-reset-b/") && p.includes("select/logsql/query?"))).toBe(true))
|
||||
const first = raw.mock.calls.find(([p]) => p.includes("/tenant-reset-b/") && p.includes("select/logsql/query?"))!
|
||||
expect(new URL(first[0], "http://x").searchParams.get("query")).toBe("*")
|
||||
expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("All")
|
||||
expect(screen.getByRole("combobox", { name: "Container" })).toHaveTextContent("All")
|
||||
expect(screen.getByLabelText("LogsQL filter")).toHaveValue("")
|
||||
})
|
||||
|
||||
it("keeps the current lines while a new filter in the same tenant loads", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query") && params.get("query") === "slow" ? new Promise(() => {}) : defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithProductionDefaults(client, "tenant-filter-a")
|
||||
expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
|
||||
|
||||
await userEvent.type(screen.getByLabelText("LogsQL filter"), "slow{Enter}")
|
||||
|
||||
expect(screen.getByText("ready to accept connections")).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("says it is loading, not that nothing matched, while a longer range replaces an empty one", async () => {
|
||||
const { client } = clientWith((endpoint, params) => {
|
||||
if (!endpoint.endsWith("query")) return defaultHandler(endpoint, params)
|
||||
return Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000
|
||||
? new Promise(() => {})
|
||||
: new Response("")
|
||||
})
|
||||
renderWithProductionDefaults(client, "tenant-range-a")
|
||||
const frame = () => screen.getByRole("region", { name: "Log lines" })
|
||||
expect(await within(frame()).findByText("No log lines for this range and filter.")).toBeInTheDocument()
|
||||
|
||||
await userEvent.click(screen.getByRole("button", { name: "24h" }))
|
||||
|
||||
await waitFor(() => expect(within(frame()).getByRole("status")).toHaveAttribute("aria-busy", "true"))
|
||||
expect(within(frame()).queryByText("No log lines for this range and filter.")).toBeNull()
|
||||
})
|
||||
|
||||
it("marks the lines of the previous range as standing in while a longer range loads", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query") && Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000
|
||||
? new Promise(() => {})
|
||||
: defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithProductionDefaults(client, "tenant-range-b")
|
||||
expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
|
||||
expect(screen.queryByText(/previous lines/)).toBeNull()
|
||||
|
||||
await userEvent.click(screen.getByRole("button", { name: "24h" }))
|
||||
|
||||
expect(await screen.findByText(/previous lines/)).toHaveAttribute("aria-busy", "true")
|
||||
expect(screen.getByText("ready to accept connections")).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("drops the standing-in note once the longer range answers", async () => {
|
||||
let answer: (r: Response) => void = () => {}
|
||||
const longer = new Promise<Response>((resolve) => { answer = resolve })
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query") && Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000
|
||||
? longer
|
||||
: defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithProductionDefaults(client, "tenant-range-c")
|
||||
expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
|
||||
await userEvent.click(screen.getByRole("button", { name: "24h" }))
|
||||
expect(await screen.findByText(/previous lines/)).toBeInTheDocument()
|
||||
|
||||
answer(new Response(`${line("2026-11-03T12:00:00Z", "checkpoint complete")}\n`))
|
||||
|
||||
expect(await screen.findByText("checkpoint complete")).toBeInTheDocument()
|
||||
expect(screen.queryByText(/previous lines/)).toBeNull()
|
||||
expect(screen.queryByText("ready to accept connections")).toBeNull()
|
||||
})
|
||||
|
||||
it("shows no standing-in note while following", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("tail")
|
||||
? new Response(`${line("2026-11-04T10:00:02Z", "live line")}\n`)
|
||||
: endpoint.endsWith("query") && params.get("query")?.includes("error")
|
||||
? new Promise(() => {})
|
||||
: defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithProductionDefaults(client, "tenant-follow-a")
|
||||
expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
|
||||
|
||||
await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}")
|
||||
expect(await screen.findByText(/previous lines/)).toBeInTheDocument()
|
||||
await userEvent.click(screen.getByRole("button", { name: "Follow" }))
|
||||
|
||||
expect(await screen.findByText("live line")).toBeInTheDocument()
|
||||
expect(screen.queryByText(/previous lines/)).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe("LogsView", () => {
|
||||
it("shows recent lines from the application root", async () => {
|
||||
const { client, raw } = clientWith(defaultHandler)
|
||||
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
|
||||
const call = raw.mock.calls.find(([p]) => p.includes("select/logsql/query?"))!
|
||||
expect(call[0].startsWith(`${ROOT}select/logsql/query?`)).toBe(true)
|
||||
const params = new URL(call[0], "http://x").searchParams
|
||||
expect(params.get("query")).toBe("*")
|
||||
expect(params.get("limit")).toBe("500")
|
||||
})
|
||||
|
||||
it("narrows to the picked pod with a stream filter", async () => {
|
||||
const { client, raw } = clientWith(defaultHandler)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
|
||||
await userEvent.click(await screen.findByRole("option", { name: "db-1" }))
|
||||
|
||||
await waitFor(() =>
|
||||
expect(
|
||||
raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("query") === '{kubernetes_pod_name="db-1"} (*)'),
|
||||
).toBe(true),
|
||||
)
|
||||
})
|
||||
|
||||
it("offers All next to every pod and starts on it", async () => {
|
||||
const { client } = clientWith(defaultHandler)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
const pod = screen.getByRole("combobox", { name: "Pod" })
|
||||
expect(pod).toHaveTextContent("All")
|
||||
|
||||
await userEvent.click(pod)
|
||||
|
||||
expect(await screen.findByRole("option", { name: "db-1" })).toBeInTheDocument()
|
||||
expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0", "db-1"])
|
||||
})
|
||||
|
||||
it("keeps the log console and says so inside it when nothing matches", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params),
|
||||
)
|
||||
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
expect(await within(frame).findByText("No log lines for this range and filter.")).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("suggests a longer range only while one exists", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
const frame = () => screen.getByRole("region", { name: "Log lines" })
|
||||
expect(await within(frame()).findByText("Try a longer range or a broader filter.")).toBeInTheDocument()
|
||||
|
||||
await userEvent.click(screen.getByRole("button", { name: "24h" }))
|
||||
|
||||
await waitFor(() => expect(within(frame()).getByText("Try a broader filter.")).toBeInTheDocument())
|
||||
expect(within(frame()).queryByText(/longer range/)).toBeNull()
|
||||
})
|
||||
|
||||
it("shows loading inside the log console", () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query") ? new Promise(() => {}) : defaultHandler(endpoint, params),
|
||||
)
|
||||
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
expect(within(frame).getByRole("status")).toHaveAttribute("aria-busy", "true")
|
||||
})
|
||||
|
||||
it("says it is waiting while Follow has no line yet", async () => {
|
||||
const { client } = clientWith(defaultHandler)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
await userEvent.click(screen.getByRole("button", { name: "Follow" }))
|
||||
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
expect(within(frame).getByText("Waiting for new log lines…")).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("applies the LogsQL filter on submit", async () => {
|
||||
const { client, raw } = clientWith(defaultHandler)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}")
|
||||
|
||||
await waitFor(() =>
|
||||
expect(raw.mock.calls.some(([p]) => p.includes("select/logsql/query?") && p.includes("query=error"))).toBe(true),
|
||||
)
|
||||
})
|
||||
|
||||
it("follows the live tail when Follow is pressed", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("tail") ? new Response(`${line("2026-11-04T10:00:05Z", "checkpoint complete")}\n`) : defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
await userEvent.click(screen.getByRole("button", { name: "Follow" }))
|
||||
|
||||
expect(await screen.findByText("checkpoint complete")).toBeInTheDocument()
|
||||
expect(screen.getByRole("button", { name: "Follow" })).toHaveAttribute("aria-pressed", "true")
|
||||
})
|
||||
|
||||
it("says the tail stopped when the proxy refuses the query", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("tail") ? new Response("pipe not allowed in tail", { status: 400 }) : defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
await userEvent.click(screen.getByRole("button", { name: "Follow" }))
|
||||
|
||||
expect(await screen.findByText(/Stopped/)).toBeInTheDocument()
|
||||
expect(screen.getByText("pipe not allowed in tail")).toBeInTheDocument()
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
expect(within(frame).getByText("Follow stopped before any line arrived.")).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("renders a cluster without the proxy as not available", async () => {
|
||||
const { client } = clientWith(() => new Response("404 page not found", { status: 404 }))
|
||||
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument()
|
||||
expect(screen.queryByLabelText("LogsQL filter")).toBeNull()
|
||||
})
|
||||
|
||||
it("renders the tenant logs page as not available when kube-apiserver answers instead of the proxy", async () => {
|
||||
const { client } = clientWith(() =>
|
||||
Response.json({ kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403 }, { status: 403 }),
|
||||
)
|
||||
|
||||
renderWithK8sProvider(<LogsView scope={{ namespace: "tenant-a" }} />, { client })
|
||||
|
||||
expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("keeps the controls and explains a rejected query", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query") ? new Response("cannot parse query", { status: 400 }) : defaultHandler(endpoint, params),
|
||||
)
|
||||
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
expect(await screen.findByText("cannot parse query")).toBeInTheDocument()
|
||||
expect(screen.getByLabelText("LogsQL filter")).toBeInTheDocument()
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
expect(within(frame).getByText("No log lines to show.")).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("offers All once when some lines carry no pod name", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("stream_field_values") && params.get("field") === "kubernetes_pod_name"
|
||||
? values("", "db-0")
|
||||
: defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
|
||||
|
||||
await screen.findByRole("option", { name: "db-0" })
|
||||
expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0"])
|
||||
})
|
||||
|
||||
it("shows rows of a query whose pipe removed _time", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query") ? new Response('{"_msg":"visible message"}\n') : defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
await userEvent.type(screen.getByLabelText("LogsQL filter"), "* | fields _msg{Enter}")
|
||||
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
expect(await within(frame).findByText("visible message")).toBeInTheDocument()
|
||||
expect(within(frame).queryByText(/No log lines/)).toBeNull()
|
||||
})
|
||||
|
||||
it("keeps the view when only the pod list is refused", async () => {
|
||||
const status = JSON.stringify({ kind: "Status", apiVersion: "v1", status: "Failure", code: 404 })
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("stream_field_values") ? new Response(status, { status: 404 }) : defaultHandler(endpoint, params),
|
||||
)
|
||||
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
|
||||
expect(screen.queryByText("Logs are not available on this cluster.")).toBeNull()
|
||||
})
|
||||
|
||||
it("retries a failed history query on request", async () => {
|
||||
let calls = 0
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query") && calls++ === 0 ? new Response("upstream down", { status: 502 }) : defaultHandler(endpoint, params),
|
||||
)
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Retry" }))
|
||||
|
||||
expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
|
||||
expect(screen.queryByRole("button", { name: "Retry" })).toBeNull()
|
||||
})
|
||||
|
||||
it("says it is loading while a retry runs", async () => {
|
||||
let calls = 0
|
||||
const { client } = clientWith((endpoint, params) => {
|
||||
if (!endpoint.endsWith("query")) return defaultHandler(endpoint, params)
|
||||
return calls++ === 0 ? new Response("upstream down", { status: 502 }) : new Promise(() => {})
|
||||
})
|
||||
renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
|
||||
await userEvent.click(await screen.findByRole("button", { name: "Retry" }))
|
||||
|
||||
const frame = screen.getByRole("region", { name: "Log lines" })
|
||||
await waitFor(() => expect(within(frame).getByRole("status")).toHaveAttribute("aria-busy", "true"))
|
||||
expect(within(frame).queryByRole("button", { name: "Retry" })).toBeNull()
|
||||
})
|
||||
|
||||
it("opens a new history result at its newest line, not the old scroll position", async () => {
|
||||
const { client } = clientWith((endpoint, params) =>
|
||||
endpoint.endsWith("query")
|
||||
? new Response(`${line("2026-11-04T10:00:01Z", `result for ${params.get("query")}`)}\n`)
|
||||
: defaultHandler(endpoint, params),
|
||||
)
|
||||
const { queryClient } = renderWithK8sProvider(<LogsView scope={app} />, { client })
|
||||
queryClient.setDefaultOptions({ queries: { ...queryClient.getDefaultOptions().queries, placeholderData: (prev: unknown) => prev } })
|
||||
await screen.findByText("result for *")
|
||||
const before = screen.getByRole("list")
|
||||
|
||||
await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}")
|
||||
|
||||
await screen.findByText("result for error")
|
||||
expect(screen.getByRole("list")).not.toBe(before)
|
||||
})
|
||||
|
||||
it("names the source pod and container of each line", async () => {
|
||||
const { client } = clientWith(defaultHandler)
|
||||
|
||||
renderWithK8sProvider(<LogsView scope={{ namespace: "tenant-a" }} />, { client })
|
||||
|
||||
expect(await screen.findByText("db-0/postgres")).toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
import { useState } from "react"
|
||||
import { useQuery } from "@tanstack/react-query"
|
||||
import { useK8sClient } from "@cozystack/k8s-client"
|
||||
import { Button } from "@cozystack/ui"
|
||||
import { Input } from "../ui/input.tsx"
|
||||
import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "../ui/select.tsx"
|
||||
import {
|
||||
CONTAINER_FIELD,
|
||||
LOG_RANGES,
|
||||
POD_FIELD,
|
||||
buildLogsQuery,
|
||||
type LogRangeKey,
|
||||
} from "../../lib/observability/logs.ts"
|
||||
import {
|
||||
fieldValues,
|
||||
isScopeWide,
|
||||
logsRoot,
|
||||
queryLogs,
|
||||
type ObservabilityScope,
|
||||
} from "../../lib/observability/proxy.ts"
|
||||
import { useLogTail } from "../../lib/observability/use-log-tail.ts"
|
||||
import { ProxyNotice } from "./ProxyNotice.tsx"
|
||||
import { RangePicker } from "./RangePicker.tsx"
|
||||
import { LogLines } from "./LogLines.tsx"
|
||||
import { EmptyState } from "./EmptyState.tsx"
|
||||
|
||||
const HISTORY_LIMIT = 500
|
||||
const FOLLOW_CAP = 1000
|
||||
const TAIL_STATE_LABEL = { connecting: "Connecting…", live: "Live", reconnecting: "Reconnecting…", stopped: "Stopped" }
|
||||
|
||||
function windowFor(range: LogRangeKey) {
|
||||
const end = Date.now()
|
||||
return { start: new Date(end - LOG_RANGES[range] * 1000).toISOString(), end: new Date(end).toISOString() }
|
||||
}
|
||||
|
||||
const ALL = ""
|
||||
|
||||
function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) {
|
||||
const items = [{ value: ALL, label: "All" }, ...options.filter((o) => o !== ALL).map((o) => ({ value: o, label: o }))]
|
||||
return (
|
||||
<SelectRoot items={items} value={value} onValueChange={(v) => onChange(v ?? ALL)}>
|
||||
<div className="flex min-w-0 items-center gap-1.5">
|
||||
<SelectLabel>{label}</SelectLabel>
|
||||
<SelectTrigger className="w-40" title={value || undefined}>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
</div>
|
||||
<SelectPopup>
|
||||
{items.map((item) => (
|
||||
<SelectItem key={item.value} value={item.value}>
|
||||
{item.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectPopup>
|
||||
</SelectRoot>
|
||||
)
|
||||
}
|
||||
|
||||
const RANGE_KEYS = Object.keys(LOG_RANGES) as LogRangeKey[]
|
||||
const LONGEST_RANGE = RANGE_KEYS[RANGE_KEYS.length - 1]
|
||||
|
||||
function placeholderFor({ follow, stopped, loading, failed, longestRange, retry }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean; longestRange: boolean; retry: () => void }) {
|
||||
if (follow) {
|
||||
return stopped ? (
|
||||
<EmptyState tone="dark" title="Follow stopped before any line arrived." />
|
||||
) : (
|
||||
<EmptyState tone="dark" busy title="Waiting for new log lines…" hint="Lines written from now on appear here." />
|
||||
)
|
||||
}
|
||||
if (loading) return <EmptyState tone="dark" busy title="Loading log lines…" />
|
||||
if (failed) {
|
||||
return (
|
||||
<EmptyState
|
||||
tone="dark"
|
||||
title="No log lines to show."
|
||||
action={<Button variant="outline" size="sm" onClick={retry}>Retry</Button>}
|
||||
/>
|
||||
)
|
||||
}
|
||||
return <EmptyState tone="dark" title="No log lines for this range and filter." hint={longestRange ? "Try a broader filter." : "Try a longer range or a broader filter."} />
|
||||
}
|
||||
|
||||
// Filter, pod, container and the lines already read belong to one scope; a new tenant or application starts clean.
|
||||
export function LogsView({ scope }: { scope: ObservabilityScope }) {
|
||||
const root = logsRoot(scope)
|
||||
return <ScopedLogsView key={root} root={root} />
|
||||
}
|
||||
|
||||
function ScopedLogsView({ root }: { root: string }) {
|
||||
const client = useK8sClient()
|
||||
const [range, setRange] = useState<LogRangeKey>("1h")
|
||||
const [draft, setDraft] = useState("")
|
||||
const [filter, setFilter] = useState("")
|
||||
const [pod, setPod] = useState("")
|
||||
const [container, setContainer] = useState("")
|
||||
const [followRun, setFollowRun] = useState<number | null>(null)
|
||||
const follow = followRun !== null
|
||||
const query = buildLogsQuery({ filter, pod, container })
|
||||
|
||||
const pickerQuery = (field: string, narrowTo?: string) => ({
|
||||
queryKey: ["observability", "logs-fields", root, field, range, narrowTo ?? ""],
|
||||
queryFn: ({ signal }: { signal: AbortSignal }) =>
|
||||
fieldValues(client, root, {
|
||||
field,
|
||||
stream: true,
|
||||
query: buildLogsQuery({ filter: "", pod: narrowTo }),
|
||||
...windowFor(range),
|
||||
}, signal),
|
||||
retry: false,
|
||||
})
|
||||
const pods = useQuery(pickerQuery(POD_FIELD))
|
||||
const containers = useQuery(pickerQuery(CONTAINER_FIELD, pod || undefined))
|
||||
|
||||
const history = useQuery({
|
||||
queryKey: ["observability", "logs", root, query, range],
|
||||
queryFn: ({ signal }) => queryLogs(client, root, { query, limit: HISTORY_LIMIT, ...windowFor(range) }, signal),
|
||||
enabled: !follow,
|
||||
retry: false,
|
||||
})
|
||||
const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP })
|
||||
|
||||
// The pod list is optional: refusing it alone must not hide lines the query can still read.
|
||||
const wide = [history.error, tail.error].find(isScopeWide)
|
||||
if (wide) {
|
||||
return (
|
||||
<div className="p-6">
|
||||
<ProxyNotice what="Logs" error={wide} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const error = follow ? tail.error : history.error
|
||||
const entries = follow ? tail.entries : (history.data ?? [])
|
||||
|
||||
return (
|
||||
<div className="space-y-3 p-6">
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<form
|
||||
className="flex min-w-64 flex-1"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault()
|
||||
setFilter(draft)
|
||||
}}
|
||||
>
|
||||
<Input
|
||||
aria-label="LogsQL filter"
|
||||
value={draft}
|
||||
onValueChange={setDraft}
|
||||
placeholder='LogsQL filter, e.g. error or "connection refused"'
|
||||
className="font-mono"
|
||||
/>
|
||||
</form>
|
||||
<Picker label="Pod" value={pod} options={pods.data ?? []} onChange={(v) => { setPod(v); setContainer("") }} />
|
||||
<Picker label="Container" value={container} options={containers.data ?? []} onChange={setContainer} />
|
||||
{!follow && <RangePicker options={RANGE_KEYS} value={range} onChange={setRange} />}
|
||||
<Button variant={follow ? "default" : "outline"} size="sm" aria-pressed={follow} onClick={() => setFollowRun(follow ? null : Date.now())}>
|
||||
Follow
|
||||
</Button>
|
||||
</div>
|
||||
{follow && (
|
||||
<p className="text-xs text-slate-500">
|
||||
{TAIL_STATE_LABEL[tail.state]}
|
||||
{` · newest ${FOLLOW_CAP} lines kept`}
|
||||
</p>
|
||||
)}
|
||||
{!follow && history.isPlaceholderData && entries.length > 0 && (
|
||||
<p role="status" aria-busy="true" className="text-xs text-slate-500">
|
||||
Showing the previous lines until the new range or filter loads…
|
||||
</p>
|
||||
)}
|
||||
{error ? <ProxyNotice what="Logs" error={error} /> : null}
|
||||
<LogLines
|
||||
// Previous results stay as placeholder data while a new query loads, so only a new key resets the scroll pin.
|
||||
key={followRun ?? `history\n${query}\n${range}`}
|
||||
entries={entries}
|
||||
placeholder={placeholderFor({ follow, stopped: tail.state === "stopped", loading: history.isLoading || history.isPlaceholderData, failed: Boolean(error), longestRange: range === LONGEST_RANGE, retry: () => void history.refetch() })}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
import { describe, it, expect, vi } from "vitest"
|
||||
import { render, screen, waitFor, within } from "@testing-library/react"
|
||||
import userEvent from "@testing-library/user-event"
|
||||
import { K8sClient, K8sProvider } from "@cozystack/k8s-client"
|
||||
import { renderWithK8sProvider } from "../../test-utils/render.tsx"
|
||||
import type { ChartData } from "../../lib/observability/metrics.ts"
|
||||
|
||||
vi.mock("./TimeSeriesChart.tsx", () => ({
|
||||
TimeSeriesChart: ({ data }: { data: ChartData }) => (
|
||||
<div data-testid="chart">{data.series.map((s) => s.label).join(",")}</div>
|
||||
),
|
||||
}))
|
||||
|
||||
const { MetricsView } = await import("./MetricsView.tsx")
|
||||
|
||||
const scope = { namespace: "tenant-a", plural: "postgreses", name: "db" }
|
||||
|
||||
function matrix(pod: string) {
|
||||
return Response.json({
|
||||
status: "success",
|
||||
data: { resultType: "matrix", result: [{ metric: { pod }, values: [[1, "1"]] }] },
|
||||
})
|
||||
}
|
||||
|
||||
function clientWith(handler: (url: URL) => Response | Promise<Response>) {
|
||||
const client = new K8sClient()
|
||||
const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => handler(new URL(path, "http://x")))
|
||||
return { client, raw }
|
||||
}
|
||||
|
||||
describe("MetricsView", () => {
|
||||
it("draws CPU, memory and network from the application root", async () => {
|
||||
const { client, raw } = clientWith(() => matrix("db-0"))
|
||||
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
|
||||
await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3))
|
||||
expect(screen.getByText("CPU (cores)")).toBeInTheDocument()
|
||||
expect(screen.getByText("Memory (working set)")).toBeInTheDocument()
|
||||
expect(screen.getByText("Network")).toBeInTheDocument()
|
||||
for (const [path] of raw.mock.calls) {
|
||||
expect(path.startsWith("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query_range?")).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it("requeries with a coarser step when a longer range is picked", async () => {
|
||||
const { client, raw } = clientWith(() => matrix("db-0"))
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
await waitFor(() => expect(raw).toHaveBeenCalled())
|
||||
|
||||
await userEvent.click(screen.getByRole("button", { name: "7d" }))
|
||||
|
||||
await waitFor(() =>
|
||||
expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("step") === "2520")).toBe(true),
|
||||
)
|
||||
})
|
||||
|
||||
it("shows one quiet notice instead of three errors when the proxy is absent", async () => {
|
||||
const { client } = clientWith(() => new Response("404 page not found", { status: 404 }))
|
||||
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
|
||||
expect(await screen.findByText("Metrics are not available on this cluster.")).toBeInTheDocument()
|
||||
expect(screen.getAllByRole("status")).toHaveLength(1)
|
||||
expect(screen.queryByText("CPU (cores)")).toBeNull()
|
||||
})
|
||||
|
||||
it("keeps the other panels when one query hits a limit", async () => {
|
||||
const { client } = clientWith((url) =>
|
||||
url.searchParams.get("query")?.includes("memory")
|
||||
? Response.json({ status: "error", error: "too many series" }, { status: 422 })
|
||||
: matrix("db-0"),
|
||||
)
|
||||
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
|
||||
expect(await screen.findByText("too many series. Choose a shorter range.")).toBeInTheDocument()
|
||||
await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(2))
|
||||
})
|
||||
|
||||
it("says so when a panel has no data", async () => {
|
||||
const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } }))
|
||||
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
|
||||
await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3))
|
||||
for (const status of screen.getAllByRole("status")) expect(status).toHaveClass("h-[230px]")
|
||||
})
|
||||
|
||||
it("suggests a longer range only while one exists", async () => {
|
||||
const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } }))
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
await waitFor(() => expect(screen.getAllByText("Try a longer range.")).toHaveLength(3))
|
||||
|
||||
await userEvent.click(screen.getByRole("button", { name: "7d" }))
|
||||
|
||||
await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3))
|
||||
expect(screen.queryByText("Try a longer range.")).toBeNull()
|
||||
})
|
||||
|
||||
it("shows a panel's samples as a table on request", async () => {
|
||||
const { client } = clientWith(() => matrix("db-0"))
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3))
|
||||
const toggle = screen.getByRole("button", { name: "Show data for CPU (cores)" })
|
||||
expect(toggle).toHaveAttribute("aria-expanded", "false")
|
||||
|
||||
await userEvent.click(toggle)
|
||||
|
||||
const table = screen.getByRole("region", { name: "CPU (cores) data" })
|
||||
expect(toggle).toHaveAttribute("aria-expanded", "true")
|
||||
expect(toggle).toHaveAttribute("aria-controls", table.id)
|
||||
expect(toggle).toHaveTextContent("Hide data")
|
||||
expect(within(table).getByRole("columnheader", { name: "db-0" })).toBeInTheDocument()
|
||||
expect(screen.queryByRole("region", { name: "Memory (working set) data" })).toBeNull()
|
||||
})
|
||||
|
||||
it("offers no table while a panel has no samples", () => {
|
||||
const { client } = clientWith(() => new Promise<Response>(() => {}))
|
||||
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
|
||||
expect(screen.getByRole("button", { name: "Show data for Network" })).toBeDisabled()
|
||||
})
|
||||
|
||||
it("offers no table for a panel with no series or an error", async () => {
|
||||
const { client } = clientWith((url) =>
|
||||
url.searchParams.get("query")?.includes("memory")
|
||||
? Response.json({ status: "error", error: "too many series" }, { status: 422 })
|
||||
: Response.json({ status: "success", data: { resultType: "matrix", result: [] } }),
|
||||
)
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
|
||||
await screen.findByText("too many series. Choose a shorter range.")
|
||||
await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(2))
|
||||
for (const title of ["CPU (cores)", "Memory (working set)", "Network"]) {
|
||||
expect(screen.getByRole("button", { name: `Show data for ${title}` })).toBeDisabled()
|
||||
}
|
||||
})
|
||||
|
||||
it("drops the previous application's charts while the next one loads, with the production query defaults", async () => {
|
||||
const { client } = clientWith((url) => (url.pathname.includes("/switch-b/") ? new Promise<Response>(() => {}) : matrix("db-0")))
|
||||
const view = (name: string) => (
|
||||
<K8sProvider client={client}>
|
||||
<MetricsView scope={{ ...scope, name }} />
|
||||
</K8sProvider>
|
||||
)
|
||||
const { rerender } = render(view("switch-a"))
|
||||
await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3))
|
||||
|
||||
rerender(view("switch-b"))
|
||||
|
||||
expect(screen.queryAllByTestId("chart")).toHaveLength(0)
|
||||
expect(screen.getAllByText("Loading…")).toHaveLength(3)
|
||||
})
|
||||
|
||||
it("holds each chart's place while it loads", () => {
|
||||
const { client } = clientWith(() => new Promise<Response>(() => {}))
|
||||
|
||||
renderWithK8sProvider(<MetricsView scope={scope} />, { client })
|
||||
|
||||
const statuses = screen.getAllByRole("status")
|
||||
expect(statuses).toHaveLength(3)
|
||||
for (const status of statuses) {
|
||||
expect(status).toHaveAttribute("aria-busy", "true")
|
||||
expect(status).toHaveClass("h-[230px]")
|
||||
}
|
||||
})
|
||||
})
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
import { useId, useState } from "react"
|
||||
import { useQueries } from "@tanstack/react-query"
|
||||
import { useK8sClient } from "@cozystack/k8s-client"
|
||||
import { Button, Section } from "@cozystack/ui"
|
||||
import {
|
||||
METRIC_PANELS,
|
||||
RANGES,
|
||||
rangeWindow,
|
||||
toChartData,
|
||||
type RangeKey,
|
||||
} from "../../lib/observability/metrics.ts"
|
||||
import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts"
|
||||
import { ProxyNotice } from "./ProxyNotice.tsx"
|
||||
import { TimeSeriesChart } from "./TimeSeriesChart.tsx"
|
||||
import { RangePicker } from "./RangePicker.tsx"
|
||||
import { EmptyState } from "./EmptyState.tsx"
|
||||
import { SeriesTable } from "./SeriesTable.tsx"
|
||||
|
||||
const REFRESH_MS = 60_000
|
||||
// The uPlot canvas plus its one-row legend, so a placeholder takes the chart's place without a jump.
|
||||
const CHART_SLOT = "h-[230px] rounded-md border border-dashed border-slate-200 bg-slate-50"
|
||||
|
||||
export function MetricsView({ scope }: { scope: ObservabilityScope }) {
|
||||
const client = useK8sClient()
|
||||
const root = metricsRoot(scope)
|
||||
const [range, setRange] = useState<RangeKey>("1h")
|
||||
const [tables, setTables] = useState<ReadonlySet<string>>(new Set())
|
||||
const idBase = useId()
|
||||
const rangeKeys = Object.keys(RANGES) as RangeKey[]
|
||||
const longestRange = range === rangeKeys[rangeKeys.length - 1]
|
||||
|
||||
const results = useQueries({
|
||||
queries: METRIC_PANELS.map((panel) => ({
|
||||
queryKey: ["observability", "metrics", root, panel.id, range],
|
||||
queryFn: async ({ signal }: { signal: AbortSignal }) => {
|
||||
const window = rangeWindow(range, Math.floor(Date.now() / 1000))
|
||||
const series = await Promise.all(
|
||||
panel.queries(window).map(async (q) => ({
|
||||
label: q.label,
|
||||
series: await queryRange(client, root, { query: q.expr, ...window }, signal),
|
||||
})),
|
||||
)
|
||||
return toChartData(series)
|
||||
},
|
||||
retry: false,
|
||||
refetchInterval: REFRESH_MS,
|
||||
})),
|
||||
})
|
||||
|
||||
const wide = results.find((r) => isScopeWide(r.error))?.error
|
||||
|
||||
return (
|
||||
<div className="space-y-4 p-6">
|
||||
<div className="flex items-center justify-between">
|
||||
<h2 className="text-sm font-semibold text-slate-900">Resource consumption</h2>
|
||||
<RangePicker options={rangeKeys} value={range} onChange={setRange} />
|
||||
</div>
|
||||
{wide ? (
|
||||
<ProxyNotice what="Metrics" error={wide} />
|
||||
) : (
|
||||
METRIC_PANELS.map((panel, i) => {
|
||||
const { data, error, isLoading } = results[i]
|
||||
const hasSeries = !error && !isLoading && !!data && data.series.length > 0
|
||||
const tableOpen = hasSeries && tables.has(panel.id)
|
||||
const tableId = `${idBase}-${panel.id}-data`
|
||||
const toggle = (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={!hasSeries}
|
||||
aria-expanded={tableOpen}
|
||||
aria-controls={tableOpen ? tableId : undefined}
|
||||
onClick={() =>
|
||||
setTables((prev) => {
|
||||
const next = new Set(prev)
|
||||
if (!next.delete(panel.id)) next.add(panel.id)
|
||||
return next
|
||||
})
|
||||
}
|
||||
>
|
||||
{tableOpen ? "Hide data" : "Show data"}{" "}
|
||||
<span className="sr-only">for {panel.title}</span>
|
||||
</Button>
|
||||
)
|
||||
return (
|
||||
<Section key={panel.id} title={panel.title} actions={toggle}>
|
||||
{error ? (
|
||||
<ProxyNotice what="Metrics" error={error} />
|
||||
) : isLoading || !data ? (
|
||||
<EmptyState busy title="Loading…" className={CHART_SLOT} />
|
||||
) : data.series.length === 0 ? (
|
||||
<EmptyState title="No data for this range." hint={longestRange ? undefined : "Try a longer range."} className={CHART_SLOT} />
|
||||
) : (
|
||||
<>
|
||||
<TimeSeriesChart data={data} format={panel.format} />
|
||||
{tableOpen && <SeriesTable id={tableId} title={panel.title} data={data} format={panel.format} />}
|
||||
</>
|
||||
)}
|
||||
</Section>
|
||||
)
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
import { describe, it, expect } from "vitest"
|
||||
import { render, screen } from "@testing-library/react"
|
||||
import { ProxyError } from "../../lib/observability/proxy.ts"
|
||||
import { ProxyNotice } from "./ProxyNotice.tsx"
|
||||
|
||||
describe("ProxyNotice", () => {
|
||||
it("renders a 404 as not available on this cluster, not as an error", () => {
|
||||
render(<ProxyNotice what="Metrics" error={new ProxyError("not-available", 404, "page not found")} />)
|
||||
|
||||
expect(screen.getByRole("status")).toHaveTextContent("Metrics are not available on this cluster.")
|
||||
expect(screen.queryByRole("alert")).toBeNull()
|
||||
})
|
||||
|
||||
it("renders a 502 as a quiet notice about the store", () => {
|
||||
render(<ProxyNotice what="Logs" error={new ProxyError("store-unreachable", 502, "dial tcp")} />)
|
||||
|
||||
expect(screen.getByRole("status")).toHaveTextContent("The logs store is unreachable right now.")
|
||||
})
|
||||
|
||||
it.each([
|
||||
["forbidden", 403, "You do not have permission to read metrics here."],
|
||||
["limit", 422, "too many points. Choose a shorter range."],
|
||||
["rate-limited", 429, "Too many requests. Try again in a moment."],
|
||||
["timeout", 504, "The store did not answer in time. Choose a shorter range or a narrower filter."],
|
||||
["unavailable", 503, "The service is temporarily unavailable. Try again in a moment."],
|
||||
] as const)("explains %s", (kind, status, text) => {
|
||||
render(<ProxyNotice what="Metrics" error={new ProxyError(kind, status, "too many points")} />)
|
||||
|
||||
expect(screen.getByRole("alert")).toHaveTextContent(text)
|
||||
})
|
||||
|
||||
it.each([
|
||||
"the application has more than 50 pods in this window; use a shorter range",
|
||||
"1200 points per series exceed the limit of 1000; raise step",
|
||||
])("does not add advice to a limit message that already carries its own: %s", (message) => {
|
||||
render(<ProxyNotice what="Metrics" error={new ProxyError("limit", 422, message)} />)
|
||||
|
||||
expect(screen.getByRole("alert").textContent).toBe(`${message}.`)
|
||||
})
|
||||
|
||||
it("still advises a shorter range for a store limit whose own advice is for operators", () => {
|
||||
const message = "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values"
|
||||
render(<ProxyNotice what="Metrics" error={new ProxyError("limit", 422, message)} />)
|
||||
|
||||
expect(screen.getByRole("alert").textContent).toBe(`${message}. Choose a shorter range.`)
|
||||
})
|
||||
|
||||
it("shows the server message for a rejected query", () => {
|
||||
render(<ProxyNotice what="Logs" error={new ProxyError("bad-request", 400, "cannot parse query")} />)
|
||||
|
||||
expect(screen.getByRole("alert")).toHaveTextContent("cannot parse query")
|
||||
})
|
||||
|
||||
it("falls back to the message of an unexpected error", () => {
|
||||
render(<ProxyNotice what="Logs" error={new Error("boom")} />)
|
||||
|
||||
expect(screen.getByRole("alert")).toHaveTextContent("boom")
|
||||
})
|
||||
})
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
import { ProxyError } from "../../lib/observability/proxy.ts"
|
||||
|
||||
interface ProxyNoticeProps {
|
||||
what: "Metrics" | "Logs"
|
||||
error: unknown
|
||||
}
|
||||
|
||||
// Remedies the observability proxy itself appends; vmselect's 422 text carries
|
||||
// advice for operators (raise -search.max* flags), so it still gets ours.
|
||||
const PROXY_REMEDIES = ["; use a shorter range", "; raise step"]
|
||||
|
||||
function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: boolean; text: string } {
|
||||
const lower = what.toLowerCase()
|
||||
if (!(error instanceof ProxyError)) {
|
||||
return { quiet: false, text: error instanceof Error ? error.message : String(error) }
|
||||
}
|
||||
switch (error.kind) {
|
||||
case "not-available":
|
||||
return { quiet: true, text: `${what} are not available on this cluster.` }
|
||||
case "store-unreachable":
|
||||
return { quiet: true, text: `The ${lower} store is unreachable right now.` }
|
||||
case "unauthorized":
|
||||
return { quiet: false, text: "Your session has expired. Sign in again." }
|
||||
case "forbidden":
|
||||
return { quiet: false, text: `You do not have permission to read ${lower} here.` }
|
||||
case "limit":
|
||||
return {
|
||||
quiet: false,
|
||||
text: PROXY_REMEDIES.some((r) => error.message.endsWith(r)) ? `${error.message}.` : `${error.message}. Choose a shorter range.`,
|
||||
}
|
||||
case "rate-limited":
|
||||
return { quiet: false, text: "Too many requests. Try again in a moment." }
|
||||
case "timeout":
|
||||
return { quiet: false, text: "The store did not answer in time. Choose a shorter range or a narrower filter." }
|
||||
case "unavailable":
|
||||
return { quiet: false, text: "The service is temporarily unavailable. Try again in a moment." }
|
||||
default:
|
||||
return { quiet: false, text: error.message }
|
||||
}
|
||||
}
|
||||
|
||||
export function ProxyNotice({ what, error }: ProxyNoticeProps) {
|
||||
const { quiet, text } = describe(what, error)
|
||||
if (quiet) {
|
||||
return (
|
||||
<p role="status" className="rounded-md border border-slate-200 bg-slate-50 px-4 py-3 text-sm text-slate-600">
|
||||
{text}
|
||||
</p>
|
||||
)
|
||||
}
|
||||
return (
|
||||
<p role="alert" className="rounded-md border border-amber-200 bg-amber-50 px-4 py-3 text-sm text-amber-800">
|
||||
{text}
|
||||
</p>
|
||||
)
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
import { describe, it, expect, vi } from "vitest"
|
||||
import { render, screen, within } from "@testing-library/react"
|
||||
import userEvent from "@testing-library/user-event"
|
||||
import { RangePicker } from "./RangePicker.tsx"
|
||||
|
||||
describe("RangePicker", () => {
|
||||
it("groups the ranges under one name and marks the chosen one pressed", () => {
|
||||
render(<RangePicker options={["1h", "6h"]} value="6h" onChange={() => {}} />)
|
||||
|
||||
const group = screen.getByRole("group", { name: "Time range" })
|
||||
expect(within(group).getByRole("button", { name: "6h" })).toHaveAttribute("aria-pressed", "true")
|
||||
expect(within(group).getByRole("button", { name: "1h" })).toHaveAttribute("aria-pressed", "false")
|
||||
})
|
||||
|
||||
it("reports the range clicked", async () => {
|
||||
const onChange = vi.fn()
|
||||
render(<RangePicker options={["1h", "6h"]} value="6h" onChange={onChange} />)
|
||||
|
||||
await userEvent.click(screen.getByRole("button", { name: "1h" }))
|
||||
|
||||
expect(onChange).toHaveBeenCalledWith("1h")
|
||||
})
|
||||
})
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
import { cn } from "@cozystack/ui"
|
||||
|
||||
export function RangePicker<K extends string>({
|
||||
options,
|
||||
value,
|
||||
onChange,
|
||||
}: {
|
||||
options: readonly K[]
|
||||
value: K
|
||||
onChange: (value: K) => void
|
||||
}) {
|
||||
return (
|
||||
<div role="group" aria-label="Time range" className="inline-flex h-7 items-center rounded-lg border border-slate-300 bg-white p-0.5 shadow-xs">
|
||||
{options.map((key) => (
|
||||
<button
|
||||
key={key}
|
||||
type="button"
|
||||
aria-pressed={value === key}
|
||||
onClick={() => onChange(key)}
|
||||
className={cn(
|
||||
"h-full rounded-md px-2.5 text-xs outline-none focus-visible:ring-2 focus-visible:ring-slate-400 focus-visible:ring-offset-2",
|
||||
value === key ? "bg-slate-900 text-white" : "text-slate-600 hover:bg-slate-100",
|
||||
)}
|
||||
>
|
||||
{key}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
import { describe, it, expect } from "vitest"
|
||||
import { render, screen, within } from "@testing-library/react"
|
||||
import { SeriesTable } from "./SeriesTable.tsx"
|
||||
|
||||
const data = {
|
||||
timestamps: [1_700_000_000, 1_700_000_060],
|
||||
series: [
|
||||
{ label: "db-0", values: [1, 2] },
|
||||
{ label: "db-1", values: [null, 3] },
|
||||
],
|
||||
}
|
||||
|
||||
describe("SeriesTable", () => {
|
||||
it("is a named region the keyboard can reach", () => {
|
||||
render(<SeriesTable id="t" title="Memory" data={data} format={String} />)
|
||||
|
||||
const region = screen.getByRole("region", { name: "Memory data" })
|
||||
expect(region).toHaveAttribute("id", "t")
|
||||
expect(region).toHaveAttribute("tabindex", "0")
|
||||
})
|
||||
|
||||
it("has a time column and one column per series", () => {
|
||||
render(<SeriesTable id="t" title="Memory" data={data} format={String} />)
|
||||
|
||||
expect(screen.getAllByRole("columnheader").map((h) => h.textContent)).toEqual(["Time", "db-0", "db-1"])
|
||||
})
|
||||
|
||||
it("lists every sample newest first in the chart's units", () => {
|
||||
render(<SeriesTable id="t" title="Memory" data={data} format={(v) => `${v} B`} />)
|
||||
|
||||
const rows = screen.getAllByRole("row").slice(1)
|
||||
expect(rows).toHaveLength(2)
|
||||
expect(within(rows[0]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["2 B", "3 B"])
|
||||
expect(within(rows[1]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["1 B", "–"])
|
||||
expect(within(rows[0]).getByRole("rowheader")).toHaveTextContent(new Date(1_700_000_060_000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" }))
|
||||
})
|
||||
})
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
import { formatSampleTime, type ChartData } from "../../lib/observability/metrics.ts"
|
||||
|
||||
interface SeriesTableProps {
|
||||
id: string
|
||||
title: string
|
||||
data: ChartData
|
||||
format: (value: number) => string
|
||||
}
|
||||
|
||||
export function SeriesTable({ id, title, data, format }: SeriesTableProps) {
|
||||
const newestFirst = data.timestamps.map((t, i) => ({ t, i })).reverse()
|
||||
return (
|
||||
<div
|
||||
id={id}
|
||||
role="region"
|
||||
aria-label={`${title} data`}
|
||||
tabIndex={0}
|
||||
className="mt-3 max-h-64 overflow-auto rounded-md border border-slate-200 outline-none focus-visible:ring-2 focus-visible:ring-slate-400 focus-visible:ring-offset-2"
|
||||
>
|
||||
<table className="w-full text-xs tabular-nums">
|
||||
<caption className="sr-only">{`${title}, newest first`}</caption>
|
||||
<thead className="sticky top-0 bg-slate-50 text-slate-500">
|
||||
<tr>
|
||||
<th scope="col" className="px-3 py-1.5 text-left font-medium">Time</th>
|
||||
{data.series.map((s, j) => (
|
||||
<th key={j} scope="col" className="px-3 py-1.5 text-right font-medium">{s.label}</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-slate-100 text-slate-700">
|
||||
{newestFirst.map(({ t, i }) => (
|
||||
<tr key={t}>
|
||||
<th scope="row" className="px-3 py-1 text-left font-normal whitespace-nowrap">{formatSampleTime(t)}</th>
|
||||
{data.series.map((s, j) => {
|
||||
const v = s.values[i]
|
||||
return <td key={j} className="px-3 py-1 text-right">{v === null ? "–" : format(v)}</td>
|
||||
})}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest"
|
||||
import { render } from "@testing-library/react"
|
||||
|
||||
type LegendValue = (u: unknown, v: number | null, sidx: number, idx: number | null) => string
|
||||
type Opts = { series: { label?: string; value?: LegendValue }[] }
|
||||
|
||||
const created: { opts: Opts; data: unknown[]; destroy: ReturnType<typeof vi.fn> }[] = []
|
||||
|
||||
vi.mock("uplot", () => ({
|
||||
default: class {
|
||||
destroy = vi.fn()
|
||||
constructor(opts: Opts, data: unknown[]) {
|
||||
created.push({ opts, data, destroy: this.destroy })
|
||||
}
|
||||
setSize() {}
|
||||
},
|
||||
}))
|
||||
|
||||
const { TimeSeriesChart } = await import("./TimeSeriesChart.tsx")
|
||||
const { formatSampleTime } = await import("../../lib/observability/metrics.ts")
|
||||
|
||||
beforeEach(() => {
|
||||
created.length = 0
|
||||
})
|
||||
|
||||
describe("TimeSeriesChart", () => {
|
||||
const data = {
|
||||
timestamps: [1, 2],
|
||||
series: [
|
||||
{ label: "db-0", values: [1, 2] },
|
||||
{ label: "db-1", values: [null, 3] },
|
||||
],
|
||||
}
|
||||
|
||||
it("hands uPlot the timestamps first and one labelled series per input", () => {
|
||||
render(<TimeSeriesChart data={data} format={String} />)
|
||||
|
||||
expect(created).toHaveLength(1)
|
||||
expect(created[0].data).toEqual([[1, 2], [1, 2], [null, 3]])
|
||||
expect(created[0].opts.series.slice(1).map((s) => s.label)).toEqual(["db-0", "db-1"])
|
||||
})
|
||||
|
||||
it("shows each series' latest value in the legend while nothing is hovered", () => {
|
||||
render(<TimeSeriesChart data={{ ...data, timestamps: [1, 2, 3], series: [{ label: "db-0", values: [1, 2, null] }] }} format={(v) => `${v} B`} />)
|
||||
|
||||
const legend = created[0].opts.series[1].value!
|
||||
|
||||
expect(legend(null, null, 1, null)).toBe("2 B")
|
||||
expect(legend(null, 1, 1, 0)).toBe("1 B")
|
||||
expect(legend(null, null, 1, 2)).toBe("–")
|
||||
})
|
||||
|
||||
it("labels the legend's time as latest while nothing is hovered", () => {
|
||||
render(<TimeSeriesChart data={data} format={String} />)
|
||||
|
||||
const time = created[0].opts.series[0].value!
|
||||
|
||||
expect(time(null, null, 0, null)).toBe("latest")
|
||||
expect(time(null, 2, 0, 1)).toBe(formatSampleTime(2))
|
||||
})
|
||||
|
||||
it("destroys the chart on unmount", () => {
|
||||
const { unmount } = render(<TimeSeriesChart data={data} format={String} />)
|
||||
|
||||
unmount()
|
||||
|
||||
expect(created[0].destroy).toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
import { useEffect, useRef } from "react"
|
||||
import uPlot from "uplot"
|
||||
import "uplot/dist/uPlot.min.css"
|
||||
import { formatSampleTime, type ChartData } from "../../lib/observability/metrics.ts"
|
||||
|
||||
const COLORS = ["#2563eb", "#16a34a", "#d97706", "#9333ea", "#dc2626", "#0891b2", "#4b5563", "#db2777"]
|
||||
const HEIGHT = 200
|
||||
|
||||
interface TimeSeriesChartProps {
|
||||
data: ChartData
|
||||
format: (value: number) => string
|
||||
}
|
||||
|
||||
export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
|
||||
const ref = useRef<HTMLDivElement>(null)
|
||||
|
||||
useEffect(() => {
|
||||
const el = ref.current
|
||||
if (!el) return
|
||||
const fmt = (v: number | null) => (v === null ? "–" : format(v))
|
||||
const latest = data.series.map((s) => s.values.findLast((v) => v !== null) ?? null)
|
||||
const chart = new uPlot(
|
||||
{
|
||||
width: el.clientWidth || 600,
|
||||
height: HEIGHT,
|
||||
scales: { x: { time: true } },
|
||||
legend: { live: true },
|
||||
series: [
|
||||
{ value: (_: uPlot, v: number | null, _sidx: number, idx: number | null) => (idx === null || v === null ? "latest" : formatSampleTime(v)) },
|
||||
...data.series.map((s, i) => ({
|
||||
label: s.label,
|
||||
stroke: COLORS[i % COLORS.length],
|
||||
width: 1.5,
|
||||
// Without a hovered point the legend shows the latest sample instead of a dash.
|
||||
value: (_: uPlot, v: number | null, sidx: number, idx: number | null) => fmt(idx === null ? latest[sidx - 1] : v),
|
||||
})),
|
||||
],
|
||||
axes: [{}, { size: 84, values: (_: uPlot, splits: number[]) => splits.map(format) }],
|
||||
},
|
||||
[data.timestamps, ...data.series.map((s) => s.values)],
|
||||
el,
|
||||
)
|
||||
if (typeof ResizeObserver === "undefined") return () => chart.destroy()
|
||||
const observer = new ResizeObserver(() => chart.setSize({ width: el.clientWidth, height: HEIGHT }))
|
||||
observer.observe(el)
|
||||
return () => {
|
||||
observer.disconnect()
|
||||
chart.destroy()
|
||||
}
|
||||
}, [data, format])
|
||||
|
||||
return <div ref={ref} className="w-full" />
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
import { describe, it, expect, vi } from "vitest"
|
||||
import { render, screen } from "@testing-library/react"
|
||||
import userEvent from "@testing-library/user-event"
|
||||
import { Input } from "./input.tsx"
|
||||
|
||||
describe("Input", () => {
|
||||
it("is a labelled text box that reports what is typed", async () => {
|
||||
const onValueChange = vi.fn()
|
||||
render(<Input aria-label="Filter" defaultValue="" onValueChange={onValueChange} />)
|
||||
|
||||
await userEvent.type(screen.getByRole("textbox", { name: "Filter" }), "x")
|
||||
|
||||
expect(onValueChange).toHaveBeenCalledWith("x", expect.anything())
|
||||
})
|
||||
|
||||
it("keeps the caller's classes next to its own", () => {
|
||||
render(<Input aria-label="Filter" className="font-mono" />)
|
||||
|
||||
expect(screen.getByRole("textbox", { name: "Filter" })).toHaveClass("font-mono", "rounded-lg")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Input as InputPrimitive } from "@base-ui/react/input"
|
||||
import { cn } from "@cozystack/ui"
|
||||
|
||||
function Input({ className, ...props }: InputPrimitive.Props) {
|
||||
return (
|
||||
<InputPrimitive
|
||||
data-slot="input"
|
||||
className={cn(
|
||||
"h-7 w-full min-w-0 rounded-lg border border-slate-300 bg-white px-2.5 text-xs text-slate-900 shadow-xs outline-none transition-colors placeholder:text-slate-400 focus-visible:ring-2 focus-visible:ring-slate-400 focus-visible:ring-offset-2 data-[disabled]:pointer-events-none data-[disabled]:opacity-50",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
export { Input }
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
import { describe, it, expect, vi } from "vitest"
|
||||
import { render, screen } from "@testing-library/react"
|
||||
import userEvent from "@testing-library/user-event"
|
||||
import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "./select.tsx"
|
||||
|
||||
const items = [
|
||||
{ value: "", label: "All" },
|
||||
{ value: "db-0", label: "db-0" },
|
||||
{ value: "db-1", label: "db-1" },
|
||||
]
|
||||
|
||||
function PodSelect({ value = "", onValueChange = () => {} }: { value?: string; onValueChange?: (v: string) => void }) {
|
||||
return (
|
||||
<SelectRoot items={items} value={value} onValueChange={(v) => onValueChange(v ?? "")}>
|
||||
<SelectLabel>Pod</SelectLabel>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectPopup>
|
||||
{items.map((i) => (
|
||||
<SelectItem key={i.value} value={i.value}>
|
||||
{i.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectPopup>
|
||||
</SelectRoot>
|
||||
)
|
||||
}
|
||||
|
||||
describe("Select", () => {
|
||||
it("names the trigger by its label and shows the chosen item", () => {
|
||||
render(<PodSelect value="db-1" />)
|
||||
|
||||
expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("db-1")
|
||||
})
|
||||
|
||||
it("shows the label of an empty-string value", () => {
|
||||
render(<PodSelect />)
|
||||
|
||||
expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("All")
|
||||
})
|
||||
|
||||
it("lists the items and reports the one clicked", async () => {
|
||||
const onValueChange = vi.fn()
|
||||
render(<PodSelect onValueChange={onValueChange} />)
|
||||
|
||||
await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
|
||||
await userEvent.click(await screen.findByRole("option", { name: "db-1" }))
|
||||
|
||||
expect(onValueChange).toHaveBeenCalledWith("db-1")
|
||||
})
|
||||
|
||||
it("can be driven from the keyboard", async () => {
|
||||
const onValueChange = vi.fn()
|
||||
render(<PodSelect onValueChange={onValueChange} />)
|
||||
|
||||
screen.getByRole("combobox", { name: "Pod" }).focus()
|
||||
await userEvent.keyboard("{ArrowDown}")
|
||||
expect(await screen.findByRole("listbox")).toBeInTheDocument()
|
||||
await userEvent.keyboard("{ArrowDown}{Enter}")
|
||||
|
||||
expect(onValueChange).toHaveBeenCalledWith("db-0")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,107 @@
|
||||
import { Select as SelectPrimitive } from "@base-ui/react/select"
|
||||
import { Check, ChevronDown } from "lucide-react"
|
||||
import { cn } from "@cozystack/ui"
|
||||
|
||||
const SelectRoot = SelectPrimitive.Root
|
||||
|
||||
function SelectLabel({
|
||||
className,
|
||||
...props
|
||||
}: SelectPrimitive.Label.Props) {
|
||||
return (
|
||||
<SelectPrimitive.Label
|
||||
data-slot="select-label"
|
||||
className={cn("text-xs text-slate-600", className)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function SelectTrigger({
|
||||
className,
|
||||
children,
|
||||
...props
|
||||
}: SelectPrimitive.Trigger.Props) {
|
||||
return (
|
||||
<SelectPrimitive.Trigger
|
||||
data-slot="select-trigger"
|
||||
className={cn(
|
||||
"inline-flex h-7 min-w-0 items-center justify-between gap-1.5 rounded-lg border border-slate-300 bg-white px-2 text-xs text-slate-900 shadow-xs outline-none transition-colors select-none hover:bg-slate-50 focus-visible:ring-2 focus-visible:ring-slate-400 focus-visible:ring-offset-2 data-[disabled]:pointer-events-none data-[disabled]:opacity-50 data-[popup-open]:bg-slate-50",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
{children}
|
||||
<SelectPrimitive.Icon className="shrink-0 text-slate-400">
|
||||
<ChevronDown className="size-3.5" />
|
||||
</SelectPrimitive.Icon>
|
||||
</SelectPrimitive.Trigger>
|
||||
)
|
||||
}
|
||||
|
||||
function SelectValue({
|
||||
className,
|
||||
...props
|
||||
}: SelectPrimitive.Value.Props) {
|
||||
return (
|
||||
<SelectPrimitive.Value
|
||||
data-slot="select-value"
|
||||
className={cn("truncate", className)}
|
||||
{...props}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
function SelectPopup({
|
||||
className,
|
||||
children,
|
||||
...props
|
||||
}: SelectPrimitive.Popup.Props) {
|
||||
return (
|
||||
<SelectPrimitive.Portal>
|
||||
<SelectPrimitive.Positioner className="z-50 outline-none" sideOffset={4} alignItemWithTrigger={false}>
|
||||
<SelectPrimitive.Popup
|
||||
data-slot="select-popup"
|
||||
className={cn(
|
||||
"max-h-[min(var(--available-height),20rem)] max-w-[min(var(--available-width),32rem)] min-w-[var(--anchor-width)] overflow-y-auto rounded-lg bg-white py-1 text-xs text-slate-900 shadow-lg ring-1 ring-slate-900/8 outline-none",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<SelectPrimitive.List>{children}</SelectPrimitive.List>
|
||||
</SelectPrimitive.Popup>
|
||||
</SelectPrimitive.Positioner>
|
||||
</SelectPrimitive.Portal>
|
||||
)
|
||||
}
|
||||
|
||||
function SelectItem({
|
||||
className,
|
||||
children,
|
||||
...props
|
||||
}: SelectPrimitive.Item.Props) {
|
||||
return (
|
||||
<SelectPrimitive.Item
|
||||
data-slot="select-item"
|
||||
className={cn(
|
||||
"grid cursor-default grid-cols-[1rem_1fr] items-center gap-1.5 px-2 py-1.5 outline-none select-none data-[highlighted]:bg-slate-100 data-[selected]:font-medium",
|
||||
className
|
||||
)}
|
||||
{...props}
|
||||
>
|
||||
<SelectPrimitive.ItemIndicator className="col-start-1">
|
||||
<Check className="size-3.5" />
|
||||
</SelectPrimitive.ItemIndicator>
|
||||
<SelectPrimitive.ItemText className="col-start-2 break-all">{children}</SelectPrimitive.ItemText>
|
||||
</SelectPrimitive.Item>
|
||||
)
|
||||
}
|
||||
|
||||
export {
|
||||
SelectRoot,
|
||||
SelectLabel,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
SelectPopup,
|
||||
SelectItem,
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
import { describe, it, expect } from "vitest"
|
||||
import { buildLogsQuery, compareTime, mergeTail } from "./logs.ts"
|
||||
|
||||
describe("buildLogsQuery", () => {
|
||||
it("passes the filter through when nothing is picked", () => {
|
||||
expect(buildLogsQuery({ filter: "error or warn" })).toBe("error or warn")
|
||||
expect(buildLogsQuery({ filter: " " })).toBe("*")
|
||||
})
|
||||
|
||||
it("ANDs the picked pod and container as a stream filter around the whole filter", () => {
|
||||
expect(buildLogsQuery({ filter: "error or warn", pod: "db-0", container: "postgres" })).toBe(
|
||||
'{kubernetes_pod_name="db-0",kubernetes_container_name="postgres"} (error or warn)',
|
||||
)
|
||||
})
|
||||
|
||||
it("keeps the user's pipes after the scoped filter", () => {
|
||||
expect(buildLogsQuery({ filter: 'a or "x|y" | fields _msg', pod: "db-0" })).toBe(
|
||||
'{kubernetes_pod_name="db-0"} (a or "x|y") | fields _msg',
|
||||
)
|
||||
})
|
||||
|
||||
it("escapes quotes in picked values", () => {
|
||||
expect(buildLogsQuery({ filter: "", pod: 'a"b' })).toBe('{kubernetes_pod_name="a\\"b"} (*)')
|
||||
})
|
||||
})
|
||||
|
||||
describe("compareTime", () => {
|
||||
it("orders RFC3339 times with differing fraction lengths", () => {
|
||||
expect(compareTime("2026-11-04T10:00:01Z", "2026-11-04T10:00:01.1Z")).toBeLessThan(0)
|
||||
expect(compareTime("2026-11-04T10:00:01.5Z", "2026-11-04T10:00:01.123Z")).toBeGreaterThan(0)
|
||||
expect(compareTime("2026-11-04T10:00:01.10Z", "2026-11-04T10:00:01.1Z")).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
const e = (time: string, msg: string, stream = "s1") => ({ _time: time, _msg: msg, _stream_id: stream })
|
||||
|
||||
describe("mergeTail", () => {
|
||||
it("appends and keeps only the newest lines over the cap", () => {
|
||||
const out = mergeTail([e("t1", "a"), e("t2", "b")], [e("t3", "c")], 2)
|
||||
|
||||
expect(out.map((x) => x._msg)).toEqual(["b", "c"])
|
||||
})
|
||||
|
||||
it("drops replayed lines at or before the resume point", () => {
|
||||
const buffer = [e("2026-01-01T00:00:01Z", "a"), e("2026-01-01T00:00:02Z", "b")]
|
||||
|
||||
const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "b"), e("2026-01-01T00:00:03Z", "c")], 10, "2026-01-01T00:00:02Z")
|
||||
|
||||
expect(out.map((x) => x._msg)).toEqual(["a", "b", "c"])
|
||||
})
|
||||
|
||||
it("keeps a line at the resume point from another stream", () => {
|
||||
const buffer = [e("2026-01-01T00:00:02Z", "b", "s1")]
|
||||
|
||||
const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "other", "s2")], 10, "2026-01-01T00:00:02Z")
|
||||
|
||||
expect(out).toHaveLength(2)
|
||||
})
|
||||
|
||||
it("never deduplicates lines without _time, even on a resume", () => {
|
||||
const out = mergeTail([{ _msg: "a" }], [{ _msg: "a" }], 10, "2026-01-01T00:00:02Z")
|
||||
|
||||
expect(out).toHaveLength(2)
|
||||
})
|
||||
|
||||
it("never deduplicates lines a pipe stripped of _stream_id, even on a resume", () => {
|
||||
const t = "2026-01-01T00:00:02Z"
|
||||
const buffer = [{ _time: t, _msg: "from stream 1" }]
|
||||
|
||||
const out = mergeTail(buffer, [{ _time: t, _msg: "from stream 2" }], 10, t)
|
||||
|
||||
expect(out.map((x) => x._msg)).toEqual(["from stream 1", "from stream 2"])
|
||||
})
|
||||
|
||||
it("deduplicates only the complete rows of a mixed resume", () => {
|
||||
const t = "2026-01-01T00:00:02Z"
|
||||
const buffer = [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }]
|
||||
|
||||
const out = mergeTail(buffer, [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }], 10, t)
|
||||
|
||||
expect(out.map((x) => x._msg)).toEqual(["seen", "no stream", "no time", "no stream", "no time"])
|
||||
})
|
||||
|
||||
it("does not deduplicate outside a resume", () => {
|
||||
const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10)
|
||||
|
||||
expect(out).toHaveLength(2)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,79 @@
|
||||
import type { LogEntry } from "./proxy.ts"
|
||||
|
||||
export const LOG_RANGES = {
|
||||
"15m": 15 * 60,
|
||||
"1h": 3600,
|
||||
"6h": 6 * 3600,
|
||||
"24h": 24 * 3600,
|
||||
} as const
|
||||
|
||||
export type LogRangeKey = keyof typeof LOG_RANGES
|
||||
|
||||
export const POD_FIELD = "kubernetes_pod_name"
|
||||
export const CONTAINER_FIELD = "kubernetes_container_name"
|
||||
|
||||
function quote(value: string): string {
|
||||
return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`
|
||||
}
|
||||
|
||||
function splitAtFirstPipe(query: string): [string, string] {
|
||||
let quoteChar = ""
|
||||
for (let i = 0; i < query.length; i++) {
|
||||
const c = query[i]
|
||||
if (quoteChar) {
|
||||
if (c === "\\") i++
|
||||
else if (c === quoteChar) quoteChar = ""
|
||||
} else if (c === '"' || c === "'" || c === "`") {
|
||||
quoteChar = c
|
||||
} else if (c === "|") {
|
||||
return [query.slice(0, i), query.slice(i + 1)]
|
||||
}
|
||||
}
|
||||
return [query, ""]
|
||||
}
|
||||
|
||||
// The picker narrows the filter part only: AND binds tighter than OR in
|
||||
// LogsQL, and the user's pipes must still run after the narrowed filter.
|
||||
export function buildLogsQuery({ filter, pod, container }: { filter: string; pod?: string; container?: string }): string {
|
||||
const trimmed = filter.trim() || "*"
|
||||
const matchers = [
|
||||
pod ? `${POD_FIELD}=${quote(pod)}` : "",
|
||||
container ? `${CONTAINER_FIELD}=${quote(container)}` : "",
|
||||
].filter(Boolean)
|
||||
if (matchers.length === 0) return trimmed
|
||||
const [head, pipes] = splitAtFirstPipe(trimmed)
|
||||
const scoped = `{${matchers.join(",")}} (${head.trim() || "*"})`
|
||||
return pipes.trim() ? `${scoped} | ${pipes.trim()}` : scoped
|
||||
}
|
||||
|
||||
// VictoriaLogs trims trailing zeros from the fraction, so the strings do not sort lexically.
|
||||
function padFraction(time: string): string {
|
||||
const m = /^([^.Z]+)(?:\.(\d+))?Z$/.exec(time)
|
||||
return m ? `${m[1]}.${(m[2] ?? "").padEnd(9, "0")}Z` : time
|
||||
}
|
||||
|
||||
export function compareTime(a: string, b: string): number {
|
||||
const x = padFraction(a)
|
||||
const y = padFraction(b)
|
||||
return x < y ? -1 : x > y ? 1 : 0
|
||||
}
|
||||
|
||||
// Null when a pipe removed either identity field: such a row cannot be told from another stream's.
|
||||
const entryKey = (e: LogEntry) =>
|
||||
e._stream_id !== undefined && e._time !== undefined ? `${e._stream_id}\u0000${e._time}` : null
|
||||
|
||||
// (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to
|
||||
// lines a resumed tail replays at or before the resume point, and only to
|
||||
// lines that still carry both fields.
|
||||
export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] {
|
||||
let fresh = incoming
|
||||
if (resumedFrom) {
|
||||
const seen = new Set(buffer.map(entryKey).filter((k) => k !== null))
|
||||
fresh = incoming.filter((e) => {
|
||||
const key = entryKey(e)
|
||||
return key === null || compareTime(e._time ?? "", resumedFrom) > 0 || !seen.has(key)
|
||||
})
|
||||
}
|
||||
const merged = buffer.concat(fresh)
|
||||
return merged.length > cap ? merged.slice(merged.length - cap) : merged
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
import { describe, it, expect } from "vitest"
|
||||
import {
|
||||
METRIC_PANELS,
|
||||
formatBytes,
|
||||
formatCores,
|
||||
rangeWindow,
|
||||
toChartData,
|
||||
} from "./metrics.ts"
|
||||
|
||||
describe("rangeWindow", () => {
|
||||
it("ends now and spans the range in about 240 points", () => {
|
||||
const w = rangeWindow("6h", 1_000_000)
|
||||
|
||||
expect(w.end).toBe(1_000_000)
|
||||
expect(w.start).toBe(1_000_000 - 6 * 3600)
|
||||
expect(w.step).toBe(90)
|
||||
})
|
||||
|
||||
it("never steps finer than 15s", () => {
|
||||
expect(rangeWindow("1h", 0).step).toBe(15)
|
||||
})
|
||||
})
|
||||
|
||||
describe("METRIC_PANELS", () => {
|
||||
const window = { start: 0, end: 3600, step: 15 }
|
||||
|
||||
it("rates CPU over at least five minutes so sparse scrapes still draw", () => {
|
||||
const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries(window)
|
||||
|
||||
expect(query.expr).toBe(
|
||||
'sum by (pod) (rate(container_cpu_usage_seconds_total{container!="",container!="POD"}[300s]))',
|
||||
)
|
||||
})
|
||||
|
||||
// Long ranges are served by the longterm store, deduplicated at 5m, where a
|
||||
// narrower window can hold no samples and the rate comes back empty.
|
||||
it.each([
|
||||
[15, "300s"],
|
||||
[75, "300s"],
|
||||
[90, "360s"],
|
||||
[2520, "10080s"],
|
||||
])("rates over max(5m, 4 x step) for step %is", (step, window) => {
|
||||
for (const panel of METRIC_PANELS) {
|
||||
for (const q of panel.queries({ start: 0, end: 1, step })) {
|
||||
if (q.expr.includes("rate(")) expect(q.expr).toContain(`[${window}]`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it("reads memory as working set per pod", () => {
|
||||
const [query] = METRIC_PANELS.find((p) => p.id === "memory")!.queries(window)
|
||||
|
||||
expect(query.expr).toBe('sum by (pod) (container_memory_working_set_bytes{container!="",container!="POD"})')
|
||||
})
|
||||
|
||||
it("draws network receive and transmit as two named series", () => {
|
||||
const queries = METRIC_PANELS.find((p) => p.id === "network")!.queries(window)
|
||||
|
||||
expect(queries.map((q) => q.label)).toEqual(["receive", "transmit"])
|
||||
expect(queries[0].expr).toBe("sum(rate(container_network_receive_bytes_total[300s]))")
|
||||
expect(queries[1].expr).toBe("sum(rate(container_network_transmit_bytes_total[300s]))")
|
||||
})
|
||||
|
||||
it("names no application kind in any query", () => {
|
||||
for (const panel of METRIC_PANELS) {
|
||||
for (const q of panel.queries(window)) expect(q.expr).not.toMatch(/apps_cozystack_io|namespace=/)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe("toChartData", () => {
|
||||
it("aligns series on the union of timestamps and leaves gaps as null", () => {
|
||||
const data = toChartData([
|
||||
{ label: "a", series: [{ metric: {}, values: [[10, "1"], [20, "2"]] }] },
|
||||
{ label: "b", series: [{ metric: {}, values: [[20, "5"], [30, "6"]] }] },
|
||||
])
|
||||
|
||||
expect(data.timestamps).toEqual([10, 20, 30])
|
||||
expect(data.series).toEqual([
|
||||
{ label: "a", values: [1, 2, null] },
|
||||
{ label: "b", values: [null, 5, 6] },
|
||||
])
|
||||
})
|
||||
|
||||
it("labels each series by its pod, falling back to the query label", () => {
|
||||
const data = toChartData([
|
||||
{
|
||||
label: "cpu",
|
||||
series: [
|
||||
{ metric: { pod: "db-1" }, values: [[1, "0.5"]] },
|
||||
{ metric: {}, values: [[1, "0.1"]] },
|
||||
],
|
||||
},
|
||||
])
|
||||
|
||||
expect(data.series.map((s) => s.label)).toEqual(["db-1", "cpu"])
|
||||
})
|
||||
|
||||
it("drops NaN samples instead of drawing them as zero", () => {
|
||||
const data = toChartData([{ label: "x", series: [{ metric: {}, values: [[1, "NaN"]] }] }])
|
||||
|
||||
expect(data.series[0].values).toEqual([null])
|
||||
})
|
||||
})
|
||||
|
||||
describe("formatters", () => {
|
||||
it("formats CPU in millicores below one core", () => {
|
||||
expect(formatCores(0.25)).toBe("250m")
|
||||
expect(formatCores(2)).toBe("2")
|
||||
})
|
||||
|
||||
it("formats bytes with binary units", () => {
|
||||
expect(formatBytes(512)).toBe("512 B")
|
||||
expect(formatBytes(3 * 1024 * 1024)).toBe("3.0 MiB")
|
||||
})
|
||||
})
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
import type { PromSeries } from "./proxy.ts"
|
||||
|
||||
export const RANGES = {
|
||||
"1h": 3600,
|
||||
"6h": 6 * 3600,
|
||||
"24h": 24 * 3600,
|
||||
"7d": 7 * 24 * 3600,
|
||||
} as const
|
||||
|
||||
export type RangeKey = keyof typeof RANGES
|
||||
|
||||
export interface RangeWindow {
|
||||
start: number
|
||||
end: number
|
||||
step: number
|
||||
}
|
||||
|
||||
const POINTS = 240
|
||||
const MIN_STEP = 15
|
||||
|
||||
export function rangeWindow(range: RangeKey, nowSeconds: number): RangeWindow {
|
||||
const span = RANGES[range]
|
||||
return { start: nowSeconds - span, end: nowSeconds, step: Math.max(MIN_STEP, Math.round(span / POINTS)) }
|
||||
}
|
||||
|
||||
// The longterm store keeps one sample per 5m, so a narrower window can be empty;
|
||||
// 4 x step also keeps samples between points from being skipped.
|
||||
function rateWindow(step: number): string {
|
||||
return `${Math.max(300, step * 4)}s`
|
||||
}
|
||||
|
||||
const CONTAINERS = 'container!="",container!="POD"'
|
||||
|
||||
export interface MetricQuery {
|
||||
label: string
|
||||
expr: string
|
||||
}
|
||||
|
||||
export interface MetricPanel {
|
||||
id: "cpu" | "memory" | "network"
|
||||
title: string
|
||||
format: (value: number) => string
|
||||
queries: (window: RangeWindow) => MetricQuery[]
|
||||
}
|
||||
|
||||
export function formatCores(value: number): string {
|
||||
if (value === 0) return "0"
|
||||
if (Math.abs(value) < 1) return `${Math.round(value * 1000)}m`
|
||||
return `${Number(value.toFixed(2))}`
|
||||
}
|
||||
|
||||
const BYTE_UNITS = ["B", "KiB", "MiB", "GiB", "TiB"]
|
||||
|
||||
export function formatBytes(value: number): string {
|
||||
let v = value
|
||||
let unit = 0
|
||||
while (Math.abs(v) >= 1024 && unit < BYTE_UNITS.length - 1) {
|
||||
v /= 1024
|
||||
unit++
|
||||
}
|
||||
return unit === 0 ? `${Math.round(v)} B` : `${v.toFixed(1)} ${BYTE_UNITS[unit]}`
|
||||
}
|
||||
|
||||
// The proxy injects the namespace or application scope, so these stay kind-agnostic.
|
||||
export const METRIC_PANELS: MetricPanel[] = [
|
||||
{
|
||||
id: "cpu",
|
||||
title: "CPU (cores)",
|
||||
format: formatCores,
|
||||
queries: ({ step }) => [
|
||||
{
|
||||
label: "cpu",
|
||||
expr: `sum by (pod) (rate(container_cpu_usage_seconds_total{${CONTAINERS}}[${rateWindow(step)}]))`,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "memory",
|
||||
title: "Memory (working set)",
|
||||
format: formatBytes,
|
||||
queries: () => [
|
||||
{ label: "memory", expr: `sum by (pod) (container_memory_working_set_bytes{${CONTAINERS}})` },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "network",
|
||||
title: "Network",
|
||||
format: (v) => `${formatBytes(v)}/s`,
|
||||
queries: ({ step }) => [
|
||||
{ label: "receive", expr: `sum(rate(container_network_receive_bytes_total[${rateWindow(step)}]))` },
|
||||
{ label: "transmit", expr: `sum(rate(container_network_transmit_bytes_total[${rateWindow(step)}]))` },
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
export function formatSampleTime(seconds: number): string {
|
||||
return new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit" })
|
||||
}
|
||||
|
||||
export interface ChartSeries {
|
||||
label: string
|
||||
values: (number | null)[]
|
||||
}
|
||||
|
||||
export interface ChartData {
|
||||
timestamps: number[]
|
||||
series: ChartSeries[]
|
||||
}
|
||||
|
||||
export function toChartData(results: { label: string; series: PromSeries[] }[]): ChartData {
|
||||
const flat = results.flatMap(({ label, series }) =>
|
||||
series.map((s) => ({ label: s.metric.pod ?? label, points: new Map(s.values.map(([t, v]) => [t, Number(v)])) })),
|
||||
)
|
||||
const timestamps = [...new Set(flat.flatMap((s) => [...s.points.keys()]))].sort((a, b) => a - b)
|
||||
return {
|
||||
timestamps,
|
||||
series: flat.map(({ label, points }) => ({
|
||||
label,
|
||||
values: timestamps.map((t) => {
|
||||
const v = points.get(t)
|
||||
return v === undefined || Number.isNaN(v) ? null : v
|
||||
}),
|
||||
})),
|
||||
}
|
||||
}
|
||||
+242
@@ -0,0 +1,242 @@
|
||||
import { describe, it, expect, vi } from "vitest"
|
||||
import { K8sClient } from "@cozystack/k8s-client"
|
||||
import {
|
||||
ProxyError,
|
||||
errorKindForStatus,
|
||||
fieldValues,
|
||||
isScopeWide,
|
||||
logsRoot,
|
||||
metricsRoot,
|
||||
queryLogs,
|
||||
queryRange,
|
||||
tailLogs,
|
||||
} from "./proxy.ts"
|
||||
|
||||
const app = { namespace: "tenant-a", plural: "postgreses", name: "db" }
|
||||
const ns = { namespace: "tenant-a" }
|
||||
|
||||
function clientReturning(...responses: Response[]) {
|
||||
const client = new K8sClient({ onUnauthorized: () => {} })
|
||||
const raw = vi.spyOn(client, "raw")
|
||||
for (const r of responses) raw.mockResolvedValueOnce(r)
|
||||
return { client, raw }
|
||||
}
|
||||
|
||||
function calledUrl(raw: ReturnType<typeof clientReturning>["raw"], call = 0): URL {
|
||||
return new URL(raw.mock.calls[call][0], "http://x")
|
||||
}
|
||||
|
||||
describe("roots", () => {
|
||||
it("builds the application roots under apps.cozystack.io", () => {
|
||||
expect(metricsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/")
|
||||
expect(logsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/")
|
||||
})
|
||||
|
||||
it("builds the namespace roots under cozystack.io", () => {
|
||||
expect(metricsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/")
|
||||
expect(logsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/")
|
||||
})
|
||||
})
|
||||
|
||||
describe("errorKindForStatus", () => {
|
||||
it.each([
|
||||
[400, "bad-request"],
|
||||
[401, "unauthorized"],
|
||||
[403, "forbidden"],
|
||||
[404, "not-available"],
|
||||
[422, "limit"],
|
||||
[429, "rate-limited"],
|
||||
[502, "store-unreachable"],
|
||||
[503, "unavailable"],
|
||||
[504, "timeout"],
|
||||
[500, "unknown"],
|
||||
])("maps %i to %s", (status, kind) => {
|
||||
expect(errorKindForStatus(status)).toBe(kind)
|
||||
})
|
||||
})
|
||||
|
||||
describe("queryRange", () => {
|
||||
it("GETs query_range under the root and returns the matrix", async () => {
|
||||
const body = { status: "success", data: { resultType: "matrix", result: [{ metric: { pod: "p" }, values: [[1, "2"]] }] } }
|
||||
const { client, raw } = clientReturning(Response.json(body))
|
||||
|
||||
const result = await queryRange(client, metricsRoot(app), { query: "up", start: 10, end: 20, step: 5 })
|
||||
|
||||
const url = calledUrl(raw)
|
||||
expect(url.pathname).toBe(`${metricsRoot(app)}api/v1/query_range`)
|
||||
expect(Object.fromEntries(url.searchParams)).toEqual({ query: "up", start: "10", end: "20", step: "5" })
|
||||
expect(result).toEqual(body.data.result)
|
||||
})
|
||||
|
||||
it("throws a ProxyError carrying the Prometheus error text", async () => {
|
||||
const { client } = clientReturning(
|
||||
Response.json({ status: "error", error: "range too long" }, { status: 422 }),
|
||||
)
|
||||
|
||||
const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
|
||||
|
||||
expect(err).toBeInstanceOf(ProxyError)
|
||||
expect(err).toMatchObject({ kind: "limit", status: 422, message: "range too long" })
|
||||
})
|
||||
|
||||
it("treats a Kubernetes Status 403 as a cluster without the proxy route", async () => {
|
||||
const status = { kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403, message: "logs.cozystack.io \"select\" is forbidden" }
|
||||
const { client } = clientReturning(Response.json(status, { status: 403 }))
|
||||
|
||||
const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
|
||||
|
||||
expect(err).toMatchObject({ kind: "not-available", status: 403 })
|
||||
})
|
||||
|
||||
it("keeps the proxy's own plain-text 403 as forbidden", async () => {
|
||||
const { client } = clientReturning(new Response("forbidden\n", { status: 403 }))
|
||||
|
||||
const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
|
||||
|
||||
expect(err).toMatchObject({ kind: "forbidden" })
|
||||
})
|
||||
|
||||
it("reports a fetch failure as a network error", async () => {
|
||||
const client = new K8sClient()
|
||||
vi.spyOn(client, "raw").mockRejectedValue(new TypeError("offline"))
|
||||
|
||||
const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
|
||||
|
||||
expect(err).toMatchObject({ kind: "network" })
|
||||
})
|
||||
})
|
||||
|
||||
describe("queryLogs", () => {
|
||||
it("parses NDJSON lines and sorts them by _time ascending", async () => {
|
||||
const ndjson = [
|
||||
JSON.stringify({ _time: "2026-11-04T10:00:02Z", _msg: "b" }),
|
||||
"",
|
||||
JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: "a" }),
|
||||
].join("\n")
|
||||
const { client, raw } = clientReturning(new Response(ndjson))
|
||||
|
||||
const entries = await queryLogs(client, logsRoot(ns), { query: "*", start: "s", end: "e", limit: 100 })
|
||||
|
||||
expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/query`)
|
||||
expect(Object.fromEntries(calledUrl(raw).searchParams)).toEqual({ query: "*", start: "s", end: "e", limit: "100" })
|
||||
expect(entries.map((e) => e._msg)).toEqual(["a", "b"])
|
||||
})
|
||||
|
||||
it("keeps rows a pipe stripped of _time, in the order the store sent them", async () => {
|
||||
const ndjson = ['{"_msg":"visible message"}', '{"_msg":"second"}', "[1]", "null", "{torn"].join("\n")
|
||||
const { client } = clientReturning(new Response(ndjson))
|
||||
|
||||
const entries = await queryLogs(client, logsRoot(ns), { query: "* | fields _msg", start: "s", end: "e", limit: 100 })
|
||||
|
||||
expect(entries).toEqual([{ _msg: "visible message" }, { _msg: "second" }])
|
||||
})
|
||||
|
||||
it("keeps a plain-text error body as the message", async () => {
|
||||
const { client } = clientReturning(new Response("cannot parse query", { status: 400 }))
|
||||
|
||||
const err = await queryLogs(client, logsRoot(ns), { query: "(", start: "s", end: "e", limit: 1 }).catch((e) => e)
|
||||
|
||||
expect(err).toMatchObject({ kind: "bad-request", message: "cannot parse query" })
|
||||
})
|
||||
})
|
||||
|
||||
describe("fieldValues", () => {
|
||||
it("uses stream_field_values for stream fields and returns the values", async () => {
|
||||
const { client, raw } = clientReturning(
|
||||
Response.json({ values: [{ value: "pod-b", hits: 1 }, { value: "pod-a", hits: 9 }] }),
|
||||
)
|
||||
|
||||
const values = await fieldValues(client, logsRoot(app), {
|
||||
field: "kubernetes_pod_name",
|
||||
stream: true,
|
||||
query: "*",
|
||||
start: "s",
|
||||
end: "e",
|
||||
})
|
||||
|
||||
expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/stream_field_values`)
|
||||
expect(calledUrl(raw).searchParams.get("field")).toBe("kubernetes_pod_name")
|
||||
expect(values).toEqual(["pod-a", "pod-b"])
|
||||
})
|
||||
|
||||
it("uses field_values for other fields", async () => {
|
||||
const { client, raw } = clientReturning(Response.json({ values: [] }))
|
||||
|
||||
await fieldValues(client, logsRoot(app), { field: "level", stream: false, query: "*", start: "s", end: "e" })
|
||||
|
||||
expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/field_values`)
|
||||
})
|
||||
})
|
||||
|
||||
describe("tailLogs", () => {
|
||||
it("delivers every line as it arrives, including one split across chunks", async () => {
|
||||
const enc = new TextEncoder()
|
||||
const chunks = ['{"_time":"t1","_msg":"a"}\n{"_ti', 'me":"t2","_msg":"b"}\n']
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(c) {
|
||||
for (const ch of chunks) c.enqueue(enc.encode(ch))
|
||||
c.close()
|
||||
},
|
||||
})
|
||||
const { client, raw } = clientReturning(new Response(body))
|
||||
const seen: (string | undefined)[] = []
|
||||
|
||||
await tailLogs(client, logsRoot(ns), { query: "*", since: "t0" }, (e) => seen.push(e._msg))
|
||||
|
||||
expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/tail`)
|
||||
expect(calledUrl(raw).searchParams.get("since")).toBe("t0")
|
||||
expect(seen).toEqual(["a", "b"])
|
||||
})
|
||||
|
||||
it("delivers a line without _time", async () => {
|
||||
const { client } = clientReturning(new Response('{"_msg":"untimed"}\n'))
|
||||
const seen: (string | undefined)[] = []
|
||||
|
||||
await tailLogs(client, logsRoot(ns), { query: "* | fields _msg" }, (e) => seen.push(e._msg))
|
||||
|
||||
expect(seen).toEqual(["untimed"])
|
||||
})
|
||||
|
||||
it("omits since on the first connect", async () => {
|
||||
const { client, raw } = clientReturning(new Response(""))
|
||||
|
||||
await tailLogs(client, logsRoot(ns), { query: "*" }, () => {})
|
||||
|
||||
expect(calledUrl(raw).searchParams.has("since")).toBe(false)
|
||||
})
|
||||
|
||||
it("throws a ProxyError when the tail is refused", async () => {
|
||||
const { client } = clientReturning(new Response("", { status: 403 }))
|
||||
|
||||
const err = await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}).catch((e) => e)
|
||||
|
||||
expect(err).toMatchObject({ kind: "forbidden" })
|
||||
})
|
||||
|
||||
it("says the stream opened before any line arrives", async () => {
|
||||
const { client } = clientReturning(new Response(""))
|
||||
const onOpen = vi.fn()
|
||||
|
||||
await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}, undefined, onOpen)
|
||||
|
||||
expect(onOpen).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("does not say the stream opened when the tail is refused", async () => {
|
||||
const { client } = clientReturning(new Response("", { status: 429 }))
|
||||
const onOpen = vi.fn()
|
||||
|
||||
await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}, undefined, onOpen).catch(() => {})
|
||||
|
||||
expect(onOpen).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe("isScopeWide", () => {
|
||||
it("is true for answers that do not depend on the query", () => {
|
||||
expect(isScopeWide(new ProxyError("not-available", 404, ""))).toBe(true)
|
||||
expect(isScopeWide(new ProxyError("forbidden", 403, ""))).toBe(true)
|
||||
expect(isScopeWide(new ProxyError("limit", 422, ""))).toBe(false)
|
||||
expect(isScopeWide(new Error("x"))).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,228 @@
|
||||
import type { K8sClient } from "@cozystack/k8s-client"
|
||||
import { compareTime } from "./logs.ts"
|
||||
|
||||
export type ObservabilityScope =
|
||||
| { namespace: string }
|
||||
| { namespace: string; plural: string; name: string }
|
||||
|
||||
function appBase(scope: ObservabilityScope): string | null {
|
||||
if (!("plural" in scope)) return null
|
||||
return `/apis/apps.cozystack.io/v1alpha1/namespaces/${scope.namespace}/${scope.plural}/${scope.name}`
|
||||
}
|
||||
|
||||
export function metricsRoot(scope: ObservabilityScope): string {
|
||||
const app = appBase(scope)
|
||||
return app ? `${app}/metrics/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/metrics/`
|
||||
}
|
||||
|
||||
export function logsRoot(scope: ObservabilityScope): string {
|
||||
const app = appBase(scope)
|
||||
return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/`
|
||||
}
|
||||
|
||||
export type ProxyErrorKind =
|
||||
| "bad-request"
|
||||
| "unauthorized"
|
||||
| "forbidden"
|
||||
| "not-available"
|
||||
| "limit"
|
||||
| "rate-limited"
|
||||
| "store-unreachable"
|
||||
| "unavailable"
|
||||
| "timeout"
|
||||
| "network"
|
||||
| "unknown"
|
||||
|
||||
const KIND_BY_STATUS: Record<number, ProxyErrorKind> = {
|
||||
400: "bad-request",
|
||||
401: "unauthorized",
|
||||
403: "forbidden",
|
||||
404: "not-available",
|
||||
422: "limit",
|
||||
429: "rate-limited",
|
||||
502: "store-unreachable",
|
||||
503: "unavailable",
|
||||
504: "timeout",
|
||||
}
|
||||
|
||||
export function errorKindForStatus(status: number): ProxyErrorKind {
|
||||
return KIND_BY_STATUS[status] ?? "unknown"
|
||||
}
|
||||
|
||||
export class ProxyError extends Error {
|
||||
kind: ProxyErrorKind
|
||||
status: number
|
||||
|
||||
constructor(kind: ProxyErrorKind, status: number, message: string) {
|
||||
super(message)
|
||||
this.name = "ProxyError"
|
||||
this.kind = kind
|
||||
this.status = status
|
||||
}
|
||||
}
|
||||
|
||||
// These answers hold for every query in the scope, so one notice replaces the whole view.
|
||||
export function isScopeWide(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof ProxyError &&
|
||||
(error.kind === "not-available" || error.kind === "forbidden" || error.kind === "unauthorized")
|
||||
)
|
||||
}
|
||||
|
||||
// The proxy answers in plain text; a Kubernetes Status body means nginx sent
|
||||
// the path to kube-apiserver, which RBAC-denies or 404s a route it does not serve.
|
||||
function isKubernetesStatus(text: string): boolean {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(text)
|
||||
return !!parsed && typeof parsed === "object" && "kind" in parsed && parsed.kind === "Status"
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text.
|
||||
function errorMessage(text: string, status: number): string {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(text)
|
||||
if (parsed && typeof parsed === "object" && "error" in parsed && typeof parsed.error === "string") {
|
||||
return parsed.error
|
||||
}
|
||||
} catch {
|
||||
// not JSON
|
||||
}
|
||||
const trimmed = text.trim()
|
||||
return trimmed && trimmed.length < 300 && !trimmed.startsWith("<") ? trimmed : `Server returned ${status}`
|
||||
}
|
||||
|
||||
async function proxyGet(
|
||||
client: K8sClient,
|
||||
path: string,
|
||||
params: Record<string, string | number | undefined>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Response> {
|
||||
const search = new URLSearchParams()
|
||||
for (const [k, v] of Object.entries(params)) if (v !== undefined) search.set(k, String(v))
|
||||
let res: Response
|
||||
try {
|
||||
res = await client.raw(`${path}?${search}`, { signal })
|
||||
} catch (err) {
|
||||
if (signal?.aborted) throw err
|
||||
throw new ProxyError("network", 0, "Unable to connect to the server")
|
||||
}
|
||||
if (!res.ok) {
|
||||
const text = await res.text().catch(() => "")
|
||||
const kind = isKubernetesStatus(text) ? "not-available" : errorKindForStatus(res.status)
|
||||
throw new ProxyError(kind, res.status, errorMessage(text, res.status))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
export interface PromSeries {
|
||||
metric: Record<string, string>
|
||||
values: [number, string][]
|
||||
}
|
||||
|
||||
export interface RangeParams {
|
||||
query: string
|
||||
start: number
|
||||
end: number
|
||||
step: number
|
||||
}
|
||||
|
||||
export async function queryRange(
|
||||
client: K8sClient,
|
||||
root: string,
|
||||
params: RangeParams,
|
||||
signal?: AbortSignal,
|
||||
): Promise<PromSeries[]> {
|
||||
const res = await proxyGet(client, `${root}api/v1/query_range`, { ...params }, signal)
|
||||
const body = (await res.json()) as { data?: { result?: PromSeries[] } }
|
||||
return body.data?.result ?? []
|
||||
}
|
||||
|
||||
// A pipe such as `fields _msg` or `stats` can drop _time and _msg.
|
||||
export interface LogEntry {
|
||||
_time?: string
|
||||
_msg?: string
|
||||
_stream_id?: string
|
||||
[field: string]: string | undefined
|
||||
}
|
||||
|
||||
function parseLine(line: string): LogEntry | null {
|
||||
if (!line.trim()) return null
|
||||
try {
|
||||
const value: unknown = JSON.parse(line)
|
||||
if (value && typeof value === "object" && !Array.isArray(value)) return value as LogEntry
|
||||
} catch {
|
||||
// a torn or non-JSON line carries nothing to show
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export interface LogsQueryParams {
|
||||
query: string
|
||||
start: string
|
||||
end: string
|
||||
limit: number
|
||||
}
|
||||
|
||||
export async function queryLogs(
|
||||
client: K8sClient,
|
||||
root: string,
|
||||
params: LogsQueryParams,
|
||||
signal?: AbortSignal,
|
||||
): Promise<LogEntry[]> {
|
||||
const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal)
|
||||
const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null)
|
||||
if (entries.some((e) => e._time === undefined)) return entries
|
||||
return entries.sort((a, b) => compareTime(a._time ?? "", b._time ?? ""))
|
||||
}
|
||||
|
||||
export interface FieldValuesParams {
|
||||
field: string
|
||||
stream: boolean
|
||||
query: string
|
||||
start: string
|
||||
end: string
|
||||
}
|
||||
|
||||
export async function fieldValues(
|
||||
client: K8sClient,
|
||||
root: string,
|
||||
{ stream, ...params }: FieldValuesParams,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string[]> {
|
||||
const endpoint = stream ? "stream_field_values" : "field_values"
|
||||
const res = await proxyGet(client, `${root}select/logsql/${endpoint}`, { ...params }, signal)
|
||||
const body = (await res.json()) as { values?: { value: string }[] }
|
||||
return (body.values ?? []).map((v) => v.value).sort()
|
||||
}
|
||||
|
||||
export async function tailLogs(
|
||||
client: K8sClient,
|
||||
root: string,
|
||||
params: { query: string; since?: string },
|
||||
onEntry: (entry: LogEntry) => void,
|
||||
signal?: AbortSignal,
|
||||
onOpen?: () => void,
|
||||
): Promise<void> {
|
||||
const res = await proxyGet(client, `${root}select/logsql/tail`, params, signal)
|
||||
onOpen?.()
|
||||
if (!res.body) return
|
||||
const reader = res.body.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ""
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const lines = buffer.split("\n")
|
||||
buffer = lines.pop() ?? ""
|
||||
for (const line of lines) {
|
||||
const entry = parseLine(line)
|
||||
if (entry) onEntry(entry)
|
||||
}
|
||||
}
|
||||
const last = parseLine(buffer)
|
||||
if (last) onEntry(last)
|
||||
}
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
import { describe, it, expect, vi } from "vitest"
|
||||
import { renderHook, waitFor } from "@testing-library/react"
|
||||
import type { ReactNode } from "react"
|
||||
import { K8sClient, K8sProvider } from "@cozystack/k8s-client"
|
||||
import { useLogTail } from "./use-log-tail.ts"
|
||||
|
||||
const line = (time: string, msg: string) => JSON.stringify({ _time: time, _msg: msg, _stream_id: "s" })
|
||||
|
||||
function ndjson(...lines: string[]): Response {
|
||||
return new Response(lines.map((l) => `${l}\n`).join(""))
|
||||
}
|
||||
|
||||
function openStream(): Response {
|
||||
return new Response(new ReadableStream<Uint8Array>({ start() {} }))
|
||||
}
|
||||
|
||||
function setup(...responses: Response[]) {
|
||||
const client = new K8sClient()
|
||||
const raw = vi.spyOn(client, "raw")
|
||||
for (const r of responses) raw.mockResolvedValueOnce(r)
|
||||
raw.mockImplementation(() => new Promise(() => {}))
|
||||
const wrapper = ({ children }: { children: ReactNode }) => <K8sProvider client={client}>{children}</K8sProvider>
|
||||
return { raw, wrapper }
|
||||
}
|
||||
|
||||
const T1 = "2026-11-04T10:00:01Z"
|
||||
const T2 = "2026-11-04T10:00:02Z"
|
||||
const T3 = "2026-11-04T10:00:03Z"
|
||||
|
||||
describe("useLogTail", () => {
|
||||
it("streams lines and resumes after a disconnect from the last _time without duplicates", async () => {
|
||||
const { raw, wrapper } = setup(ndjson(line(T1, "a"), line(T2, "b")), ndjson(line(T2, "b"), line(T3, "c")))
|
||||
|
||||
const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper })
|
||||
|
||||
await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a", "b", "c"]))
|
||||
expect(new URL(raw.mock.calls[0][0], "http://x").searchParams.has("since")).toBe(false)
|
||||
expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T2)
|
||||
})
|
||||
|
||||
it("stops and reports an error the proxy will repeat on every retry", async () => {
|
||||
const { raw, wrapper } = setup(new Response("", { status: 403 }))
|
||||
|
||||
const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper })
|
||||
|
||||
await waitFor(() => expect(result.current.state).toBe("stopped"))
|
||||
expect(result.current.error).toMatchObject({ kind: "forbidden" })
|
||||
expect(raw).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("starts a new follow session with an empty buffer", async () => {
|
||||
const { wrapper } = setup(ndjson(line(T1, "first")), ndjson(line(T3, "second")))
|
||||
const { result, rerender } = renderHook(
|
||||
({ session }: { session: number | null }) => useLogTail("/root/", "*", session, { retryDelayMs: 60_000 }),
|
||||
{ wrapper, initialProps: { session: 1 as number | null } },
|
||||
)
|
||||
await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["first"]))
|
||||
|
||||
rerender({ session: null })
|
||||
rerender({ session: 2 })
|
||||
|
||||
await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["second"]))
|
||||
})
|
||||
|
||||
it("resumes from the newest line seen, not the last one received", async () => {
|
||||
const { raw, wrapper } = setup(ndjson(line(T3, "c"), line(T1, "a")), ndjson())
|
||||
|
||||
renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
|
||||
|
||||
await waitFor(() => expect(raw.mock.calls.length).toBeGreaterThan(1))
|
||||
expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T3)
|
||||
})
|
||||
|
||||
it("retries a tail refused for load instead of stopping", async () => {
|
||||
const { wrapper } = setup(new Response("too many concurrent requests", { status: 429 }), ndjson(line(T1, "a")))
|
||||
|
||||
const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
|
||||
|
||||
await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a"]))
|
||||
})
|
||||
|
||||
it("goes live and clears a load warning once the stream opens, before any line", async () => {
|
||||
const { wrapper } = setup(new Response("too many concurrent requests", { status: 429 }), openStream())
|
||||
|
||||
const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
|
||||
|
||||
await waitFor(() => expect(result.current.state).toBe("live"))
|
||||
expect(result.current.error).toBeNull()
|
||||
})
|
||||
|
||||
it("reconnects after an idle close at the base delay, from when the quiet stream opened", async () => {
|
||||
const delays: number[] = []
|
||||
const realSetTimeout = globalThis.setTimeout
|
||||
const spy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((fn: () => void, ms?: number) => {
|
||||
if (ms !== undefined && ms % 7 === 0 && ms > 0) delays.push(ms)
|
||||
return realSetTimeout(fn, ms)
|
||||
}) as typeof setTimeout)
|
||||
const refused = () => new Response("", { status: 429 })
|
||||
const { raw, wrapper } = setup(refused(), refused(), ndjson())
|
||||
|
||||
try {
|
||||
renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 7 }), { wrapper })
|
||||
await waitFor(() => expect(raw).toHaveBeenCalledTimes(4))
|
||||
} finally {
|
||||
spy.mockRestore()
|
||||
}
|
||||
|
||||
expect(delays).toEqual([7, 14, 7])
|
||||
expect(new URL(raw.mock.calls[2][0], "http://x").searchParams.has("since")).toBe(false)
|
||||
const since = new URL(raw.mock.calls[3][0], "http://x").searchParams.get("since")
|
||||
expect(since).not.toBeNull()
|
||||
expect(new Date(since!).toISOString()).toBe(since)
|
||||
})
|
||||
|
||||
it("does not connect while disabled", () => {
|
||||
const { raw, wrapper } = setup()
|
||||
|
||||
renderHook(() => useLogTail("/root/", "*", null), { wrapper })
|
||||
|
||||
expect(raw).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it("aborts the stream on unmount", async () => {
|
||||
const { raw, wrapper } = setup()
|
||||
const { unmount } = renderHook(() => useLogTail("/root/", "*", 1), { wrapper })
|
||||
await waitFor(() => expect(raw).toHaveBeenCalled())
|
||||
|
||||
unmount()
|
||||
|
||||
const init = raw.mock.calls[0][1] as RequestInit
|
||||
expect(init.signal?.aborted).toBe(true)
|
||||
})
|
||||
})
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
import { useEffect, useState } from "react"
|
||||
import { useK8sClient } from "@cozystack/k8s-client"
|
||||
import { compareTime, mergeTail } from "./logs.ts"
|
||||
import { ProxyError, tailLogs, type LogEntry } from "./proxy.ts"
|
||||
|
||||
export type TailState = "connecting" | "live" | "reconnecting" | "stopped"
|
||||
|
||||
interface TailSnapshot {
|
||||
key: string
|
||||
entries: LogEntry[]
|
||||
state: TailState
|
||||
error: unknown
|
||||
}
|
||||
|
||||
const FLUSH_MS = 100
|
||||
const MAX_RETRY_MS = 30_000
|
||||
// Load refusals retry: the proxy caps rate and concurrent tails with 429 (422 in
|
||||
// its first builds), and a reconnect can race the previous tail's freed slot.
|
||||
const FATAL: ReadonlySet<string> = new Set(["bad-request", "unauthorized", "forbidden", "not-available"])
|
||||
|
||||
const sleep = (ms: number, signal: AbortSignal) =>
|
||||
new Promise<void>((resolve) => {
|
||||
const onAbort = () => {
|
||||
clearTimeout(t)
|
||||
resolve()
|
||||
}
|
||||
const t = setTimeout(() => {
|
||||
signal.removeEventListener("abort", onAbort)
|
||||
resolve()
|
||||
}, ms)
|
||||
signal.addEventListener("abort", onAbort, { once: true })
|
||||
})
|
||||
|
||||
// Each non-null session is one follow run with its own buffer; null stops following.
|
||||
export function useLogTail(
|
||||
root: string,
|
||||
query: string,
|
||||
session: number | null,
|
||||
{ cap = 1000, retryDelayMs = 2000 }: { cap?: number; retryDelayMs?: number } = {},
|
||||
): Omit<TailSnapshot, "key"> {
|
||||
const client = useK8sClient()
|
||||
const key = `${root}\n${query}\n${session}`
|
||||
const [snap, setSnap] = useState<TailSnapshot>({ key, entries: [], state: "connecting", error: null })
|
||||
|
||||
useEffect(() => {
|
||||
if (session === null) return
|
||||
const controller = new AbortController()
|
||||
const { signal } = controller
|
||||
let pending: LogEntry[] = []
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
let resumedFrom: string | undefined
|
||||
let lastTime: string | undefined
|
||||
|
||||
const update = (patch: Partial<TailSnapshot>, batch: LogEntry[] = [], from = resumedFrom) =>
|
||||
setSnap((prev) => {
|
||||
const base = prev.key === key ? prev : { key, entries: [], state: "connecting" as const, error: null }
|
||||
return { ...base, ...patch, entries: batch.length ? mergeTail(base.entries, batch, cap, from) : base.entries }
|
||||
})
|
||||
const flush = () => {
|
||||
clearTimeout(timer)
|
||||
timer = undefined
|
||||
if (pending.length === 0 || signal.aborted) return
|
||||
const batch = pending
|
||||
pending = []
|
||||
update({ state: "live", error: null }, batch)
|
||||
}
|
||||
|
||||
const run = async () => {
|
||||
let failures = 0
|
||||
let openedAt: string | undefined
|
||||
while (!signal.aborted) {
|
||||
resumedFrom = lastTime
|
||||
let delay = retryDelayMs
|
||||
try {
|
||||
await tailLogs(client, root, { query, since: lastTime ?? openedAt }, (entry) => {
|
||||
if (entry._time && (!lastTime || compareTime(entry._time, lastTime) > 0)) lastTime = entry._time
|
||||
pending.push(entry)
|
||||
timer ??= setTimeout(flush, FLUSH_MS)
|
||||
}, signal, () => {
|
||||
failures = 0
|
||||
// With no line yet, the next reconnect resumes from here so the gap is not lost.
|
||||
openedAt = new Date().toISOString()
|
||||
update({ state: "live", error: null })
|
||||
})
|
||||
flush()
|
||||
} catch (err) {
|
||||
flush()
|
||||
if (signal.aborted) return
|
||||
if (err instanceof ProxyError && FATAL.has(err.kind)) {
|
||||
update({ state: "stopped", error: err })
|
||||
return
|
||||
}
|
||||
update({ error: err })
|
||||
delay = Math.min(retryDelayMs * 2 ** failures, MAX_RETRY_MS)
|
||||
failures++
|
||||
}
|
||||
if (signal.aborted) return
|
||||
update({ state: "reconnecting" })
|
||||
await sleep(delay, signal)
|
||||
}
|
||||
}
|
||||
void run()
|
||||
return () => {
|
||||
controller.abort()
|
||||
clearTimeout(timer)
|
||||
}
|
||||
}, [client, root, query, session, key, cap, retryDelayMs])
|
||||
|
||||
const current = snap.key === key ? snap : { key, entries: [], state: "connecting" as const, error: null }
|
||||
return { entries: current.entries, state: current.state, error: current.error }
|
||||
}
|
||||
+50
@@ -70,4 +70,54 @@ describe("ConsolePage routing", () => {
|
||||
await waitFor(() => expect(client.list).toHaveBeenCalled())
|
||||
expect(screen.queryByText(/Cluster-scoped capacity/i)).toBeNull()
|
||||
})
|
||||
|
||||
it("serves the tenant logs page under the reserved -/ segment", async () => {
|
||||
renderWithK8sProvider(
|
||||
<TenantProvider>
|
||||
<ConsolePage />
|
||||
</TenantProvider>,
|
||||
{ client: makeClient(), initialRoute: "/-/observability/logs" },
|
||||
)
|
||||
|
||||
expect(await screen.findByText(/^Every application in/)).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("leaves /logs to an application kind whose plural is logs", async () => {
|
||||
const client = makeClient()
|
||||
renderWithK8sProvider(
|
||||
<TenantProvider>
|
||||
<ConsolePage />
|
||||
</TenantProvider>,
|
||||
{ client, initialRoute: "/logs" },
|
||||
)
|
||||
|
||||
await waitFor(() => expect(client.list).toHaveBeenCalled())
|
||||
expect(screen.queryByText(/^Every application in/)).toBeNull()
|
||||
})
|
||||
|
||||
// No ApplicationDefinition plural or application name can be "-", so only
|
||||
// routes under it are safe from shadowing an application's own pages.
|
||||
it("leaves /observability/logs to an application named logs of kind observability", async () => {
|
||||
const client = makeClient()
|
||||
vi.mocked(client.list).mockImplementation(async (_g, _v, plural) => ({
|
||||
apiVersion: "v1",
|
||||
kind: `${plural}List`,
|
||||
metadata: {},
|
||||
items:
|
||||
plural === "tenantnamespaces"
|
||||
? [{ apiVersion: "core.cozystack.io/v1alpha1", kind: "TenantNamespace", metadata: { name: "tenant-root" } }]
|
||||
: [],
|
||||
}) as K8sList<unknown>)
|
||||
const get = vi.spyOn(client, "get").mockRejectedValue(new Error("not found"))
|
||||
renderWithK8sProvider(
|
||||
<TenantProvider>
|
||||
<ConsolePage />
|
||||
</TenantProvider>,
|
||||
{ client, initialRoute: "/observability/logs" },
|
||||
)
|
||||
|
||||
expect(await screen.findByText(/^Application/)).toHaveTextContent("Application logs not found.")
|
||||
expect(get).toHaveBeenCalledWith(expect.anything(), expect.anything(), "observability", "logs", expect.anything())
|
||||
expect(screen.queryByText(/^Every application in/)).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -15,6 +15,7 @@ import { MigrationSourceEditPage } from "./MigrationSourceEditPage.tsx"
|
||||
import { MigrationTaskCreatePage } from "./MigrationTaskCreatePage.tsx"
|
||||
import { MigrationTaskDetailPage } from "./MigrationTaskDetailPage.tsx"
|
||||
import { ApplicationOrderPage } from "./ApplicationOrderPage.tsx"
|
||||
import { LogsPage } from "./LogsPage.tsx"
|
||||
|
||||
export function ConsolePage() {
|
||||
return (
|
||||
@@ -96,6 +97,7 @@ export function ConsolePage() {
|
||||
path="migration/vmimporttasks/:name"
|
||||
element={<MigrationTaskDetailPage />}
|
||||
/>
|
||||
<Route path="-/observability/logs" element={<LogsPage />} />
|
||||
<Route path="new/:appName" element={<ApplicationOrderPage />} />
|
||||
<Route path=":plural/:name/edit" element={<ApplicationEditRoute />} />
|
||||
<Route path=":plural/:name/*" element={<ApplicationDetailPage />} />
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, it, expect, vi } from "vitest"
|
||||
import { render, screen } from "@testing-library/react"
|
||||
|
||||
vi.mock("../lib/tenant-context.tsx", () => ({
|
||||
useTenantContext: () => ({ tenantNamespace: "tenant-a" }),
|
||||
}))
|
||||
vi.mock("../components/observability/LogsView.tsx", () => ({
|
||||
LogsView: ({ scope }: { scope: object }) => <pre data-testid="scope">{JSON.stringify(scope)}</pre>,
|
||||
}))
|
||||
|
||||
const { LogsPage } = await import("./LogsPage.tsx")
|
||||
|
||||
describe("LogsPage", () => {
|
||||
it("reads logs from the selected tenant's namespace root", () => {
|
||||
render(<LogsPage />)
|
||||
|
||||
expect(screen.getByRole("heading", { name: "Logs" })).toBeInTheDocument()
|
||||
expect(JSON.parse(screen.getByTestId("scope").textContent ?? "")).toEqual({ namespace: "tenant-a" })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,15 @@
|
||||
import { useTenantContext } from "../lib/tenant-context.tsx"
|
||||
import { LogsView } from "../components/observability/LogsView.tsx"
|
||||
|
||||
export function LogsPage() {
|
||||
const { tenantNamespace } = useTenantContext()
|
||||
return (
|
||||
<div className="flex h-full flex-col">
|
||||
<div className="border-b border-slate-200 bg-white px-6 py-4">
|
||||
<h1 className="text-lg font-semibold text-slate-900">Logs</h1>
|
||||
<p className="text-xs text-slate-500">Every application in {tenantNamespace ?? "this tenant"}</p>
|
||||
</div>
|
||||
{tenantNamespace && <LogsView scope={{ namespace: tenantNamespace }} />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest"
|
||||
import { act, render, screen, waitFor } from "@testing-library/react"
|
||||
import { createMemoryRouter, RouterProvider } from "react-router"
|
||||
import { K8sProvider } from "@cozystack/k8s-client"
|
||||
import { TenantContext, type TenantContextValue } from "../../lib/tenant-context.tsx"
|
||||
import type { ChartData } from "../../lib/observability/metrics.ts"
|
||||
|
||||
// Keep the consoles' and the chart's browser-only libraries out of jsdom; the
|
||||
// Logs and Metrics tabs, the page and the providers stay real.
|
||||
vi.mock("./VncTab.tsx", () => ({ VncTab: () => null }))
|
||||
vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null }))
|
||||
vi.mock("../../components/observability/TimeSeriesChart.tsx", () => ({
|
||||
TimeSeriesChart: ({ data }: { data: ChartData }) => (
|
||||
<div data-testid="chart">{data.series.map((s) => s.label).join(",")}</div>
|
||||
),
|
||||
}))
|
||||
|
||||
const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx")
|
||||
|
||||
const NS = "tenant-test"
|
||||
const APPS = `/apis/apps.cozystack.io/v1alpha1/namespaces/${NS}/postgreses`
|
||||
|
||||
const tenant: TenantContextValue = {
|
||||
tenants: [],
|
||||
selectedTenant: "test",
|
||||
selectTenant: () => {},
|
||||
tenantNamespace: NS,
|
||||
isLoading: false,
|
||||
error: null,
|
||||
}
|
||||
|
||||
const definitions = {
|
||||
items: [{ metadata: { name: "postgres" }, spec: { application: { plural: "postgreses", kind: "Postgres" } } }],
|
||||
}
|
||||
|
||||
const instance = (name: string) => ({
|
||||
apiVersion: "apps.cozystack.io/v1alpha1",
|
||||
kind: "Postgres",
|
||||
metadata: { name, namespace: NS },
|
||||
})
|
||||
|
||||
const logLine = (name: string) =>
|
||||
`${JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: `${name} is ready`, _stream_id: "s", kubernetes_pod_name: `${name}-0`, kubernetes_container_name: "postgres" })}\n`
|
||||
|
||||
const matrix = (name: string) =>
|
||||
Response.json({ status: "success", data: { resultType: "matrix", result: [{ metric: { pod: `${name}-0` }, values: [[1, "1"]] }] } })
|
||||
|
||||
// app-b's own GET never answers, so the page keeps the placeholder of app-a
|
||||
// that the production query defaults hand it; its proxy reads do answer.
|
||||
function handle(url: URL): Response | Promise<Response> {
|
||||
const path = url.pathname
|
||||
if (path.endsWith("/applicationdefinitions")) return Response.json(definitions)
|
||||
if (path === `${APPS}/app-a`) return Response.json(instance("app-a"))
|
||||
if (path === `${APPS}/app-b`) return new Promise(() => {})
|
||||
const app = path.match(/\/postgreses\/(app-[ab])\//)?.[1]
|
||||
if (app && path.endsWith("/select/logsql/query")) return new Response(logLine(app))
|
||||
if (app && path.endsWith("/stream_field_values")) return Response.json({ values: [] })
|
||||
if (app && path.includes("/metrics/api/v1/query_range")) return matrix(app)
|
||||
return Response.json({ items: [] })
|
||||
}
|
||||
|
||||
const fetchMock = vi.fn(async (input: RequestInfo | URL) => handle(new URL(String(input), "http://x")))
|
||||
|
||||
function renderAt(path: string) {
|
||||
const router = createMemoryRouter(
|
||||
[{ path: "/console/:plural/:name/*", element: <ApplicationDetailPage /> }],
|
||||
{ initialEntries: [path] },
|
||||
)
|
||||
// No queryClient: the provider's production defaults keep a query's previous
|
||||
// result while its key changes, which is what exposes the stale instance.
|
||||
render(
|
||||
<K8sProvider>
|
||||
<TenantContext.Provider value={tenant}>
|
||||
<RouterProvider router={router} />
|
||||
</TenantContext.Provider>
|
||||
</K8sProvider>,
|
||||
)
|
||||
return { go: (to: string) => act(() => router.navigate(to)) }
|
||||
}
|
||||
|
||||
const requested = (prefix: string) =>
|
||||
fetchMock.mock.calls.some(([input]) => new URL(String(input), "http://x").pathname.startsWith(prefix))
|
||||
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal("fetch", fetchMock)
|
||||
fetchMock.mockClear()
|
||||
})
|
||||
|
||||
describe("ApplicationDetailPage observability while the next application loads", () => {
|
||||
it("shows the selected application's logs, not the previous one's", async () => {
|
||||
const { go } = renderAt("/console/postgreses/app-a/logs")
|
||||
expect(await screen.findByText("app-a is ready")).toBeInTheDocument()
|
||||
|
||||
await go("/console/postgreses/app-b/logs")
|
||||
|
||||
expect(screen.getByRole("heading", { name: "app-b" })).toBeInTheDocument()
|
||||
expect(screen.queryByText("app-a is ready")).toBeNull()
|
||||
await waitFor(() => expect(requested(`${APPS}/app-b/log/`)).toBe(true))
|
||||
expect(await screen.findByText("app-b is ready")).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it("shows the selected application's metrics, not the previous one's", async () => {
|
||||
const { go } = renderAt("/console/postgreses/app-a/metrics")
|
||||
await waitFor(() => expect(screen.getAllByTestId("chart")[0]).toHaveTextContent("app-a-0"))
|
||||
|
||||
await go("/console/postgreses/app-b/metrics")
|
||||
|
||||
expect(screen.getByRole("heading", { name: "app-b" })).toBeInTheDocument()
|
||||
expect(screen.queryAllByText("app-a-0", { exact: false })).toEqual([])
|
||||
await waitFor(() => expect(requested(`${APPS}/app-b/metrics/`)).toBe(true))
|
||||
await waitFor(() => expect(screen.getAllByTestId("chart")[0]).toHaveTextContent("app-b-0"))
|
||||
})
|
||||
})
|
||||
+31
@@ -59,6 +59,8 @@ vi.mock("./EventsTab.tsx", () => ({ EventsTab: () => null }))
|
||||
vi.mock("./VncTab.tsx", () => ({ VncTab: () => null }))
|
||||
vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null }))
|
||||
vi.mock("./VMPowerControls.tsx", () => ({ VMPowerControls: () => null }))
|
||||
vi.mock("./MetricsTab.tsx", () => ({ MetricsTab: () => null }))
|
||||
vi.mock("./LogsTab.tsx", () => ({ LogsTab: () => null }))
|
||||
|
||||
const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx")
|
||||
|
||||
@@ -111,11 +113,40 @@ describe("ApplicationDetailPage tabs for a virtual machine", () => {
|
||||
"VNC",
|
||||
"Workloads",
|
||||
"Services",
|
||||
"Metrics",
|
||||
"Logs",
|
||||
"Events",
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe("ApplicationDetailPage observability tabs", () => {
|
||||
function showKind(kind: string, plural: string) {
|
||||
h.appKind = kind
|
||||
h.appPlural = plural
|
||||
h.get = {
|
||||
data: { kind, metadata: { name: "demo", namespace: "tenant-test" } },
|
||||
isLoading: false,
|
||||
error: undefined,
|
||||
}
|
||||
h.configMaps = []
|
||||
h.configMapsError = undefined
|
||||
h.tabLabels = []
|
||||
renderPage()
|
||||
}
|
||||
|
||||
it("offers Metrics and Logs on any application kind, ahead of Events", () => {
|
||||
showKind("Postgres", "postgreses")
|
||||
expect(h.tabLabels.slice(-4)).toEqual(["Secrets", "Metrics", "Logs", "Events"])
|
||||
})
|
||||
|
||||
it("leaves Metrics and Logs off a VM disk, which runs no containers", () => {
|
||||
showKind("VMDisk", "vmdisks")
|
||||
expect(h.tabLabels).not.toContain("Metrics")
|
||||
expect(h.tabLabels).not.toContain("Logs")
|
||||
})
|
||||
})
|
||||
|
||||
describe("ApplicationDetailPage configuration", () => {
|
||||
it("offers ConfigMaps declared by the application's resource map", () => {
|
||||
h.appKind = "FoundationDB"
|
||||
|
||||
+16
@@ -37,11 +37,14 @@ import { VncTab } from "./VncTab.tsx"
|
||||
// fetched only when a VM's console is opened. Every dashboard page reaches this
|
||||
// file, and most of them never touch a VM.
|
||||
const SerialTab = lazy(() => import("./SerialTab.tsx").then((m) => ({ default: m.SerialTab })))
|
||||
// Same for the charting library behind the Metrics tab.
|
||||
const MetricsTab = lazy(() => import("./MetricsTab.tsx").then((m) => ({ default: m.MetricsTab })))
|
||||
import { VMPowerControls } from "./VMPowerControls.tsx"
|
||||
import { useResourceBasePath } from "../../lib/portal.ts"
|
||||
import { useResourcePresence } from "./use-resource-presence.ts"
|
||||
import { useApplicationConfigMaps } from "./use-app-configmaps.ts"
|
||||
import { ConfigMapsTab } from "./ConfigMapsTab.tsx"
|
||||
import { LogsTab } from "./LogsTab.tsx"
|
||||
|
||||
export function ApplicationDetailPage() {
|
||||
const { plural, name } = useParams<{ plural: string; name: string }>()
|
||||
@@ -143,6 +146,8 @@ export function ApplicationDetailPage() {
|
||||
{ to: `${base}/vnc`, label: "VNC", end: false },
|
||||
{ to: `${base}/workloads`, label: "Workloads", end: false },
|
||||
{ to: `${base}/services`, label: "Services", end: false },
|
||||
{ to: `${base}/metrics`, label: "Metrics", end: false },
|
||||
{ to: `${base}/logs`, label: "Logs", end: false },
|
||||
{ to: `${base}/events`, label: "Events", end: false },
|
||||
)
|
||||
} else {
|
||||
@@ -163,6 +168,8 @@ export function ApplicationDetailPage() {
|
||||
}
|
||||
tabs.push(
|
||||
{ to: `${base}/secrets`, label: "Secrets", end: false },
|
||||
{ to: `${base}/metrics`, label: "Metrics", end: false },
|
||||
{ to: `${base}/logs`, label: "Logs", end: false },
|
||||
{ to: `${base}/events`, label: "Events", end: false },
|
||||
)
|
||||
}
|
||||
@@ -262,6 +269,15 @@ export function ApplicationDetailPage() {
|
||||
}
|
||||
/>
|
||||
<Route path="vnc" element={<VncTab ad={ad} instance={instance} />} />
|
||||
<Route
|
||||
path="metrics"
|
||||
element={
|
||||
<Suspense fallback={<Spinner />}>
|
||||
<MetricsTab ad={ad} name={name} />
|
||||
</Suspense>
|
||||
}
|
||||
/>
|
||||
<Route path="logs" element={<LogsTab ad={ad} name={name} />} />
|
||||
</Routes>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { ApplicationDefinition } from "@cozystack/types"
|
||||
import { useTenantContext } from "../../lib/tenant-context.tsx"
|
||||
import { LogsView } from "../../components/observability/LogsView.tsx"
|
||||
|
||||
interface LogsTabProps {
|
||||
ad: ApplicationDefinition
|
||||
// The route's name: while the next application loads, the page still holds the previous instance.
|
||||
name: string
|
||||
}
|
||||
|
||||
export function LogsTab({ ad, name }: LogsTabProps) {
|
||||
const { tenantNamespace } = useTenantContext()
|
||||
const plural = ad.spec?.application.plural
|
||||
if (!tenantNamespace || !plural) return null
|
||||
return <LogsView scope={{ namespace: tenantNamespace, plural, name }} />
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
import type { ApplicationDefinition } from "@cozystack/types"
|
||||
import { useTenantContext } from "../../lib/tenant-context.tsx"
|
||||
import { MetricsView } from "../../components/observability/MetricsView.tsx"
|
||||
|
||||
interface MetricsTabProps {
|
||||
ad: ApplicationDefinition
|
||||
// The route's name: while the next application loads, the page still holds the previous instance.
|
||||
name: string
|
||||
}
|
||||
|
||||
export function MetricsTab({ ad, name }: MetricsTabProps) {
|
||||
const { tenantNamespace } = useTenantContext()
|
||||
const plural = ad.spec?.application.plural
|
||||
if (!tenantNamespace || !plural) return null
|
||||
return <MetricsView scope={{ namespace: tenantNamespace, plural, name }} />
|
||||
}
|
||||
+10
@@ -110,6 +110,16 @@ describe("useConsoleSidebarSections — admin areas moved out", () => {
|
||||
expect(findItem(result.current, "Imports")?.to).toBe("/console/migration/vmimporttasks")
|
||||
expect(result.current.some((s) => s.title === "Migration")).toBe(true)
|
||||
})
|
||||
|
||||
it("offers the tenant's namespace logs in an Observability group", async () => {
|
||||
const client = makeClient({ nodes: true, backupclasses: true })
|
||||
const { result } = renderHook(() => useConsoleSidebarSections(), {
|
||||
wrapper: makeWrapper(client),
|
||||
})
|
||||
await waitFor(() => expect(result.current.length).toBeGreaterThan(0))
|
||||
expect(findItem(result.current, "Logs")?.to).toBe("/console/-/observability/logs")
|
||||
expect(result.current.find((s) => s.title === "Observability")?.items).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
function appDef(name: string, kind: string, category: string, module = false) {
|
||||
|
||||
+7
-1
@@ -11,6 +11,7 @@ import {
|
||||
Layers,
|
||||
Network,
|
||||
Package,
|
||||
ScrollText,
|
||||
Plug,
|
||||
Server,
|
||||
ToyBrick,
|
||||
@@ -118,7 +119,12 @@ export function useConsoleSidebarSections(): SidebarSection[] {
|
||||
],
|
||||
}
|
||||
|
||||
return [...categorySections, backupsSection, migrationSection]
|
||||
const observabilitySection: SidebarSection = {
|
||||
title: "Observability",
|
||||
items: [{ label: "Logs", to: "/console/-/observability/logs", icon: ScrollText }],
|
||||
}
|
||||
|
||||
return [...categorySections, backupsSection, migrationSection, observabilitySection]
|
||||
}, [grouped])
|
||||
}
|
||||
|
||||
|
||||
@@ -50,14 +50,12 @@ export class K8sClient {
|
||||
}
|
||||
|
||||
private async request<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
...(init?.headers as Record<string, string>),
|
||||
}
|
||||
const headers = new Headers(init?.headers)
|
||||
if (!headers.has("Content-Type")) headers.set("Content-Type", "application/json")
|
||||
|
||||
if (this.getToken) {
|
||||
const token = await this.getToken()
|
||||
headers["Authorization"] = `Bearer ${token}`
|
||||
headers.set("Authorization", `Bearer ${token}`)
|
||||
}
|
||||
|
||||
let res: Response
|
||||
@@ -241,6 +239,22 @@ export class K8sClient {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a same-origin path with the client's credentials and 401 handling,
|
||||
* leaving status and body to the caller. For endpoints that do not speak
|
||||
* Kubernetes JSON, such as the observability proxy's streamed NDJSON.
|
||||
*/
|
||||
async raw(path: string, init?: RequestInit): Promise<Response> {
|
||||
const headers = new Headers(init?.headers)
|
||||
if (this.getToken) {
|
||||
headers.set("Authorization", `Bearer ${await this.getToken()}`)
|
||||
}
|
||||
const res = await fetch(`${this.baseUrl}${path}`, { ...init, headers })
|
||||
if (res.status === 401) this.handleUnauthorized()
|
||||
else if (res.ok) this.unauthorizedHandled = false
|
||||
return res
|
||||
}
|
||||
|
||||
getApiGroups(): Promise<APIGroupList> {
|
||||
return this.request<APIGroupList>("/apis")
|
||||
}
|
||||
|
||||
@@ -92,6 +92,9 @@ importers:
|
||||
simple-icons:
|
||||
specifier: ^16.22.0
|
||||
version: 16.22.0
|
||||
uplot:
|
||||
specifier: ^1.6.32
|
||||
version: 1.6.32
|
||||
devDependencies:
|
||||
'@tailwindcss/vite':
|
||||
specifier: ^4.2.2
|
||||
@@ -1773,6 +1776,9 @@ packages:
|
||||
peerDependencies:
|
||||
browserslist: '>= 4.21.0'
|
||||
|
||||
uplot@1.6.32:
|
||||
resolution: {integrity: sha512-KIMVnG68zvu5XXUbC4LQEPnhwOxBuLyW1AHtpm6IKTXImkbLgkMy+jabjLgSLMasNuGGzQm/ep3tOkyTxpiQIw==}
|
||||
|
||||
uri-js@4.4.1:
|
||||
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
|
||||
|
||||
@@ -3405,6 +3411,8 @@ snapshots:
|
||||
escalade: 3.2.0
|
||||
picocolors: 1.1.1
|
||||
|
||||
uplot@1.6.32: {}
|
||||
|
||||
uri-js@4.4.1:
|
||||
dependencies:
|
||||
punycode: 2.3.1
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ spec:
|
||||
plural: etcds
|
||||
singular: etcd
|
||||
openAPISchema: |-
|
||||
{"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}
|
||||
{"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}
|
||||
release:
|
||||
prefix: ""
|
||||
labels:
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
suite: ingress-nginx client keepalive_time
|
||||
release:
|
||||
name: ingress-nginx
|
||||
namespace: cozy-ingress-nginx
|
||||
templates:
|
||||
- charts/ingress-nginx/templates/controller-configmap.yaml
|
||||
tests:
|
||||
- it: raises the client keepalive_time above the nginx 1h default
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.http-snippet
|
||||
value: keepalive_time 12h;
|
||||
- it: keeps the snippet when a tenant ingress sets its own config keys
|
||||
# The tenant ingress HelmRelease passes controller.config for whitelist,
|
||||
# Cloudflare and PROXY protocol; Helm merges maps, so those keys must not
|
||||
# drop the default snippet.
|
||||
set:
|
||||
ingress-nginx.controller.config.use-proxy-protocol: "true"
|
||||
ingress-nginx.controller.config.whitelist-source-range: 10.0.0.0/8
|
||||
asserts:
|
||||
- equal:
|
||||
path: data.http-snippet
|
||||
value: keepalive_time 12h;
|
||||
- equal:
|
||||
path: data.use-proxy-protocol
|
||||
value: "true"
|
||||
@@ -47,6 +47,13 @@ ingress-nginx:
|
||||
upstream-keepalive-timeout: "60"
|
||||
upstream-keepalive-connections: "320"
|
||||
ssl-session-tickets: "true"
|
||||
# Client-side keepalive_time has no ConfigMap key, so nginx keeps its
|
||||
# 1h default and sends GOAWAY to any HTTP/2 connection older than that.
|
||||
# A long S3 operation (kopia maintenance behind Velero) multiplexes its
|
||||
# requests over one connection, and the in-flight object listing fails
|
||||
# instead of being retried. Idle connections still close after
|
||||
# keep-alive (75s).
|
||||
http-snippet: "keepalive_time 12h;"
|
||||
resources:
|
||||
limits:
|
||||
cpu: "1"
|
||||
|
||||
@@ -4,11 +4,11 @@ export NAMESPACE=cozy-$(NAME)
|
||||
include ../../../hack/common-envs.mk
|
||||
include ../../../hack/package.mk
|
||||
|
||||
LINSTOR_VERSION ?= 1.33.3
|
||||
LINSTOR_VERSION ?= 1.35.3
|
||||
# The golden-image clone guard in packages/apps/vm-disk/templates/dv.yaml
|
||||
# reads allowRemoteVolumeAccess and layerList the way this release does and
|
||||
# has to be re-checked against the new one.
|
||||
LINSTOR_CSI_VERSION ?= v1.11.2
|
||||
LINSTOR_CSI_VERSION ?= v1.13.2
|
||||
|
||||
test:
|
||||
helm unittest .
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
FROM --platform=$BUILDPLATFORM golang:1.27@sha256:3680233e3204827fbdc66088528ae6d4b3d034f51d03a99d454f6de034888244 AS builder
|
||||
ARG GOPROXY
|
||||
|
||||
ARG VERSION=v1.11.2
|
||||
ARG LINSTOR_WAIT_UNTIL_VERSION=v0.3.1
|
||||
ARG VERSION=v1.13.2
|
||||
ARG LINSTOR_WAIT_UNTIL_VERSION=v1.0.1
|
||||
ARG TARGETARCH
|
||||
ARG TARGETOS
|
||||
|
||||
@@ -15,13 +15,6 @@ RUN git apply /patches/*.diff
|
||||
|
||||
RUN for attempt in 1 2 3; do go mod download && break; [ "$attempt" = 3 ] && exit 1; sleep $((attempt * 10)); done
|
||||
|
||||
# Upstream pins grpc v1.81.0, which uses http2.TrailerPrefix; on Go 1.27
|
||||
# golang.org/x/net builds http2 on top of net/http and no longer defines it.
|
||||
# v1.83.2 also carries the fix for GO-2026-6443, a server panic reachable
|
||||
# through the CSI gRPC endpoint.
|
||||
RUN go get google.golang.org/grpc@v1.83.2 && \
|
||||
go mod tidy
|
||||
|
||||
# go build never compiles a _test.go, so nothing else runs the tests of the
|
||||
# packages the patches touch. Each package is tested from its own directory:
|
||||
# that finds the module it belongs to, and an empty list runs nothing.
|
||||
|
||||
+54
-49
@@ -1,38 +1,37 @@
|
||||
diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go
|
||||
index ac4651d..46d565d 100644
|
||||
--- a/pkg/client/linstor.go
|
||||
+++ b/pkg/client/linstor.go
|
||||
@@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu
|
||||
@@ -498,6 +498,14 @@
|
||||
return err
|
||||
}
|
||||
|
||||
+ if params.RelocateAfterClone {
|
||||
+ logger.Debug("relocate resources to optimal nodes")
|
||||
+
|
||||
+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil {
|
||||
+ if err := s.relocateResources(ctx, vol.ResourceName, rGroup.Name); err != nil {
|
||||
+ logger.WithError(err).Warn("resource relocation failed, volume is still usable")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
logger.Debug("reconcile extra properties")
|
||||
|
||||
err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
|
||||
@@ -1297,6 +1305,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v
|
||||
err = s.client.ResourceDefinitions.Modify(ctx, vol.ResourceName, lapi.GenericPropsModify{OverrideProps: vol.Properties})
|
||||
@@ -1614,6 +1622,14 @@
|
||||
return err
|
||||
}
|
||||
|
||||
+ if snapParams != nil && snapParams.RelocateAfterRestore {
|
||||
+ logger.Debug("relocate resources to optimal nodes")
|
||||
+
|
||||
+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil {
|
||||
+ if err := s.relocateResources(ctx, vol.ResourceName, rGroup.Name); err != nil {
|
||||
+ logger.WithError(err).Warn("resource relocation failed, volume is still usable")
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
logger.Debug("reconcile extra properties")
|
||||
|
||||
err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
|
||||
@@ -1487,9 +1503,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
|
||||
err = s.client.ResourceDefinitions.Modify(ctx, vol.ResourceName, lapi.GenericPropsModify{OverrideProps: vol.Properties})
|
||||
@@ -1818,9 +1834,8 @@
|
||||
return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name)
|
||||
}
|
||||
|
||||
@@ -44,7 +43,7 @@ index ac4651d..46d565d 100644
|
||||
for _, snapNode := range snap.Nodes {
|
||||
if err := s.NodeAvailable(ctx, snapNode); err != nil {
|
||||
logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available")
|
||||
@@ -1497,13 +1512,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
|
||||
@@ -1828,15 +1843,25 @@
|
||||
}
|
||||
|
||||
if slices.Contains(preferredNodes, snapNode) {
|
||||
@@ -58,8 +57,8 @@ index ac4651d..46d565d 100644
|
||||
}
|
||||
+
|
||||
+ available = append(available, snapNode)
|
||||
+ }
|
||||
+
|
||||
}
|
||||
|
||||
+ // Pick a random node from preferred candidates, falling back to any available node.
|
||||
+ // Randomization distributes restore load across snapshot nodes, preventing all
|
||||
+ // clones of the same source from concentrating on a single node.
|
||||
@@ -71,13 +70,20 @@ index ac4651d..46d565d 100644
|
||||
+ var selectedNode string
|
||||
+ if len(candidates) > 0 {
|
||||
+ selectedNode = candidates[rand.Intn(len(candidates))]
|
||||
}
|
||||
|
||||
+ }
|
||||
+
|
||||
if selectedNode == "" {
|
||||
@@ -1696,6 +1721,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("no node [%s] available for snapshot '%s'", strings.Join(snap.Nodes, ","), snap.Name)
|
||||
}
|
||||
@@ -2030,6 +2055,114 @@
|
||||
err = volumeScheduler.Create(ctx, vol.ResourceName, params, topologies)
|
||||
if err != nil {
|
||||
return err
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
+const propertyRelocationTriggered = "Aux/csi-relocation-triggered"
|
||||
+
|
||||
+// relocateResources migrates replicas from their current nodes to the nodes that
|
||||
@@ -181,19 +187,13 @@ index ac4651d..46d565d 100644
|
||||
+ logger.WithError(err).Warn("migrate-disk failed, skipping this pair")
|
||||
+ continue
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
// FindSnapsByID searches the snapshot in the backend
|
||||
func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) {
|
||||
snapshotId, err := volume.ParseSnapshotId(id)
|
||||
}
|
||||
|
||||
return nil
|
||||
diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go
|
||||
index 39acd95..54d1dfb 100644
|
||||
--- a/pkg/volume/parameter.go
|
||||
+++ b/pkg/volume/parameter.go
|
||||
@@ -50,6 +50,7 @@ const (
|
||||
@@ -51,6 +51,7 @@
|
||||
nfsservicename
|
||||
nfssquash
|
||||
nfsrecoveryvolumesize
|
||||
@@ -201,24 +201,24 @@ index 39acd95..54d1dfb 100644
|
||||
)
|
||||
|
||||
// Parameters configuration for linstor volumes.
|
||||
@@ -118,6 +119,8 @@ type Parameters struct {
|
||||
// NfsRecoveryVolumeSize sets the volume size (in bytes) of the recovery volume used by the NFS server.
|
||||
// Defaults to 300MiB.
|
||||
NfsRecoveryVolumeBytes int64
|
||||
@@ -122,6 +123,8 @@
|
||||
// IOQoSLimits holds the IO QoS limits parsed from the qos.linbit.com/* parameters. They are stored on
|
||||
// the LINSTOR resource definition and passed to the nri-volume-qos plugin via VolumeAttachment metadata.
|
||||
IOQoSLimits meta.Limits
|
||||
+ // RelocateAfterClone triggers asynchronous relocation of replicas to optimal nodes after cloning.
|
||||
+ RelocateAfterClone bool
|
||||
}
|
||||
|
||||
const DefaultDisklessStoragePoolName = "DfltDisklessStorPool"
|
||||
@@ -140,6 +143,7 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters,
|
||||
@@ -144,6 +147,7 @@
|
||||
NfsServiceName: "linstor-csi-nfs",
|
||||
NfsSquash: "no_root_squash",
|
||||
NfsRecoveryVolumeBytes: 300 * 1024 * 1024,
|
||||
+ RelocateAfterClone: true,
|
||||
}
|
||||
|
||||
for k, v := range params {
|
||||
@@ -287,6 +291,13 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters,
|
||||
limits, err := meta.ParseLimits(params)
|
||||
@@ -298,6 +302,13 @@
|
||||
}
|
||||
|
||||
p.NfsRecoveryVolumeBytes = s.Value()
|
||||
@@ -233,10 +233,9 @@ index 39acd95..54d1dfb 100644
|
||||
}
|
||||
|
||||
diff --git a/pkg/volume/paramkey_enumer.go b/pkg/volume/paramkey_enumer.go
|
||||
index 5474963..2eb8a7c 100644
|
||||
--- a/pkg/volume/paramkey_enumer.go
|
||||
+++ b/pkg/volume/paramkey_enumer.go
|
||||
@@ -7,11 +7,11 @@ import (
|
||||
@@ -7,11 +7,11 @@
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -251,7 +250,7 @@ index 5474963..2eb8a7c 100644
|
||||
|
||||
func (i paramKey) String() string {
|
||||
if i < 0 || i >= paramKey(len(_paramKeyIndex)-1) {
|
||||
@@ -50,9 +50,10 @@ func _paramKeyNoOp() {
|
||||
@@ -50,9 +50,10 @@
|
||||
_ = x[nfsservicename-(23)]
|
||||
_ = x[nfssquash-(24)]
|
||||
_ = x[nfsrecoveryvolumesize-(25)]
|
||||
@@ -262,17 +261,24 @@ index 5474963..2eb8a7c 100644
|
||||
+var _paramKeyValues = []paramKey{allowremotevolumeaccess, autoplace, clientlist, disklessonremaining, disklessstoragepool, donotplacewithregex, encryption, fsopts, layerlist, mountopts, nodelist, placementcount, placementpolicy, replicasondifferent, replicasonsame, sizekib, storagepool, postmountxfsopts, resourcegroup, usepvcname, overprovision, xreplicasondifferent, nfsconfigtemplate, nfsservicename, nfssquash, nfsrecoveryvolumesize, relocateafterclone}
|
||||
|
||||
var _paramKeyNameToValueMap = map[string]paramKey{
|
||||
_paramKeyName[0:23]: allowremotevolumeaccess,
|
||||
@@ -107,6 +108,8 @@ var _paramKeyNameToValueMap = map[string]paramKey{
|
||||
_paramKeyName[0:23]: allowremotevolumeaccess,
|
||||
@@ -81,6 +82,7 @@
|
||||
_paramKeyName[310:324]: nfsservicename,
|
||||
_paramKeyName[324:333]: nfssquash,
|
||||
_paramKeyName[333:354]: nfsrecoveryvolumesize,
|
||||
+ _paramKeyName[354:372]: relocateafterclone,
|
||||
}
|
||||
|
||||
var _paramKeyLowerNameToValueMap = map[string]paramKey{
|
||||
@@ -110,6 +112,7 @@
|
||||
_paramKeyLowerName[310:324]: nfsservicename,
|
||||
_paramKeyLowerName[324:333]: nfssquash,
|
||||
_paramKeyName[333:354]: nfsrecoveryvolumesize,
|
||||
_paramKeyLowerName[333:354]: nfsrecoveryvolumesize,
|
||||
+ _paramKeyName[354:372]: relocateafterclone,
|
||||
+ _paramKeyLowerName[354:372]: relocateafterclone,
|
||||
}
|
||||
|
||||
var _paramKeyNames = []string{
|
||||
@@ -136,6 +139,7 @@ var _paramKeyNames = []string{
|
||||
@@ -139,6 +142,7 @@
|
||||
_paramKeyName[310:324],
|
||||
_paramKeyName[324:333],
|
||||
_paramKeyName[333:354],
|
||||
@@ -281,18 +287,17 @@ index 5474963..2eb8a7c 100644
|
||||
|
||||
// paramKeyString retrieves an enum value from the enum constants string name.
|
||||
diff --git a/pkg/volume/snapshot_params.go b/pkg/volume/snapshot_params.go
|
||||
index d167cb8..50b70fb 100644
|
||||
--- a/pkg/volume/snapshot_params.go
|
||||
+++ b/pkg/volume/snapshot_params.go
|
||||
@@ -35,6 +35,7 @@ type SnapshotParameters struct {
|
||||
LinstorTargetUrl string `json:"linstor-target-url,omitempty"`
|
||||
LinstorTargetClusterID string `json:"linstor-target-cluster-id,omitempty"`
|
||||
LinstorTargetStoragePool string `json:"linstor-target-storage-pool,omitempty"`
|
||||
+ RelocateAfterRestore bool `json:"relocate-after-restore,omitempty"`
|
||||
@@ -38,6 +38,7 @@
|
||||
LinstorTargetUrl string `json:"linstor-target-url,omitempty"`
|
||||
LinstorTargetClusterID string `json:"linstor-target-cluster-id,omitempty"`
|
||||
LinstorTargetStoragePool string `json:"linstor-target-storage-pool,omitempty"`
|
||||
+ RelocateAfterRestore bool `json:"relocate-after-restore,omitempty"`
|
||||
}
|
||||
|
||||
func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParameters, error) {
|
||||
@@ -91,6 +92,13 @@ func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParamete
|
||||
@@ -116,6 +117,13 @@
|
||||
p.LinstorTargetClusterID = v
|
||||
case "/linstor-target-storage-pool":
|
||||
p.LinstorTargetStoragePool = v
|
||||
|
||||
-132
@@ -1,132 +0,0 @@
|
||||
diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go
|
||||
index ac4651d..9d75c79 100644
|
||||
--- a/pkg/client/linstor.go
|
||||
+++ b/pkg/client/linstor.go
|
||||
@@ -653,11 +653,35 @@ func (s *Linstor) Attach(ctx context.Context, volId, node string, rwxBlock bool)
|
||||
}
|
||||
|
||||
if otherResInUse > 0 && rwxBlock {
|
||||
- rdPropsModify := lapi.GenericPropsModify{OverrideProps: map[string]string{
|
||||
+ rdProps := map[string]string{
|
||||
linstor.PropertyAllowTwoPrimaries: "yes",
|
||||
- }}
|
||||
+ }
|
||||
|
||||
- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify)
|
||||
+ // DRBD requires Protocol C whenever allow-two-primaries is enabled.
|
||||
+ // If the resource is configured with Protocol A or B (typically
|
||||
+ // through its resource-group), drbdadm adjust on the satellites
|
||||
+ // fails with "Protocol C required" (errno 139) and live migration
|
||||
+ // cannot proceed. Override Protocol to C on the resource-definition
|
||||
+ // itself so it applies to every connection (including diskless
|
||||
+ // peers, e.g. a TieBreaker). The marker lets Detach revert exactly
|
||||
+ // the override we installed without disturbing operator-set props.
|
||||
+ proto, protoErr := s.getEffectiveDrbdProtocol(ctx, volId)
|
||||
+ if protoErr != nil {
|
||||
+ s.log.WithError(protoErr).WithField("volume", volId).
|
||||
+ Warn("failed to determine effective DRBD protocol; skipping Protocol=C override for dual-attach")
|
||||
+ } else if proto == "A" || proto == "B" {
|
||||
+ s.log.WithFields(logrus.Fields{
|
||||
+ "volume": volId,
|
||||
+ "protocol": proto,
|
||||
+ }).Info("installing Protocol=C override on resource-definition for dual-attach")
|
||||
+
|
||||
+ rdProps[linstor.PropertyDrbdNetProtocol] = "C"
|
||||
+ rdProps[linstor.PropertyCsiProtocolOverride] = "yes"
|
||||
+ }
|
||||
+
|
||||
+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{
|
||||
+ OverrideProps: rdProps,
|
||||
+ })
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -774,11 +798,26 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error {
|
||||
}
|
||||
|
||||
if resInUse == 1 {
|
||||
- rdPropsModify := lapi.GenericPropsModify{DeleteProps: []string{
|
||||
- linstor.PropertyAllowTwoPrimaries,
|
||||
- }}
|
||||
+ deleteProps := []string{linstor.PropertyAllowTwoPrimaries}
|
||||
+
|
||||
+ // Drop the Protocol=C override only if Attach installed it (gated by
|
||||
+ // our marker), so an operator-set Protocol property on the
|
||||
+ // resource-definition is preserved.
|
||||
+ rd, getErr := s.client.ResourceDefinitions.Get(ctx, volId)
|
||||
+ if getErr != nil {
|
||||
+ log.WithError(getErr).Warn("failed to fetch resource-definition props; skipping Protocol=C override removal")
|
||||
+ } else if rd.Props[linstor.PropertyCsiProtocolOverride] != "" {
|
||||
+ log.Info("removing Protocol=C override from resource-definition")
|
||||
|
||||
- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify)
|
||||
+ deleteProps = append(deleteProps,
|
||||
+ linstor.PropertyDrbdNetProtocol,
|
||||
+ linstor.PropertyCsiProtocolOverride,
|
||||
+ )
|
||||
+ }
|
||||
+
|
||||
+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{
|
||||
+ DeleteProps: deleteProps,
|
||||
+ })
|
||||
if err != nil {
|
||||
return nil404(err)
|
||||
}
|
||||
@@ -805,6 +844,33 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error {
|
||||
return nil404(s.client.Resources.Delete(ctx, volId, node))
|
||||
}
|
||||
|
||||
+// getEffectiveDrbdProtocol returns the DRBD network protocol ("A", "B" or "C")
|
||||
+// that LINSTOR will hand to drbdadm for the given resource. Resource-definition
|
||||
+// properties take precedence; otherwise the value is inherited from the
|
||||
+// resource-group. An empty string is returned when neither level sets the
|
||||
+// property (LINSTOR's compiled-in default is "C").
|
||||
+func (s *Linstor) getEffectiveDrbdProtocol(ctx context.Context, volId string) (string, error) {
|
||||
+ rd, err := s.client.ResourceDefinitions.Get(ctx, volId)
|
||||
+ if err != nil {
|
||||
+ return "", err
|
||||
+ }
|
||||
+
|
||||
+ if v, ok := rd.Props[linstor.PropertyDrbdNetProtocol]; ok {
|
||||
+ return v, nil
|
||||
+ }
|
||||
+
|
||||
+ if rd.ResourceGroupName == "" {
|
||||
+ return "", nil
|
||||
+ }
|
||||
+
|
||||
+ rg, err := s.client.ResourceGroups.Get(ctx, rd.ResourceGroupName)
|
||||
+ if err != nil {
|
||||
+ return "", err
|
||||
+ }
|
||||
+
|
||||
+ return rg.Props[linstor.PropertyDrbdNetProtocol], nil
|
||||
+}
|
||||
+
|
||||
// CapacityBytes returns the amount of free space in the storage pool specified by the params and topology.
|
||||
func (s *Linstor) CapacityBytes(ctx context.Context, storagePools []string, overProvision *float64, segments map[string]string) (int64, error) {
|
||||
log := s.log.WithField("storage-pools", storagePools).WithField("segments", segments)
|
||||
diff --git a/pkg/linstor/const.go b/pkg/linstor/const.go
|
||||
index 9a5f79c..c8bc9c3 100644
|
||||
--- a/pkg/linstor/const.go
|
||||
+++ b/pkg/linstor/const.go
|
||||
@@ -44,6 +44,19 @@ const (
|
||||
// PropertyAllowTwoPrimaries is DRBD option to allow second primary. Mainly used for live-migration.
|
||||
PropertyAllowTwoPrimaries = lc.NamespcDrbdNetOptions + "/allow-two-primaries"
|
||||
|
||||
+ // PropertyDrbdNetProtocol is the DRBD network replication protocol option
|
||||
+ // (A = async, B = semi-sync, C = sync). DRBD requires Protocol C whenever
|
||||
+ // allow-two-primaries is enabled, otherwise drbdadm adjust fails with
|
||||
+ // "Protocol C required" (errno 139).
|
||||
+ PropertyDrbdNetProtocol = lc.NamespcDrbdNetOptions + "/protocol"
|
||||
+
|
||||
+ // PropertyCsiProtocolOverride marks a resource-connection where this driver
|
||||
+ // has installed a temporary Protocol=C override to make a Protocol-A/B
|
||||
+ // resource compatible with allow-two-primaries during live migration. The
|
||||
+ // marker lets us safely remove only the overrides we set, without touching
|
||||
+ // connection-level overrides installed by the operator.
|
||||
+ PropertyCsiProtocolOverride = lc.NamespcAuxiliary + "/csi-protocol-override"
|
||||
+
|
||||
// CreatedForTemporaryDisklessAttach marks a resource as temporary, i.e. it should be removed after it is no longer
|
||||
// needed.
|
||||
CreatedForTemporaryDisklessAttach = "temporary-diskless-attach"
|
||||
@@ -1,9 +1,10 @@
|
||||
ARG DISTRO=bookworm@sha256:f37a335e82bca302e955fa39f9dfe28f1be618f016f8a2b56318e5a5111afc26
|
||||
# LINSTOR 1.35 builds and runs on Java 21, which bookworm does not ship.
|
||||
ARG DISTRO=trixie@sha256:9c6e381421d0f48d1b8133e573f33c185ee16cbf505eda9c4a8a4791b25ea8a6
|
||||
ARG LINSTOR_VERSION
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Build linstor-server from source
|
||||
FROM --platform=$BUILDPLATFORM debian:bookworm@sha256:f37a335e82bca302e955fa39f9dfe28f1be618f016f8a2b56318e5a5111afc26 AS builder
|
||||
FROM --platform=$BUILDPLATFORM debian:trixie@sha256:9c6e381421d0f48d1b8133e573f33c185ee16cbf505eda9c4a8a4791b25ea8a6 AS builder
|
||||
|
||||
ARG LINSTOR_VERSION
|
||||
ARG VERSION=${LINSTOR_VERSION}
|
||||
@@ -13,10 +14,10 @@ ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update \
|
||||
&& apt-get -y upgrade \
|
||||
&& apt-get -y install build-essential git default-jdk-headless python3-all debhelper wget unzip && \
|
||||
wget https://services.gradle.org/distributions/gradle-8.9-bin.zip -O /tmp/gradle.zip && \
|
||||
wget https://services.gradle.org/distributions/gradle-8.14.2-bin.zip -O /tmp/gradle.zip && \
|
||||
unzip -d /opt /tmp/gradle.zip && \
|
||||
rm /tmp/gradle.zip && \
|
||||
ln -s /opt/gradle-8.9/bin/gradle /usr/local/bin/gradle
|
||||
ln -s /opt/gradle-8.14.2/bin/gradle /usr/local/bin/gradle
|
||||
|
||||
RUN git clone https://github.com/LINBIT/linstor-server.git /linstor-server
|
||||
WORKDIR /linstor-server
|
||||
|
||||
@@ -1,36 +1,22 @@
|
||||
# LINSTOR Server Patches
|
||||
|
||||
Custom patches for piraeus-server (linstor-server) v1.33.3.
|
||||
Custom patches for piraeus-server (linstor-server) v1.35.3.
|
||||
|
||||
- **allow-toggle-disk-retry.diff** — Backport maintainer implementation of toggle-disk retry/abort
|
||||
- Source PR/comment: [#475](https://github.com/LINBIT/linstor-server/pull/475), [maintainer note](https://github.com/LINBIT/linstor-server/pull/475#issuecomment-3949630419)
|
||||
- Backported from upstream commit: [`3d97f71c9`](https://github.com/LINBIT/linstor-server/commit/3d97f71c95a493588d3d521c63eac4d846935fb3)
|
||||
- **fix-duplicate-tcp-ports.diff** — Preserve DRBD TCP ports during toggle-disk and avoid redundant `ensureStackDataExists()`
|
||||
- Source PR/review: [#476](https://github.com/LINBIT/linstor-server/pull/476), [review suggestion](https://github.com/LINBIT/linstor-server/pull/476#discussion_r3007725079)
|
||||
- Backported from commits: [`79d6375c5`](https://github.com/kvaps/linstor-server/commit/79d6375c55d6181b35a7b7f0fe8dbdfb86e126cd), [`bcc89902f`](https://github.com/kvaps/linstor-server/commit/bcc89902f4f61ac1589dd07ebb7f5aae1935370d)
|
||||
- **fix-luks-header-size.diff** — Account for LUKS2 `--offset`, metadata/keyslots sizing, and device `optimal_io_size`
|
||||
- Source PR/comment: [#472](https://github.com/LINBIT/linstor-server/pull/472), [maintainer note](https://github.com/LINBIT/linstor-server/pull/472#issuecomment-3949687603)
|
||||
- Backported from commits: [`ccc85fbd2`](https://github.com/LINBIT/linstor-server/commit/ccc85fbd2c65f0b97c52403fa80f1efdb886ec4e), [`71b601554`](https://github.com/LINBIT/linstor-server/commit/71b601554f41bcb50cd5bd06989c5b0d3a814acd)
|
||||
- Note: upstream commit [`3d0402a0c`](https://github.com/LINBIT/linstor-server/commit/3d0402a0c25f0a4b57b380321f10e89982f26e7a) is already included in `v1.33.1`
|
||||
- **fix-k8s-create-already-exists.diff** — Adopt the existing object when a create in the Kubernetes CRD backend returns 409 AlreadyExists. The HTTP client retries a POST after a 5xx or a dropped connection, so a create the apiserver already persisted comes back as 409; a single-object transaction has no rollback entry, the key is dropped from memory while the CR stays, and every later write of that key fails until the controller restarts. Seen as a PV stuck in `Released` because linstor-csi could not set `BalanceResourcesEnabled` on the resource definition.
|
||||
- Source PR: [#531](https://github.com/LINBIT/linstor-server/pull/531)
|
||||
- Backported from commit: [`8badf8f05`](https://github.com/lexfrei/linstor-server/commit/8badf8f0526ff3b1b391857e4a046ed46b95a79c)
|
||||
- **retry-adjust-after-stale-bitmap.diff** — Retry `drbdadm adjust` after detaching a stale local bitmap state
|
||||
- Source PR: [#491](https://github.com/LINBIT/linstor-server/pull/491)
|
||||
- Backported from commit: [`51ae50a84`](https://github.com/kvaps/linstor-server/commit/51ae50a84dcb98093f543b819652c750a94d96c9)
|
||||
- **retry-secondary-after-mkfs.diff** — Retry `drbdadm secondary` after mkfs when DRBD reports the device as held open by an external probe (Talos block-controller, udev, multipathd, etc.). Without this retry, a transient `Device is held open by someone` aborts resource initialization, leaves the satellite in an intermediate state, and prevents the controller from receiving the final `UpToDate` event — orphan PVs in `Released` then cannot be cleaned up.
|
||||
- Related upstream issue: [#268](https://github.com/LINBIT/linstor-server/issues/268)
|
||||
- Related upstream issue: [drbd #74](https://github.com/LINBIT/drbd/issues/74) (same EBUSY pattern with multipathd)
|
||||
|
||||
- **fix-min-io-probe-device-wait.diff** — Wait for temporary probe devices before reading pool I/O properties; retry failures and fall back to a probe when ZFS volumes have no usable device path.
|
||||
- Source PR: [#528](https://github.com/LINBIT/linstor-server/pull/528), fixes [#527](https://github.com/LINBIT/linstor-server/issues/527)
|
||||
- Adapted for v1.33.3; inactive LVM volumes retain the existing no-probe behavior.
|
||||
|
||||
- Rebased onto v1.35.3, where `DrbdAdm` gained the `DFLT_WINDOWS_TIMEOUT_MS` constant; `DrbdadmAdjust` still receives the failed `OutputData` through `ExtCmdFailedException`, so its own retry on "Resize not allowed during resync" keeps working on top of this one.
|
||||
- **native-protoc.diff** — Download the protoc build for the build host's architecture instead of always `linux-x86_64`. The builder stage runs on `$BUILDPLATFORM`, so on a native arm64 builder the x86_64 protoc cannot execute and `./gradlew generateJava` fails with `Syntax error: ")" unexpected`.
|
||||
- Not proposed upstream yet.
|
||||
|
||||
- **snapshot-restore-drbd-initialized.diff** — Mark the volume definitions of a restored snapshot as DRBD-initialized, unless the restored resource definition carries `DrbdOptions/ForceInitialSyncPermanent`, which keeps the existing path that recreates metadata, as for a backup. Without the flag the initial UpToDate race is still open on a restored volume, and `Linstor/Drbd/InitialUptoDateOn`, copied from the source volume through the snapshot, names whichever node won it for the source. When linstor-csi autoplaces a replica on that node, the empty new volume wins the race, gets mkfs and goes StandAlone against the restored replica, so a pod attached there sees an empty filesystem. With the flag set, nothing wins the race, and an autoplaced replica resyncs from the restored one like any replica added to an existing volume.
|
||||
- Backported from upstream commit: [`9d8cdf511`](https://github.com/LINBIT/linstor-server/commit/9d8cdf5118338fc72a34bb99799e452e031ea8f6), first released in v1.34.0; the `CHANGELOG.md` hunk is left out
|
||||
- Drop when LINSTOR is upgraded to v1.34.0 or later
|
||||
- On ZFS and thin pools the autoplaced replicas still skip the initial sync and rely on the source volume's day0 UUID being in the restored replica's history, upstream included; if it is not, DRBD reports unrelated data and goes StandAlone rather than syncing
|
||||
- Fixes [cozystack/cozystack#4862](https://github.com/cozystack/cozystack/issues/4862)
|
||||
Dropped on the move from v1.33.3, because v1.35.3 already carries them:
|
||||
|
||||
- `allow-toggle-disk-retry.diff` — upstream [`3d97f71c9`](https://github.com/LINBIT/linstor-server/commit/3d97f71c95a493588d3d521c63eac4d846935fb3), v1.34.0
|
||||
- `fix-luks-header-size.diff` — upstream [`ccc85fbd2`](https://github.com/LINBIT/linstor-server/commit/ccc85fbd2c65f0b97c52403fa80f1efdb886ec4e) and [`71b601554`](https://github.com/LINBIT/linstor-server/commit/71b601554f41bcb50cd5bd06989c5b0d3a814acd), v1.34.0
|
||||
- `snapshot-restore-drbd-initialized.diff` — upstream [`9d8cdf511`](https://github.com/LINBIT/linstor-server/commit/9d8cdf5118338fc72a34bb99799e452e031ea8f6), v1.34.0
|
||||
- `fix-duplicate-tcp-ports.diff` — upstream [`521c6d211`](https://github.com/LINBIT/linstor-server/commit/521c6d21121dbbcc26cf6eda723c24a5b34889be) (from [#476](https://github.com/LINBIT/linstor-server/pull/476)), v1.34.0
|
||||
- `retry-secondary-after-mkfs.diff` — upstream [`677789c02`](https://github.com/LINBIT/linstor-server/commit/677789c028ac), v1.34.0
|
||||
- `fix-min-io-probe-device-wait.diff` — upstream [`6e5557839`](https://github.com/LINBIT/linstor-server/commit/6e5557839c911c7a1db240dcb6fc348383a74266) (from [#528](https://github.com/LINBIT/linstor-server/pull/528)), v1.35.1
|
||||
@@ -1,261 +0,0 @@
|
||||
diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
|
||||
index d93a18014..a944cb809 100644
|
||||
--- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
|
||||
+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
|
||||
@@ -111,6 +111,14 @@ import reactor.util.function.Tuple2;
|
||||
@Singleton
|
||||
public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionListener
|
||||
{
|
||||
+ private enum ToggleDiskAction
|
||||
+ {
|
||||
+ NOOP,
|
||||
+ NORMAL,
|
||||
+ ABORT,
|
||||
+ RETRY
|
||||
+ }
|
||||
+
|
||||
private final AccessContext apiCtx;
|
||||
private final ScopeRunner scopeRunner;
|
||||
private final BackgroundRunner backgroundRunner;
|
||||
@@ -386,69 +394,33 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
|
||||
"Toggle Disk on %s/%s %s", nodeNameStr, rscNameStr, removeDisk ? "removing disk" : "adding disk");
|
||||
|
||||
Resource rsc = ctrlApiDataLoader.loadRsc(nodeName, rscName, true);
|
||||
+ ResourceDefinition rscDfn = rsc.getResourceDefinition();
|
||||
|
||||
- if (hasDiskAddRequested(rsc))
|
||||
- {
|
||||
- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
|
||||
- ApiConsts.FAIL_RSC_BUSY,
|
||||
- "Addition of disk to resource already requested",
|
||||
- true
|
||||
- ));
|
||||
- }
|
||||
- if (hasDiskRemoveRequested(rsc))
|
||||
- {
|
||||
- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
|
||||
- ApiConsts.FAIL_RSC_BUSY,
|
||||
- "Removal of disk from resource already requested",
|
||||
- true
|
||||
- ));
|
||||
- }
|
||||
+ ToggleDiskAction action = determineToggleDiskAction(rsc, removeDisk);
|
||||
+ errorReporter.logDebug("Toggle Disk action: %s", action);
|
||||
|
||||
- if (!removeDisk && !ctrlVlmCrtApiHelper.isDiskless(rsc))
|
||||
+ switch (action)
|
||||
{
|
||||
- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
|
||||
- ApiConsts.WARN_RSC_ALREADY_HAS_DISK,
|
||||
- "Resource already has disk",
|
||||
- true
|
||||
- ));
|
||||
- }
|
||||
- if (removeDisk && ctrlVlmCrtApiHelper.isDiskless(rsc))
|
||||
- {
|
||||
- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
|
||||
- ApiConsts.WARN_RSC_ALREADY_DISKLESS,
|
||||
- "Resource already diskless",
|
||||
- true
|
||||
- ));
|
||||
+ case NOOP:
|
||||
+ return handleNoopAction(rsc, removeDisk);
|
||||
+ case RETRY:
|
||||
+ return handleRetryAction(rsc, removeDisk, toggleIntoTiebreakerRef, context);
|
||||
+ case ABORT:
|
||||
+ clearToggleDiskFlags(rsc);
|
||||
+ errorReporter.logInfo(
|
||||
+ "Aborting previous toggle disk transition, starting new transition to %s",
|
||||
+ removeDisk ? "diskless" : "diskful"
|
||||
+ );
|
||||
+ break;
|
||||
+ case NORMAL:
|
||||
+ break;
|
||||
+ default:
|
||||
+ throw new ImplementationError("Unhandled case: " + action);
|
||||
}
|
||||
|
||||
- ResourceDefinition rscDfn = rsc.getResourceDefinition();
|
||||
- AccessContext peerCtx = peerAccCtx.get();
|
||||
- if (removeDisk)
|
||||
- {
|
||||
- // Prevent removal of the last disk
|
||||
- int haveDiskCount = countDisksAndIsOnline(rscDfn);
|
||||
- if (haveDiskCount <= 1)
|
||||
- {
|
||||
- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
|
||||
- ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT,
|
||||
- "Cannot remove the disk from the only online resource with a disk",
|
||||
- true
|
||||
- ));
|
||||
- }
|
||||
+ validateToggleDiskPreconditions(rsc, removeDisk);
|
||||
|
||||
- if (!LayerUtils.hasLayer(getLayerData(peerCtx, rsc), DeviceLayerKind.DRBD))
|
||||
- {
|
||||
- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
|
||||
- ApiConsts.FAIL_INVLD_LAYER_STACK,
|
||||
- "Toggle disk is only supported in combination with DRBD",
|
||||
- true
|
||||
- ));
|
||||
- }
|
||||
- }
|
||||
- else
|
||||
- {
|
||||
- ensureAllPeersHavePeerSlotLeft(rscDfn);
|
||||
- }
|
||||
+ AccessContext peerCtx = peerAccCtx.get();
|
||||
|
||||
// Save the requested storage pool in the resource properties.
|
||||
// This does not cause the storage pool to be used automatically.
|
||||
@@ -628,10 +600,10 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
|
||||
|
||||
ctrlTransactionHelper.commit();
|
||||
|
||||
- String action = removeDisk ? "Removal of disk from" : "Addition of disk to";
|
||||
+ String actionStr = removeDisk ? "Removal of disk from" : "Addition of disk to";
|
||||
responses.addEntry(ApiCallRcImpl.simpleEntry(
|
||||
ApiConsts.MODIFIED,
|
||||
- action + " resource '" + rscDfn.getName().displayValue + "' " +
|
||||
+ actionStr + " resource '" + rscDfn.getName().displayValue + "' " +
|
||||
"on node '" + rsc.getNode().getName().displayValue + "' registered"
|
||||
));
|
||||
|
||||
@@ -651,6 +623,40 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
|
||||
);
|
||||
}
|
||||
|
||||
+ private Flux<ApiCallRc> handleNoopAction(Resource rscRef, boolean removeDiskRef)
|
||||
+ {
|
||||
+ String state = removeDiskRef ? "diskless" : "diskful";
|
||||
+ ApiCallRcImpl responses = new ApiCallRcImpl();
|
||||
+ responses.addEntry(ApiCallRcImpl.simpleEntry(
|
||||
+ ApiConsts.INFO_NOOP,
|
||||
+ "Resource '" + rscRef.getResourceDefinition().getName().displayValue + "' on node '" +
|
||||
+ rscRef.getNode().getName().displayValue + "' is already " + state
|
||||
+ ));
|
||||
+ return Flux.just(responses);
|
||||
+ }
|
||||
+
|
||||
+ private Flux<ApiCallRc> handleRetryAction(
|
||||
+ Resource rscRef,
|
||||
+ boolean removeDisk,
|
||||
+ boolean toggleIntoTiebreakerRef,
|
||||
+ ResponseContext context
|
||||
+ )
|
||||
+ {
|
||||
+ String direction = removeDisk ? "diskless" : "diskful";
|
||||
+ ApiCallRcImpl responses = new ApiCallRcImpl();
|
||||
+ NodeName nodeName = rscRef.getNode().getName();
|
||||
+ ResourceName rscName = rscRef.getResourceDefinition().getName();
|
||||
+ responses.addEntry(ApiCallRcImpl.simpleEntry(
|
||||
+ ApiConsts.INFO_NOOP,
|
||||
+ "Retrying toggle disk to " + direction + " for resource '" + rscName.displayValue +
|
||||
+ "' on node '" + nodeName.displayValue + "'"
|
||||
+ ));
|
||||
+
|
||||
+ return Flux
|
||||
+ .<ApiCallRc>just(responses)
|
||||
+ .concatWith(updateAndAdjustDisk(nodeName, rscName, removeDisk, toggleIntoTiebreakerRef, context));
|
||||
+ }
|
||||
+
|
||||
private long getVlmDfnSizePrivileged(VolumeDefinition vlmDfnRef)
|
||||
{
|
||||
try
|
||||
@@ -781,6 +787,96 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
|
||||
}
|
||||
}
|
||||
|
||||
+ private ToggleDiskAction determineToggleDiskAction(Resource rsc, boolean removeDisk)
|
||||
+ {
|
||||
+ boolean isDiskless = ctrlVlmCrtApiHelper.isDiskless(rsc);
|
||||
+ boolean diskAddRequested = hasDiskAddRequested(rsc);
|
||||
+ boolean diskRemoveRequested = hasDiskRemoveRequested(rsc);
|
||||
+
|
||||
+ ToggleDiskAction action;
|
||||
+ if (isDiskless)
|
||||
+ {
|
||||
+ if (diskAddRequested)
|
||||
+ {
|
||||
+ action = removeDisk ? ToggleDiskAction.ABORT : ToggleDiskAction.RETRY;
|
||||
+ }
|
||||
+ else if (diskRemoveRequested)
|
||||
+ {
|
||||
+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT;
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ action = removeDisk ? ToggleDiskAction.NOOP : ToggleDiskAction.NORMAL;
|
||||
+ }
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ if (diskRemoveRequested)
|
||||
+ {
|
||||
+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT;
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ action = removeDisk ? ToggleDiskAction.NORMAL : ToggleDiskAction.NOOP;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return action;
|
||||
+ }
|
||||
+
|
||||
+ private void validateToggleDiskPreconditions(Resource rsc, boolean removeDisk)
|
||||
+ {
|
||||
+ ResourceDefinition rscDfn = rsc.getResourceDefinition();
|
||||
+ if (removeDisk)
|
||||
+ {
|
||||
+ validateDiskRemovalAllowed(rsc, rscDfn);
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ ensureAllPeersHavePeerSlotLeft(rscDfn);
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ private void clearToggleDiskFlags(Resource rsc)
|
||||
+ {
|
||||
+ try
|
||||
+ {
|
||||
+ rsc.getStateFlags().disableFlags(
|
||||
+ apiCtx,
|
||||
+ Resource.Flags.DISK_ADD_REQUESTED,
|
||||
+ Resource.Flags.DISK_ADDING,
|
||||
+ Resource.Flags.DISK_REMOVE_REQUESTED,
|
||||
+ Resource.Flags.DISK_REMOVING
|
||||
+ );
|
||||
+ }
|
||||
+ catch (AccessDeniedException | DatabaseException exc)
|
||||
+ {
|
||||
+ throw new ImplementationError(exc);
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ private void validateDiskRemovalAllowed(Resource rsc, ResourceDefinition rscDfn)
|
||||
+ {
|
||||
+ int haveDiskCount = countDisksAndIsOnline(rscDfn);
|
||||
+ if (haveDiskCount <= 1)
|
||||
+ {
|
||||
+ throw new ApiRcException(ApiCallRcImpl.simpleEntry(
|
||||
+ ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT,
|
||||
+ "Cannot remove the disk from the only online resource with a disk",
|
||||
+ true
|
||||
+ ));
|
||||
+ }
|
||||
+
|
||||
+ if (!LayerUtils.hasLayer(getLayerData(peerAccCtx.get(), rsc), DeviceLayerKind.DRBD))
|
||||
+ {
|
||||
+ throw new ApiRcException(ApiCallRcImpl.simpleEntry(
|
||||
+ ApiConsts.FAIL_INVLD_LAYER_STACK,
|
||||
+ "Toggle disk is only supported in combination with DRBD",
|
||||
+ true
|
||||
+ ));
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
/**
|
||||
* Although we need to rebuild the layerData as the layerList might have changed, if we do not
|
||||
* deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData
|
||||
@@ -1,137 +0,0 @@
|
||||
diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
|
||||
index d93a18014..01cfbbacf 100644
|
||||
--- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
|
||||
+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
|
||||
@@ -37,6 +37,7 @@ import com.linbit.linstor.core.objects.StorPool;
|
||||
import com.linbit.linstor.core.objects.Volume;
|
||||
import com.linbit.linstor.core.objects.VolumeDefinition;
|
||||
import com.linbit.linstor.core.objects.utils.MixedStorPoolHelper;
|
||||
+import com.linbit.linstor.core.types.TcpPortNumber;
|
||||
import com.linbit.linstor.dbdrivers.DatabaseException;
|
||||
import com.linbit.linstor.event.EventWaiter;
|
||||
import com.linbit.linstor.event.ObjectIdentifier;
|
||||
@@ -85,6 +86,7 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
+import java.util.TreeSet;
|
||||
|
||||
import org.reactivestreams.Publisher;
|
||||
import reactor.core.publisher.Flux;
|
||||
@@ -575,12 +577,13 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
|
||||
|
||||
/*
|
||||
* We also have to remove the currently diskless DrbdRscData and free up the node-id as now we must
|
||||
- * use the shared resource's node-id
|
||||
+ * use the shared resource's node-id. We still need to preserve TCP ports though.
|
||||
*/
|
||||
+ copyDrbdTcpPortsIfExists(rsc, payload);
|
||||
}
|
||||
else
|
||||
{
|
||||
- copyDrbdNodeIdIfExists(rsc, payload);
|
||||
+ copyDrbdSettings(rsc, payload);
|
||||
}
|
||||
/*
|
||||
* rebuilds the layerdata in case we just removed it..
|
||||
@@ -782,10 +785,20 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
|
||||
}
|
||||
|
||||
/**
|
||||
- * Although we need to rebuild the layerData as the layerList might have changed, if we do not
|
||||
- * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData
|
||||
- * and recreating a new DrbdRscData ends up with the same node-id as before.
|
||||
+ * Copies DRBD settings (node-id and TCP ports) from the existing DrbdRscData into the payload
|
||||
+ * before removeLayerData() deletes them. This ensures that recreated DrbdRscData ends up with
|
||||
+ * the same node-id and TCP ports as before.
|
||||
+ *
|
||||
+ * TCP ports must be preserved because if the satellite misses the update (e.g. due to controller
|
||||
+ * restart or connectivity issues), it will keep the old ports while peers receive the new ones,
|
||||
+ * causing DRBD connections to fail with StandAlone state.
|
||||
*/
|
||||
+ private void copyDrbdSettings(Resource rsc, LayerPayload payload) throws ImplementationError
|
||||
+ {
|
||||
+ copyDrbdNodeIdIfExists(rsc, payload);
|
||||
+ copyDrbdTcpPortsIfExists(rsc, payload);
|
||||
+ }
|
||||
+
|
||||
private void copyDrbdNodeIdIfExists(Resource rsc, LayerPayload payload) throws ImplementationError
|
||||
{
|
||||
Set<AbsRscLayerObject<Resource>> drbdRscDataSet = LayerRscUtils.getRscDataByLayer(
|
||||
@@ -804,6 +817,28 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
|
||||
}
|
||||
}
|
||||
|
||||
+ private void copyDrbdTcpPortsIfExists(Resource rsc, LayerPayload payload) throws ImplementationError
|
||||
+ {
|
||||
+ Set<AbsRscLayerObject<Resource>> drbdRscDataSet = LayerRscUtils.getRscDataByLayer(
|
||||
+ getLayerData(apiCtx, rsc),
|
||||
+ DeviceLayerKind.DRBD
|
||||
+ );
|
||||
+ if (!drbdRscDataSet.isEmpty())
|
||||
+ {
|
||||
+ DrbdRscData<Resource> drbdRscData = (DrbdRscData<Resource>) drbdRscDataSet.iterator().next();
|
||||
+ Collection<TcpPortNumber> tcpPorts = drbdRscData.getTcpPortList();
|
||||
+ if (tcpPorts != null && !tcpPorts.isEmpty())
|
||||
+ {
|
||||
+ Set<Integer> portInts = new TreeSet<>();
|
||||
+ for (TcpPortNumber port : tcpPorts)
|
||||
+ {
|
||||
+ portInts.add(port.value);
|
||||
+ }
|
||||
+ payload.drbdRsc.tcpPorts = portInts;
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
private List<DeviceLayerKind> removeLayerData(Resource rscRef)
|
||||
{
|
||||
List<DeviceLayerKind> layerList;
|
||||
@@ -1058,15 +1093,15 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
|
||||
/*
|
||||
* We also have to remove the possible meta-children of previous StorageRscData.
|
||||
* LayerData will be recreated with ensureStackDataExists.
|
||||
- * However, we still need to remember our node-id if we had / have DRBD in the list
|
||||
+ * However, we still need to remember our DRBD settings if we had / have DRBD in the list
|
||||
*/
|
||||
- copyDrbdNodeIdIfExists(rsc, payload);
|
||||
+ copyDrbdSettings(rsc, payload);
|
||||
layerList = removeLayerData(rsc);
|
||||
}
|
||||
else
|
||||
{
|
||||
markDiskAdded(rsc);
|
||||
- ctrlLayerStackHelper.resetStoragePools(rsc);
|
||||
+ ctrlLayerStackHelper.resetStoragePools(rsc, false);
|
||||
}
|
||||
ctrlLayerStackHelper.ensureStackDataExists(rsc, layerList, payload);
|
||||
|
||||
diff --git a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java
|
||||
index 3538b380c..f9733b6f1 100644
|
||||
--- a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java
|
||||
+++ b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java
|
||||
@@ -263,6 +263,11 @@ public class CtrlRscLayerDataFactory
|
||||
}
|
||||
|
||||
public void resetStoragePools(Resource rscRef)
|
||||
+ {
|
||||
+ resetStoragePools(rscRef, true);
|
||||
+ }
|
||||
+
|
||||
+ public void resetStoragePools(Resource rscRef, boolean callEnsureStackDataExistsRef)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -276,8 +281,10 @@ public class CtrlRscLayerDataFactory
|
||||
|
||||
rscDataToProcess.addAll(rscData.getChildren());
|
||||
}
|
||||
-
|
||||
- ensureStackDataExists(rscRef, null, new LayerPayload());
|
||||
+ if (callEnsureStackDataExistsRef)
|
||||
+ {
|
||||
+ ensureStackDataExists(rscRef, null, new LayerPayload());
|
||||
+ }
|
||||
}
|
||||
catch (AccessDeniedException exc)
|
||||
{
|
||||
@@ -1,570 +0,0 @@
|
||||
diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
|
||||
--- a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
|
||||
+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
|
||||
@@ -2081,7 +2081,6 @@ public abstract class AbsStorageProvider<
|
||||
)
|
||||
throws IOException, AccessDeniedException
|
||||
{
|
||||
- final StorPoolName storPoolObjName = storPoolObj.getName();
|
||||
if (storDevicePath != null)
|
||||
{
|
||||
errorReporter.logDebug("updateMinIoSize: Have storDevicePath \"%s\"", storDevicePath);
|
||||
@@ -2094,35 +2093,18 @@ public abstract class AbsStorageProvider<
|
||||
"updateMinIoSize: Block device path is \"%s\"",
|
||||
blockDevicePath.toString()
|
||||
);
|
||||
- final long minIoSize = BlockSizeInfo.getBlockSize(blockDevicePath);
|
||||
-
|
||||
- boolean updateValue = true;
|
||||
-
|
||||
- final String propKey = StorageConstants.NAMESPACE_INTERNAL + '/' +
|
||||
- StorageConstants.BLK_DEV_MIN_IO_SIZE;
|
||||
- final Props storPoolProps = storPoolObj.getProps(storDriverAccCtx);
|
||||
- final @Nullable String currentPropValue = storPoolProps.getProp(propKey);
|
||||
- if (currentPropValue != null)
|
||||
- {
|
||||
- try
|
||||
- {
|
||||
- final long currentPropMinIoSize = Long.parseLong(currentPropValue);
|
||||
- updateValue = currentPropMinIoSize != minIoSize;
|
||||
- }
|
||||
- catch (NumberFormatException ignored)
|
||||
- {
|
||||
- }
|
||||
- }
|
||||
-
|
||||
- if (updateValue)
|
||||
- {
|
||||
- final String propValue = Long.toString(minIoSize);
|
||||
- errorReporter.logDebug(
|
||||
- "Storage pool \"%s\": Set property \"%s\" = \"%s\"",
|
||||
- storPoolObjName.displayValue, propKey, propValue
|
||||
- );
|
||||
- propsChange.changeStorPoolProp(storPoolObj, propKey, propValue);
|
||||
- }
|
||||
+ updatePropIfNeeded(
|
||||
+ propsChange,
|
||||
+ storPoolObj,
|
||||
+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_MIN_IO_SIZE,
|
||||
+ Long.toString(BlockSizeInfo.getBlockSize(blockDevicePath))
|
||||
+ );
|
||||
+ updatePropIfNeeded(
|
||||
+ propsChange,
|
||||
+ storPoolObj,
|
||||
+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_OPT_IO_SIZE,
|
||||
+ Long.toString(BlockSizeInfo.getOptimalIoSize(blockDevicePath))
|
||||
+ );
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -2130,6 +2112,28 @@ public abstract class AbsStorageProvider<
|
||||
}
|
||||
}
|
||||
|
||||
+ private void updatePropIfNeeded(
|
||||
+ LocalPropsChangePojo propsChangeRef,
|
||||
+ StorPool storPoolObjRef,
|
||||
+ String propKeyRef,
|
||||
+ String propValueRef
|
||||
+ )
|
||||
+ throws AccessDeniedException
|
||||
+ {
|
||||
+ final Props storPoolProps = storPoolObjRef.getProps(storDriverAccCtx);
|
||||
+ final @Nullable String currentPropValue = storPoolProps.getProp(propKeyRef);
|
||||
+ if (currentPropValue == null || !currentPropValue.equals(propValueRef))
|
||||
+ {
|
||||
+ errorReporter.logDebug(
|
||||
+ "Storage pool \"%s\": Set property \"%s\" = \"%s\"",
|
||||
+ storPoolObjRef.getName().displayValue,
|
||||
+ propKeyRef,
|
||||
+ propValueRef
|
||||
+ );
|
||||
+ propsChangeRef.changeStorPoolProp(storPoolObjRef, propKeyRef, propValueRef);
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public void updateAllocatedSize(VlmProviderObject<Resource> vlmDataRef)
|
||||
diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java
|
||||
--- a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java
|
||||
+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java
|
||||
@@ -3,8 +3,11 @@ package com.linbit.linstor.layer.storage.utils;
|
||||
import com.linbit.utils.MathUtils;
|
||||
import com.linbit.utils.SymbolicLinkResolver;
|
||||
|
||||
+import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_OPT_IO_SIZE;
|
||||
import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_IO_SIZE;
|
||||
+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_OPT_IO_SIZE;
|
||||
import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_IO_SIZE;
|
||||
+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_OPT_IO_SIZE;
|
||||
import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_IO_SIZE;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
@@ -13,6 +16,8 @@ import java.nio.file.Path;
|
||||
|
||||
public class BlockSizeInfo
|
||||
{
|
||||
+ private static final int NUMBER_BUFFER_SIZE = 32;
|
||||
+
|
||||
/**
|
||||
* Determines the blocksize, aka minimum I/O size, for the specified backing storage path.
|
||||
*
|
||||
@@ -51,7 +56,7 @@ public class BlockSizeInfo
|
||||
final Path infoSourceName = blockDevice.getFileName();
|
||||
final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/physical_block_size");
|
||||
|
||||
- final byte[] data = new byte[32];
|
||||
+ final byte[] data = new byte[NUMBER_BUFFER_SIZE];
|
||||
try (final FileInputStream fileIn = new FileInputStream(infoSource.toString()))
|
||||
{
|
||||
final int readCount = fileIn.read(data);
|
||||
@@ -75,4 +80,38 @@ public class BlockSizeInfo
|
||||
}
|
||||
return blockSize;
|
||||
}
|
||||
+
|
||||
+ public static long getOptimalIoSize(final Path storageObj)
|
||||
+ {
|
||||
+ long optIoSize = DFLT_OPT_IO_SIZE;
|
||||
+ try
|
||||
+ {
|
||||
+ final Path blockDevice = SymbolicLinkResolver.resolveSymLink(storageObj);
|
||||
+ final Path infoSourceName = blockDevice.getFileName();
|
||||
+ final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/optimal_io_size");
|
||||
+
|
||||
+ final byte[] data = new byte[NUMBER_BUFFER_SIZE];
|
||||
+ try (final FileInputStream fileIn = new FileInputStream(infoSource.toString()))
|
||||
+ {
|
||||
+ final int readCount = fileIn.read(data);
|
||||
+ if (readCount > 0)
|
||||
+ {
|
||||
+ String numberStr = new String(data, 0, readCount);
|
||||
+ numberStr = numberStr.trim();
|
||||
+ try
|
||||
+ {
|
||||
+ final long unboundedOptIoSize = Long.parseLong(numberStr);
|
||||
+ optIoSize = MathUtils.bounds(MIN_OPT_IO_SIZE, unboundedOptIoSize, MAX_OPT_IO_SIZE);
|
||||
+ }
|
||||
+ catch (NumberFormatException ignored)
|
||||
+ {
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ catch (IOException ignored)
|
||||
+ {
|
||||
+ }
|
||||
+ return optIoSize;
|
||||
+ }
|
||||
}
|
||||
diff --git a/server/src/main/java/com/linbit/SizeConv.java b/server/src/main/java/com/linbit/SizeConv.java
|
||||
--- a/server/src/main/java/com/linbit/SizeConv.java
|
||||
+++ b/server/src/main/java/com/linbit/SizeConv.java
|
||||
@@ -16,6 +16,7 @@ public class SizeConv
|
||||
public enum SizeUnit
|
||||
{
|
||||
UNIT_B,
|
||||
+ UNIT_SECTORS,
|
||||
UNIT_KiB,
|
||||
UNIT_MiB,
|
||||
UNIT_GiB,
|
||||
@@ -41,6 +42,9 @@ public class SizeConv
|
||||
case UNIT_B:
|
||||
factor = FACTOR_B;
|
||||
break;
|
||||
+ case UNIT_SECTORS:
|
||||
+ factor = FACTOR_SECTORS;
|
||||
+ break;
|
||||
case UNIT_KiB:
|
||||
factor = FACTOR_KiB;
|
||||
break;
|
||||
@@ -111,6 +115,9 @@ public class SizeConv
|
||||
case "b":
|
||||
unit = SizeUnit.UNIT_B;
|
||||
break;
|
||||
+ case "s":
|
||||
+ unit = SizeUnit.UNIT_SECTORS;
|
||||
+ break;
|
||||
case "k":
|
||||
// fall-through
|
||||
case "kb":
|
||||
@@ -217,6 +224,9 @@ public class SizeConv
|
||||
// Factor 1
|
||||
public static final BigInteger FACTOR_B = BigInteger.valueOf(1L);
|
||||
|
||||
+ // Factor 512
|
||||
+ public static final BigInteger FACTOR_SECTORS = BigInteger.valueOf(512L);
|
||||
+
|
||||
// Factor 1,024
|
||||
// Naming convention exception: SI unit capitalization rules
|
||||
@SuppressWarnings("checkstyle:constantname")
|
||||
diff --git a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java
|
||||
--- a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java
|
||||
+++ b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java
|
||||
@@ -1,27 +1,62 @@
|
||||
package com.linbit.linstor.layer.luks;
|
||||
|
||||
+import com.linbit.SizeConv;
|
||||
+import com.linbit.SizeConv.SizeUnit;
|
||||
import com.linbit.exceptions.InvalidSizeException;
|
||||
+import com.linbit.linstor.PriorityProps;
|
||||
+import com.linbit.linstor.annotation.Nullable;
|
||||
+import com.linbit.linstor.api.ApiConsts;
|
||||
+import com.linbit.linstor.core.objects.AbsResource;
|
||||
+import com.linbit.linstor.core.objects.AbsVolume;
|
||||
+import com.linbit.linstor.core.objects.Resource;
|
||||
+import com.linbit.linstor.core.objects.ResourceDefinition;
|
||||
+import com.linbit.linstor.core.objects.ResourceGroup;
|
||||
+import com.linbit.linstor.core.objects.Snapshot;
|
||||
+import com.linbit.linstor.core.objects.SnapshotVolume;
|
||||
+import com.linbit.linstor.core.objects.StorPool;
|
||||
+import com.linbit.linstor.core.objects.Volume;
|
||||
+import com.linbit.linstor.core.objects.VolumeDefinition;
|
||||
import com.linbit.linstor.dbdrivers.DatabaseException;
|
||||
import com.linbit.linstor.layer.AbsLayerSizeCalculator;
|
||||
+import com.linbit.linstor.netcom.Peer;
|
||||
+import com.linbit.linstor.propscon.InvalidKeyException;
|
||||
+import com.linbit.linstor.propscon.ReadOnlyProps;
|
||||
import com.linbit.linstor.security.AccessDeniedException;
|
||||
+import com.linbit.linstor.storage.StorageConstants;
|
||||
import com.linbit.linstor.storage.data.adapter.luks.LuksVlmData;
|
||||
import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject;
|
||||
import com.linbit.linstor.storage.kinds.DeviceLayerKind;
|
||||
import com.linbit.linstor.storage.kinds.ExtTools;
|
||||
import com.linbit.linstor.storage.kinds.ExtToolsInfo;
|
||||
import com.linbit.linstor.storage.kinds.ExtToolsInfo.Version;
|
||||
+import com.linbit.linstor.utils.layer.LayerVlmUtils;
|
||||
+import com.linbit.utils.ShellUtils;
|
||||
+import com.linbit.utils.SignedAlign;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Singleton;
|
||||
|
||||
+import java.util.Iterator;
|
||||
+import java.util.LinkedList;
|
||||
+import java.util.List;
|
||||
+import java.util.Set;
|
||||
+import java.util.regex.Matcher;
|
||||
+import java.util.regex.Pattern;
|
||||
+
|
||||
@Singleton
|
||||
public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator<LuksVlmData<?>>
|
||||
{
|
||||
+ public static final String LUKS2_OPT_METADATA_SIZE = "--luks2-metadata-size";
|
||||
+ public static final String LUKS2_OPT_KEYSLOTS_SIZE = "--luks2-keyslots-size";
|
||||
+ public static final String LUKS2_OPT_OFFSET = "--offset";
|
||||
+ private static final String LUKS2_OPT_ALIGN_PAYLOAD = "--align-payload";
|
||||
+ private static final long DFLT_LUKS2_METADATA_SIZE_IN_BYTES = 16L << 10;
|
||||
+ private static final long DFLT_ALIGNMENT_1MIB_IN_BYTES = 1L << 20;
|
||||
+ private static final long LUKS1_HEADER_SIZE_IN_KIB = 2L << 10;
|
||||
+ private static final long LUKS2_HEADER_SIZE_IN_KIB = 16L << 10;
|
||||
+ private static final long LUKS2_HEADER_SIZE_IN_BYTES = LUKS2_HEADER_SIZE_IN_KIB << 10;
|
||||
|
||||
- // linstor calculates in KiB
|
||||
- private static final int MIB = 1024;
|
||||
- private static final int LUKS1_HEADER_SIZE = 2 * MIB;
|
||||
- private static final int LUKS2_HEADER_SIZE = 16 * MIB;
|
||||
+ private static final Pattern PATTERN_SIZE = Pattern.compile("(\\d+)([kKmMgGtT]i?[bB]?|[sS]|)");
|
||||
|
||||
@Inject
|
||||
public LuksLayerSizeCalculator(AbsLayerSizeCalculatorInit initRef)
|
||||
@@ -74,30 +109,266 @@ public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator<LuksVlmData<
|
||||
}
|
||||
|
||||
private long getLuksHeaderSize(VlmProviderObject<?> vlmDataRef)
|
||||
- throws AccessDeniedException
|
||||
+ throws AccessDeniedException, InvalidSizeException
|
||||
{
|
||||
- ExtToolsInfo cryptSetupInfo = vlmDataRef.getRscLayerObject()
|
||||
+ @Nullable Peer peer = vlmDataRef.getRscLayerObject()
|
||||
.getAbsResource()
|
||||
.getNode()
|
||||
- .getPeer(sysCtx)
|
||||
- .getExtToolsManager()
|
||||
+ .getPeer(sysCtx);
|
||||
+ if (peer == null)
|
||||
+ {
|
||||
+ throw new InvalidSizeException(
|
||||
+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined",
|
||||
+ null
|
||||
+ );
|
||||
+ }
|
||||
+ ExtToolsInfo cryptSetupInfo = peer.getExtToolsManager()
|
||||
.getExtToolInfo(ExtTools.CRYPT_SETUP);
|
||||
- long luksHeaderSize;
|
||||
+
|
||||
+ final long luksHeaderSize;
|
||||
if (cryptSetupInfo != null && cryptSetupInfo.isSupported())
|
||||
{
|
||||
if (cryptSetupInfo.hasVersionOrHigher(new Version(2, 1)))
|
||||
{
|
||||
- luksHeaderSize = LUKS2_HEADER_SIZE;
|
||||
+ luksHeaderSize = calcLuks2HeaderSize(vlmDataRef);
|
||||
}
|
||||
else
|
||||
{
|
||||
- luksHeaderSize = LUKS1_HEADER_SIZE;
|
||||
+ luksHeaderSize = LUKS1_HEADER_SIZE_IN_KIB;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
- luksHeaderSize = -1;
|
||||
+ throw new InvalidSizeException(
|
||||
+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined",
|
||||
+ null
|
||||
+ );
|
||||
}
|
||||
return luksHeaderSize;
|
||||
}
|
||||
+
|
||||
+ private long calcLuks2HeaderSize(VlmProviderObject<?> vlmDataRef) throws AccessDeniedException
|
||||
+ {
|
||||
+ PriorityProps prioProps = getPrioProps(vlmDataRef);
|
||||
+
|
||||
+ final @Nullable String userOptProp = prioProps.getProp(
|
||||
+ ApiConsts.KEY_STOR_DRIVER_LUKS_FORMAT_OPTIONS,
|
||||
+ ApiConsts.NAMESPC_STORAGE_DRIVER
|
||||
+ );
|
||||
+ List<String> userOptions = userOptProp != null ?
|
||||
+ ShellUtils.shellSplit(userOptProp) :
|
||||
+ new LinkedList<>();
|
||||
+
|
||||
+ final long alignedHeaderSizeInBytes;
|
||||
+ final long unalignedHeaderSizeInBytes;
|
||||
+ @Nullable Long cryptsetupOffsetInBytes = getLongOptionValue(
|
||||
+ userOptions,
|
||||
+ LUKS2_OPT_OFFSET,
|
||||
+ SizeUnit.UNIT_SECTORS
|
||||
+ );
|
||||
+ if (cryptsetupOffsetInBytes != null)
|
||||
+ {
|
||||
+ alignedHeaderSizeInBytes = cryptsetupOffsetInBytes;
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ @Nullable Long cryptsetupLuks2KeyslotsSize = getLongOptionValue(
|
||||
+ userOptions,
|
||||
+ LUKS2_OPT_KEYSLOTS_SIZE
|
||||
+ );
|
||||
+ if (cryptsetupLuks2KeyslotsSize == null)
|
||||
+ {
|
||||
+ unalignedHeaderSizeInBytes = LUKS2_HEADER_SIZE_IN_BYTES;
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ @Nullable Long cryptsetupLuks2MetadataSize = getLongOptionValue(
|
||||
+ userOptions,
|
||||
+ LUKS2_OPT_METADATA_SIZE
|
||||
+ );
|
||||
+ cryptsetupLuks2MetadataSize = cryptsetupLuks2MetadataSize == null ?
|
||||
+ DFLT_LUKS2_METADATA_SIZE_IN_BYTES :
|
||||
+ cryptsetupLuks2MetadataSize;
|
||||
+
|
||||
+ unalignedHeaderSizeInBytes = 2 * cryptsetupLuks2MetadataSize + cryptsetupLuks2KeyslotsSize;
|
||||
+ }
|
||||
+
|
||||
+ long alignment = getAlignment(vlmDataRef, userOptions);
|
||||
+ alignedHeaderSizeInBytes = new SignedAlign(alignment).ceiling(unalignedHeaderSizeInBytes);
|
||||
+ }
|
||||
+ return SizeConv.convert(
|
||||
+ alignedHeaderSizeInBytes,
|
||||
+ SizeUnit.UNIT_B,
|
||||
+ SizeUnit.UNIT_KiB
|
||||
+ );
|
||||
+ }
|
||||
+
|
||||
+ private long getAlignment(VlmProviderObject<?> vlmDataRef, List<String> userOptions) throws AccessDeniedException
|
||||
+ {
|
||||
+ @Nullable Long cryptsetupAlignPayloadInBytes = getLongOptionValue(
|
||||
+ userOptions,
|
||||
+ LUKS2_OPT_ALIGN_PAYLOAD,
|
||||
+ SizeUnit.UNIT_SECTORS
|
||||
+ );
|
||||
+
|
||||
+ long alignment = DFLT_ALIGNMENT_1MIB_IN_BYTES;
|
||||
+ if (cryptsetupAlignPayloadInBytes != null)
|
||||
+ {
|
||||
+ alignment = cryptsetupAlignPayloadInBytes;
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ final long maxOptIoSize = getMaxOptIoSize(vlmDataRef);
|
||||
+ alignment = Math.max(alignment, maxOptIoSize);
|
||||
+ }
|
||||
+ return alignment;
|
||||
+ }
|
||||
+
|
||||
+ private long getMaxOptIoSize(VlmProviderObject<?> vlmDataRef) throws InvalidKeyException, AccessDeniedException
|
||||
+ {
|
||||
+ long ret = 0;
|
||||
+ Set<StorPool> storPoolSet = LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx);
|
||||
+ for (StorPool sp : storPoolSet)
|
||||
+ {
|
||||
+ @Nullable String strValue = sp.getProps(sysCtx)
|
||||
+ .getProp(
|
||||
+ StorageConstants.BLK_DEV_OPT_IO_SIZE,
|
||||
+ StorageConstants.NAMESPACE_INTERNAL
|
||||
+ );
|
||||
+ if (strValue != null)
|
||||
+ {
|
||||
+ try
|
||||
+ {
|
||||
+ long parsed = Long.parseLong(strValue);
|
||||
+ ret = Math.max(parsed, ret);
|
||||
+ }
|
||||
+ catch (NumberFormatException ignored)
|
||||
+ {
|
||||
+ errorReporter.logWarning(
|
||||
+ "LuksHeaderSize: Failed to parse '%s' from prop %s. Defaulting to 0 opt_io_size " +
|
||||
+ "(no recommendation/hint)",
|
||||
+ strValue,
|
||||
+ StorageConstants.NAMESPACE_INTERNAL + "/" + StorageConstants.BLK_DEV_OPT_IO_SIZE
|
||||
+ );
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ return ret;
|
||||
+ }
|
||||
+
|
||||
+ private @Nullable Long getLongOptionValue(List<String> userOptionsRef, String optRef)
|
||||
+ {
|
||||
+ return getLongOptionValue(userOptionsRef, optRef, SizeUnit.UNIT_B);
|
||||
+ }
|
||||
+
|
||||
+ @SuppressWarnings("checkstyle:magicnumber")
|
||||
+ private @Nullable Long getLongOptionValue(List<String> userOptionsRef, String optRef, SizeUnit dfltSizeUnit)
|
||||
+ {
|
||||
+ @Nullable Long ret = null;
|
||||
+ @Nullable String val = findLastValue(userOptionsRef, optRef);
|
||||
+ if (val != null && !val.isBlank())
|
||||
+ {
|
||||
+ Matcher matcher = PATTERN_SIZE.matcher(val);
|
||||
+ if (matcher.matches())
|
||||
+ {
|
||||
+ try
|
||||
+ {
|
||||
+ ret = Long.parseLong(matcher.group(1));
|
||||
+ String unit = matcher.group(2);
|
||||
+ SizeUnit sizeUnit;
|
||||
+ if (unit.isBlank())
|
||||
+ {
|
||||
+ sizeUnit = dfltSizeUnit;
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ boolean forcePowerOfTwo = unit.length() == 1 || unit.length() == 3;
|
||||
+ sizeUnit = SizeUnit.parse(unit, forcePowerOfTwo);
|
||||
+ }
|
||||
+
|
||||
+ ret = SizeConv.convert(ret, sizeUnit, SizeUnit.UNIT_B);
|
||||
+ }
|
||||
+ catch (NumberFormatException ignored)
|
||||
+ {
|
||||
+ errorReporter.logWarning(
|
||||
+ "LuksHeaderSize: Failed to parse '%s' from option '%s %s'.",
|
||||
+ matcher.group(1),
|
||||
+ optRef,
|
||||
+ val
|
||||
+ );
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ return ret;
|
||||
+ }
|
||||
+
|
||||
+ private @Nullable String findLastValue(List<String> userOptionsRef, String optRef)
|
||||
+ {
|
||||
+ Iterator<String> it = userOptionsRef.iterator();
|
||||
+ @Nullable String val = null;
|
||||
+ while (it.hasNext())
|
||||
+ {
|
||||
+ String opt = it.next();
|
||||
+ if (opt.equals(optRef))
|
||||
+ {
|
||||
+ if (it.hasNext())
|
||||
+ {
|
||||
+ val = it.next();
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ val = null;
|
||||
+ }
|
||||
+ }
|
||||
+ else if (opt.startsWith(optRef + "="))
|
||||
+ {
|
||||
+ val = opt.substring(optRef.length() + 1);
|
||||
+ if (val.isBlank())
|
||||
+ {
|
||||
+ val = null;
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ return val;
|
||||
+ }
|
||||
+
|
||||
+ private PriorityProps getPrioProps(VlmProviderObject<?> vlmDataRef) throws AccessDeniedException
|
||||
+ {
|
||||
+ final AbsVolume<?> vlm = vlmDataRef.getVolume();
|
||||
+ final AbsResource<?> rsc = vlm.getAbsResource();
|
||||
+ final ResourceDefinition rscDfn = vlm.getResourceDefinition();
|
||||
+ final VolumeDefinition vlmDfn = vlm.getVolumeDefinition();
|
||||
+ final ResourceGroup rscGrp = rscDfn.getResourceGroup();
|
||||
+
|
||||
+ final ReadOnlyProps vlmProps;
|
||||
+ final ReadOnlyProps rscProps;
|
||||
+ if (vlm instanceof Volume)
|
||||
+ {
|
||||
+ vlmProps = ((Volume) vlm).getProps(sysCtx);
|
||||
+ rscProps = ((Resource) rsc).getProps(sysCtx);
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ vlmProps = ((SnapshotVolume) vlm).getVlmProps(sysCtx);
|
||||
+ rscProps = ((Snapshot) rsc).getRscProps(sysCtx);
|
||||
+ }
|
||||
+
|
||||
+ final PriorityProps prioProps = new PriorityProps(
|
||||
+ vlmProps,
|
||||
+ rscProps
|
||||
+ );
|
||||
+ for (StorPool storPool : LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx))
|
||||
+ {
|
||||
+ prioProps.addProps(storPool.getProps(sysCtx));
|
||||
+ }
|
||||
+ prioProps.addProps(
|
||||
+ rsc.getNode().getProps(sysCtx),
|
||||
+ vlmDfn.getProps(sysCtx),
|
||||
+ rscDfn.getProps(sysCtx),
|
||||
+ rscGrp.getVolumeGroupProps(sysCtx, vlmDfn.getVolumeNumber()),
|
||||
+ rscGrp.getProps(sysCtx),
|
||||
+ stltProps
|
||||
+ );
|
||||
+ return prioProps;
|
||||
+ }
|
||||
}
|
||||
diff --git a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java
|
||||
--- a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java
|
||||
+++ b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java
|
||||
@@ -13,4 +13,9 @@ public class BlockSizeConsts
|
||||
|
||||
// Default value for the minimum_io_size value of non-storage layers
|
||||
public static final long DFLT_SPECIAL_IO_SIZE = (1L << 12);
|
||||
+
|
||||
+ // optimal_io_size may be 0 to indicate no recommendation.
|
||||
+ public static final long MIN_OPT_IO_SIZE = 0;
|
||||
+ public static final long DFLT_OPT_IO_SIZE = 0;
|
||||
+ public static final long MAX_OPT_IO_SIZE = Long.MAX_VALUE;
|
||||
}
|
||||
diff --git a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java
|
||||
--- a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java
|
||||
+++ b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java
|
||||
@@ -11,6 +11,7 @@ public class StorageConstants
|
||||
public static final String NAMESPACE_NVME = ApiConsts.NAMESPC_STORAGE_DRIVER + "/NVME";
|
||||
public static final String NAMESPACE_INTERNAL = NAMESPACE_STOR_DRIVER + "/internal/";
|
||||
|
||||
+ public static final String BLK_DEV_OPT_IO_SIZE = "optIoSize";
|
||||
public static final String BLK_DEV_MIN_IO_SIZE = "minIoSize";
|
||||
public static final String BLK_DEV_MIN_IO_SIZE_AUTO = "minIoSizeAuto";
|
||||
public static final String BLK_DEV_MAX_BIO_SIZE = "maxBioSize";
|
||||
-660
@@ -1,660 +0,0 @@
|
||||
diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
|
||||
--- a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
|
||||
+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
|
||||
@@ -163,6 +163,8 @@
|
||||
}
|
||||
|
||||
private static final long DFLT_WAIT_UNTIL_DEVICE_CREATED_TIMEOUT_IN_MS = 5000;
|
||||
+ private static final int PROBE_VLM_ATTEMPTS = 3;
|
||||
+ private static final long PROBE_VLM_RETRY_DELAY_IN_MS = 1000;
|
||||
protected static final int DFLT_STRIPES = 1;
|
||||
public static final long SIZE_OF_NOT_FOUND_STOR_POOL = ApiConsts.VAL_STOR_POOL_SPACE_NOT_FOUND;
|
||||
|
||||
@@ -184,6 +186,7 @@
|
||||
protected final Set<String> changedStoragePoolStrings = new HashSet<>();
|
||||
private final String typeDescr;
|
||||
private final FileSystemWatch fsWatch;
|
||||
+ private final Object probeVlmLock = new Object();
|
||||
protected final DeviceProviderKind kind;
|
||||
private final DrbdInvalidateUtils drbdInvalidateUtils;
|
||||
|
||||
@@ -2022,61 +2025,126 @@
|
||||
final StorPoolName storPoolObjName = storPoolObj.getName();
|
||||
errorReporter.logDebug("ENTER updateMinIoSize method: Storage pool \"%s\"", storPoolObjName.displayValue);
|
||||
Collection<VlmProviderObject<Resource>> vlmProviderList = storPoolObj.getVolumes(storDriverAccCtx);
|
||||
+ boolean haveInfo = false;
|
||||
if (vlmProviderList != null && vlmProviderList.size() >= 1)
|
||||
{
|
||||
errorReporter.logDebug("updateMinIoSize: Have vlmProviderList with %d items", vlmProviderList.size());
|
||||
Iterator<VlmProviderObject<Resource>> vlmProviderIter = vlmProviderList.iterator();
|
||||
- boolean haveInfo = false;
|
||||
while (vlmProviderIter.hasNext() && !haveInfo)
|
||||
{
|
||||
VlmProviderObject<Resource> vlmProvider = vlmProviderIter.next();
|
||||
errorReporter.logDebug("updateMinIoSize: Have vlmProvider");
|
||||
final String storDevicePath = vlmProvider.getDevicePath();
|
||||
+ if (storDevicePath != null)
|
||||
+ {
|
||||
+ try
|
||||
+ {
|
||||
+ updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange);
|
||||
+ haveInfo = true;
|
||||
+ }
|
||||
+ catch (IOException ignored)
|
||||
+ {
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+ // An inactive shared LVM volume can be owned by another node; do not probe it by creating a volume.
|
||||
+ if (!haveInfo && this instanceof ProbeVlmStorageProvider &&
|
||||
+ (vlmProviderList == null || vlmProviderList.isEmpty() ||
|
||||
+ kind == DeviceProviderKind.ZFS || kind == DeviceProviderKind.ZFS_THIN))
|
||||
+ {
|
||||
+ updateMinIoSizeByProbeVlm((ProbeVlmStorageProvider) this, storPoolObj, propsChange);
|
||||
+ }
|
||||
+ errorReporter.logDebug("EXIT updateMinIoSize method");
|
||||
+ }
|
||||
+
|
||||
+ private void updateMinIoSizeByProbeVlm(
|
||||
+ ProbeVlmStorageProvider storPrv,
|
||||
+ StorPool storPoolObj,
|
||||
+ LocalPropsChangePojo propsChange
|
||||
+ )
|
||||
+ throws AccessDeniedException
|
||||
+ {
|
||||
+ // Providers use a fixed probe name. Keep creation, probing and cleanup in one critical section.
|
||||
+ synchronized (probeVlmLock)
|
||||
+ {
|
||||
+ boolean haveInfo = false;
|
||||
+ boolean interrupted = false;
|
||||
+ String lastFailure = "";
|
||||
+ int attempts = 0;
|
||||
+ while (attempts < PROBE_VLM_ATTEMPTS && !haveInfo && !interrupted)
|
||||
+ {
|
||||
+ if (attempts > 0)
|
||||
+ {
|
||||
+ try
|
||||
+ {
|
||||
+ Thread.sleep(PROBE_VLM_RETRY_DELAY_IN_MS);
|
||||
+ }
|
||||
+ catch (InterruptedException exc)
|
||||
+ {
|
||||
+ Thread.currentThread().interrupt();
|
||||
+ lastFailure = "interrupted before the next probe attempt";
|
||||
+ break;
|
||||
+ }
|
||||
+ }
|
||||
+ attempts++;
|
||||
+ boolean cleaned = false;
|
||||
try
|
||||
{
|
||||
- updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange);
|
||||
- haveInfo = true;
|
||||
- }
|
||||
- catch (IOException ignored)
|
||||
- {
|
||||
- }
|
||||
- }
|
||||
- }
|
||||
- else
|
||||
- {
|
||||
- errorReporter.logDebug("updateMinIoSize: Don't have vlmProviderList, using temporary volumes");
|
||||
- if (this instanceof ProbeVlmStorageProvider)
|
||||
- {
|
||||
- final ProbeVlmStorageProvider storPrv = (ProbeVlmStorageProvider) this;
|
||||
- try
|
||||
- {
|
||||
- final String storDevicePath = storPrv.createTmpProbeVlm(storPoolObj);
|
||||
+ @Nullable String storDevicePath = storPrv.createTmpProbeVlm(storPoolObj);
|
||||
+ if (storDevicePath == null)
|
||||
+ {
|
||||
+ lastFailure = "temporary volume has no device path";
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ waitUntilDeviceCreated(storDevicePath, getWaitTimeoutAfterCreate(storPoolObj));
|
||||
+ updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange);
|
||||
+ haveInfo = true;
|
||||
+ }
|
||||
+ }
|
||||
+ catch (StorageException | IOException exc)
|
||||
+ {
|
||||
+ lastFailure = exc.getMessage();
|
||||
+ interrupted = exc.getCause() instanceof InterruptedException;
|
||||
+ errorReporter.logDebug("Temporary volume probe attempt %d failed: %s", attempts, lastFailure);
|
||||
+ }
|
||||
+ finally
|
||||
+ {
|
||||
try
|
||||
{
|
||||
- updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange);
|
||||
- }
|
||||
- catch (IOException ignored)
|
||||
- {
|
||||
- }
|
||||
- }
|
||||
- catch (StorageException ignored)
|
||||
- {
|
||||
- errorReporter.logDebug("updateMinIoSize: Temporary volume creation failed");
|
||||
- }
|
||||
- finally
|
||||
- {
|
||||
- try
|
||||
- {
|
||||
storPrv.deleteTmpProbeVlm(storPoolObj);
|
||||
- }
|
||||
- catch (StorageException ignored)
|
||||
- {
|
||||
- errorReporter.logDebug("updateMinIoSize: Temporary volume deletion failed");
|
||||
- }
|
||||
- }
|
||||
- }
|
||||
- }
|
||||
- errorReporter.logDebug("EXIT updateMinIoSize method");
|
||||
+ cleaned = true;
|
||||
+ }
|
||||
+ catch (StorageException exc)
|
||||
+ {
|
||||
+ interrupted |= exc.getCause() instanceof InterruptedException;
|
||||
+ errorReporter.logWarning(
|
||||
+ "Failed to delete temporary probe volume for storage pool '%s': %s",
|
||||
+ storPoolObj.getName().displayValue,
|
||||
+ exc.getMessage()
|
||||
+ );
|
||||
+ }
|
||||
+ }
|
||||
+ if (interrupted)
|
||||
+ {
|
||||
+ Thread.currentThread().interrupt();
|
||||
+ }
|
||||
+ if (!cleaned)
|
||||
+ {
|
||||
+ break;
|
||||
+ }
|
||||
+ }
|
||||
+ if (!haveInfo)
|
||||
+ {
|
||||
+ errorReporter.logWarning(
|
||||
+ "Could not determine block device information for storage pool '%s' after %d probe attempts: %s",
|
||||
+ storPoolObj.getName().displayValue,
|
||||
+ attempts,
|
||||
+ lastFailure
|
||||
+ );
|
||||
+ }
|
||||
+ }
|
||||
}
|
||||
|
||||
private void updateMinIoSizeByDevice(
|
||||
diff --git a/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java b/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java
|
||||
new file mode 100644
|
||||
--- /dev/null
|
||||
+++ b/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java
|
||||
@@ -0,0 +1,467 @@
|
||||
+package com.linbit.fsevent;
|
||||
+
|
||||
+import com.linbit.Platform;
|
||||
+import com.linbit.linstor.core.identifier.StorPoolName;
|
||||
+import com.linbit.linstor.core.objects.Resource;
|
||||
+import com.linbit.linstor.core.objects.StorPool;
|
||||
+import com.linbit.linstor.core.pojos.LocalPropsChangePojo;
|
||||
+import com.linbit.linstor.layer.storage.AbsStorageProvider.AbsStorageProviderInit;
|
||||
+import com.linbit.linstor.layer.storage.BlockSizeConsts;
|
||||
+import com.linbit.linstor.layer.storage.lvm.LvmProvider;
|
||||
+import com.linbit.linstor.layer.storage.zfs.ZfsThinProvider;
|
||||
+import com.linbit.linstor.logging.ErrorReporter;
|
||||
+import com.linbit.linstor.propscon.Props;
|
||||
+import com.linbit.linstor.storage.StorageConstants;
|
||||
+import com.linbit.linstor.storage.StorageException;
|
||||
+import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject;
|
||||
+
|
||||
+import java.io.IOException;
|
||||
+import java.nio.file.Files;
|
||||
+import java.nio.file.Path;
|
||||
+import java.util.Collections;
|
||||
+import java.util.List;
|
||||
+import java.util.Map;
|
||||
+import java.util.concurrent.Callable;
|
||||
+import java.util.concurrent.CountDownLatch;
|
||||
+import java.util.concurrent.ExecutorService;
|
||||
+import java.util.concurrent.Executors;
|
||||
+import java.util.concurrent.Future;
|
||||
+import java.util.concurrent.ScheduledExecutorService;
|
||||
+import java.util.concurrent.ScheduledFuture;
|
||||
+import java.util.concurrent.TimeUnit;
|
||||
+import java.util.concurrent.atomic.AtomicInteger;
|
||||
+
|
||||
+import org.junit.After;
|
||||
+import org.junit.Before;
|
||||
+import org.junit.Rule;
|
||||
+import org.junit.Test;
|
||||
+import org.junit.rules.TemporaryFolder;
|
||||
+import org.junit.rules.Timeout;
|
||||
+
|
||||
+import static org.junit.Assert.assertEquals;
|
||||
+import static org.junit.Assert.assertFalse;
|
||||
+import static org.junit.Assert.assertTrue;
|
||||
+import static org.junit.Assume.assumeTrue;
|
||||
+import static org.mockito.Mockito.doReturn;
|
||||
+import static org.mockito.Mockito.mock;
|
||||
+import static org.mockito.Mockito.mockingDetails;
|
||||
+import static org.mockito.Mockito.when;
|
||||
+
|
||||
+public class ProbeVolumeDeviceWaitTest
|
||||
+{
|
||||
+ @Rule
|
||||
+ public final TemporaryFolder directory = new TemporaryFolder();
|
||||
+
|
||||
+ @Rule
|
||||
+ public final Timeout timeout = Timeout.seconds(15);
|
||||
+
|
||||
+ private FileSystemWatch watch;
|
||||
+ private Path device;
|
||||
+ private Path link;
|
||||
+ private StorPool pool;
|
||||
+ private ProbeProvider provider;
|
||||
+ private ErrorReporter reporter;
|
||||
+ private ScheduledExecutorService executor;
|
||||
+
|
||||
+ @Before
|
||||
+ public void setUp() throws Exception
|
||||
+ {
|
||||
+ assumeTrue(Platform.isLinux());
|
||||
+ device = directory.newFile("zd-test").toPath();
|
||||
+ link = directory.getRoot().toPath().resolve("probe");
|
||||
+ executor = Executors.newSingleThreadScheduledExecutor();
|
||||
+ reporter = mock(ErrorReporter.class);
|
||||
+ watch = new FileSystemWatch(reporter);
|
||||
+ watch.start();
|
||||
+ pool = mock(StorPool.class);
|
||||
+ when(pool.getName()).thenReturn(new StorPoolName("data"));
|
||||
+ when(pool.getVolumes(null)).thenReturn(Collections.emptyList());
|
||||
+ when(pool.getProps(null)).thenReturn(mock(Props.class));
|
||||
+ provider = new ProbeProvider(providerInit());
|
||||
+ }
|
||||
+
|
||||
+ @After
|
||||
+ public void tearDown() throws Exception
|
||||
+ {
|
||||
+ if (executor != null)
|
||||
+ {
|
||||
+ executor.shutdownNow();
|
||||
+ assertTrue(executor.awaitTermination(2, TimeUnit.SECONDS));
|
||||
+ }
|
||||
+ if (watch != null)
|
||||
+ {
|
||||
+ watch.shutdown(false);
|
||||
+ watch.awaitShutdown(2000);
|
||||
+ watch.cancelAllWatchKeys();
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void delayedDevicePublishesPropertiesBeforeCleanup() throws Exception
|
||||
+ {
|
||||
+ provider.delayMillis = 50;
|
||||
+ assertPublished(probe());
|
||||
+ assertCleaned(1);
|
||||
+ assertEquals(0, warningCount());
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void existingProbeDevicePublishesProperties() throws Exception
|
||||
+ {
|
||||
+ assertPublished(probe());
|
||||
+ assertCleaned(1);
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void missingDeviceTimesOutAndCleansEveryAttempt() throws Exception
|
||||
+ {
|
||||
+ provider.delayMillis = -1;
|
||||
+ provider.timeoutMillis = 20;
|
||||
+ assertTrue(probe().isEmpty());
|
||||
+ assertCleaned(3);
|
||||
+ assertEquals(1, warningCount());
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void failedCreationStillCleansEveryAttempt() throws Exception
|
||||
+ {
|
||||
+ provider.failCreateTimes = Integer.MAX_VALUE;
|
||||
+ assertTrue(probe().isEmpty());
|
||||
+ assertCleaned(3);
|
||||
+ assertEquals(1, warningCount());
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void transientCreationFailureIsRetried() throws Exception
|
||||
+ {
|
||||
+ provider.failCreateTimes = 1;
|
||||
+ provider.delayMillis = 50;
|
||||
+ assertPublished(probe());
|
||||
+ assertCleaned(2);
|
||||
+ assertEquals(0, warningCount());
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void nullProbePathDoesNotPublishProperties() throws Exception
|
||||
+ {
|
||||
+ provider.nullPath = true;
|
||||
+ assertTrue(probe().isEmpty());
|
||||
+ assertCleaned(3);
|
||||
+ assertEquals(1, warningCount());
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void nullFirstPathDoesNotSkipUsableVolume() throws Exception
|
||||
+ {
|
||||
+ Files.createSymbolicLink(link, device);
|
||||
+ doReturn(List.of(volume(null), volume(link))).when(pool).getVolumes(null);
|
||||
+ assertPublished(probe());
|
||||
+ assertEquals(0, provider.createCalls);
|
||||
+ assertEquals(0, provider.deleteCalls);
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void missingFirstPathDoesNotSkipUsableVolume() throws Exception
|
||||
+ {
|
||||
+ Files.createSymbolicLink(link, device);
|
||||
+ doReturn(List.of(volume(link.resolveSibling("missing")), volume(link))).when(pool).getVolumes(null);
|
||||
+ assertPublished(probe());
|
||||
+ assertEquals(0, provider.createCalls);
|
||||
+ assertEquals(0, provider.deleteCalls);
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void allNullPathsFallBackToProbe() throws Exception
|
||||
+ {
|
||||
+ doReturn(List.of(volume(null))).when(pool).getVolumes(null);
|
||||
+ provider.delayMillis = 50;
|
||||
+ assertPublished(probe());
|
||||
+ assertCleaned(1);
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void allMissingPathsFallBackToProbe() throws Exception
|
||||
+ {
|
||||
+ doReturn(List.of(
|
||||
+ volume(link.resolveSibling("missing-a")), volume(link.resolveSibling("missing-b"))
|
||||
+ )).when(pool).getVolumes(null);
|
||||
+ provider.delayMillis = 50;
|
||||
+ assertPublished(probe());
|
||||
+ assertCleaned(1);
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void cleanupFailureAfterSuccessIsReported() throws Exception
|
||||
+ {
|
||||
+ provider.failDelete = true;
|
||||
+ assertPublished(probe());
|
||||
+ assertEquals(1, provider.createCalls);
|
||||
+ assertEquals(1, provider.deleteCalls);
|
||||
+ assertEquals(1, warningCount());
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void cleanupFailurePreventsAnotherCreationAttempt() throws Exception
|
||||
+ {
|
||||
+ provider.delayMillis = -1;
|
||||
+ provider.timeoutMillis = 20;
|
||||
+ provider.failDelete = true;
|
||||
+ assertTrue(probe().isEmpty());
|
||||
+ assertEquals(1, provider.createCalls);
|
||||
+ assertEquals(1, provider.deleteCalls);
|
||||
+ assertEquals(2, warningCount());
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void interruptedDeviceWaitCleansUpAndPreservesInterrupt() throws Exception
|
||||
+ {
|
||||
+ provider.delayMillis = -1;
|
||||
+ Thread callingThread = Thread.currentThread();
|
||||
+ executor.schedule(callingThread::interrupt, 50, TimeUnit.MILLISECONDS);
|
||||
+ try
|
||||
+ {
|
||||
+ assertTrue(probe().isEmpty());
|
||||
+ assertTrue(callingThread.isInterrupted());
|
||||
+ assertCleaned(1);
|
||||
+ assertEquals(1, warningCount());
|
||||
+ }
|
||||
+ finally
|
||||
+ {
|
||||
+ Thread.interrupted();
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void interruptedRetryDelayDoesNotCreateAnotherProbe() throws Exception
|
||||
+ {
|
||||
+ provider.failCreateTimes = Integer.MAX_VALUE;
|
||||
+ Thread callingThread = Thread.currentThread();
|
||||
+ executor.schedule(callingThread::interrupt, 100, TimeUnit.MILLISECONDS);
|
||||
+ try
|
||||
+ {
|
||||
+ assertTrue(probe().isEmpty());
|
||||
+ assertTrue(callingThread.isInterrupted());
|
||||
+ assertCleaned(1);
|
||||
+ assertEquals(1, warningCount());
|
||||
+ }
|
||||
+ finally
|
||||
+ {
|
||||
+ Thread.interrupted();
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void interruptedCleanupPreservesInterruptAndStopsRetries() throws Exception
|
||||
+ {
|
||||
+ provider.failCreateTimes = Integer.MAX_VALUE;
|
||||
+ provider.interruptDelete = true;
|
||||
+ try
|
||||
+ {
|
||||
+ assertTrue(probe().isEmpty());
|
||||
+ assertTrue(Thread.currentThread().isInterrupted());
|
||||
+ assertEquals(1, provider.createCalls);
|
||||
+ assertEquals(1, provider.deleteCalls);
|
||||
+ }
|
||||
+ finally
|
||||
+ {
|
||||
+ Thread.interrupted();
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void concurrentProbesDoNotShareTheTemporaryVolume() throws Exception
|
||||
+ {
|
||||
+ provider.creationDelayMillis = 200;
|
||||
+ ExecutorService callers = Executors.newFixedThreadPool(2);
|
||||
+ CountDownLatch ready = new CountDownLatch(2);
|
||||
+ CountDownLatch start = new CountDownLatch(1);
|
||||
+ Callable<LocalPropsChangePojo> call = () ->
|
||||
+ {
|
||||
+ ready.countDown();
|
||||
+ start.await();
|
||||
+ return probe();
|
||||
+ };
|
||||
+ try
|
||||
+ {
|
||||
+ Future<LocalPropsChangePojo> first = callers.submit(call);
|
||||
+ Future<LocalPropsChangePojo> second = callers.submit(call);
|
||||
+ assertTrue(ready.await(2, TimeUnit.SECONDS));
|
||||
+ start.countDown();
|
||||
+ assertFalse(first.get(10, TimeUnit.SECONDS).isEmpty());
|
||||
+ assertFalse(second.get(10, TimeUnit.SECONDS).isEmpty());
|
||||
+ assertEquals(1, provider.peakActiveProbes.get());
|
||||
+ assertCleaned(2);
|
||||
+ }
|
||||
+ finally
|
||||
+ {
|
||||
+ callers.shutdownNow();
|
||||
+ assertTrue(callers.awaitTermination(2, TimeUnit.SECONDS));
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ @Test
|
||||
+ public void inactiveLvmVolumesDoNotCreateProbe() throws Exception
|
||||
+ {
|
||||
+ doReturn(List.of(volume(null))).when(pool).getVolumes(null);
|
||||
+ LvmProvider lvm = new LvmProvider(providerInit())
|
||||
+ {
|
||||
+ @Override
|
||||
+ public String createTmpProbeVlm(StorPool ignored)
|
||||
+ {
|
||||
+ throw new AssertionError("Inactive shared LVM volumes must not trigger a probe volume");
|
||||
+ }
|
||||
+
|
||||
+ @Override
|
||||
+ public void deleteTmpProbeVlm(StorPool ignored)
|
||||
+ {
|
||||
+ throw new AssertionError("No probe volume was created");
|
||||
+ }
|
||||
+ };
|
||||
+ LocalPropsChangePojo changes = new LocalPropsChangePojo();
|
||||
+ lvm.updateMinIoSize(pool, changes);
|
||||
+ assertTrue(changes.isEmpty());
|
||||
+ }
|
||||
+
|
||||
+ private AbsStorageProviderInit providerInit()
|
||||
+ {
|
||||
+ return new AbsStorageProviderInit(
|
||||
+ reporter, null, null, null, null, null, null, null, null, null, watch, null, null, null
|
||||
+ );
|
||||
+ }
|
||||
+
|
||||
+ private LocalPropsChangePojo probe() throws Exception
|
||||
+ {
|
||||
+ LocalPropsChangePojo changes = new LocalPropsChangePojo();
|
||||
+ provider.updateMinIoSize(pool, changes);
|
||||
+ return changes;
|
||||
+ }
|
||||
+
|
||||
+ private void assertPublished(LocalPropsChangePojo changesRef)
|
||||
+ {
|
||||
+ String namespace = StorageConstants.NAMESPACE_INTERNAL + '/';
|
||||
+ assertEquals(Map.of(
|
||||
+ namespace + StorageConstants.BLK_DEV_MIN_IO_SIZE, Long.toString(BlockSizeConsts.DFLT_IO_SIZE),
|
||||
+ namespace + StorageConstants.BLK_DEV_OPT_IO_SIZE, Long.toString(BlockSizeConsts.DFLT_OPT_IO_SIZE)
|
||||
+ ), changesRef.changedStorPoolProps.get(pool.getName()));
|
||||
+ }
|
||||
+
|
||||
+ private void assertCleaned(int attemptsRef)
|
||||
+ {
|
||||
+ assertEquals(attemptsRef, provider.createCalls);
|
||||
+ assertEquals(attemptsRef, provider.deleteCalls);
|
||||
+ assertFalse(Files.exists(link));
|
||||
+ }
|
||||
+
|
||||
+ @SuppressWarnings("unchecked")
|
||||
+ private VlmProviderObject<Resource> volume(Path pathRef)
|
||||
+ {
|
||||
+ VlmProviderObject<Resource> volume = mock(VlmProviderObject.class);
|
||||
+ when(volume.getDevicePath()).thenReturn(pathRef == null ? null : pathRef.toString());
|
||||
+ return volume;
|
||||
+ }
|
||||
+
|
||||
+ private long warningCount()
|
||||
+ {
|
||||
+ return mockingDetails(reporter).getInvocations().stream()
|
||||
+ .filter(invocation -> invocation.getMethod().getName().equals("logWarning"))
|
||||
+ .count();
|
||||
+ }
|
||||
+
|
||||
+ private class ProbeProvider extends ZfsThinProvider
|
||||
+ {
|
||||
+ private long delayMillis;
|
||||
+ private long creationDelayMillis;
|
||||
+ private long timeoutMillis = 4000;
|
||||
+ private int failCreateTimes;
|
||||
+ private boolean nullPath;
|
||||
+ private boolean failDelete;
|
||||
+ private boolean interruptDelete;
|
||||
+ private int createCalls;
|
||||
+ private int deleteCalls;
|
||||
+ private ScheduledFuture<?> create;
|
||||
+ private final AtomicInteger activeProbes = new AtomicInteger();
|
||||
+ private final AtomicInteger peakActiveProbes = new AtomicInteger();
|
||||
+
|
||||
+ ProbeProvider(AbsStorageProviderInit initRef)
|
||||
+ {
|
||||
+ super(initRef);
|
||||
+ }
|
||||
+
|
||||
+ @Override
|
||||
+ protected long getWaitTimeoutAfterCreate(StorPool ignored)
|
||||
+ {
|
||||
+ return timeoutMillis;
|
||||
+ }
|
||||
+
|
||||
+ @Override
|
||||
+ public String createTmpProbeVlm(StorPool ignored) throws StorageException
|
||||
+ {
|
||||
+ peakActiveProbes.accumulateAndGet(activeProbes.incrementAndGet(), Math::max);
|
||||
+ createCalls++;
|
||||
+ if (creationDelayMillis > 0)
|
||||
+ {
|
||||
+ try
|
||||
+ {
|
||||
+ Thread.sleep(creationDelayMillis);
|
||||
+ }
|
||||
+ catch (InterruptedException exc)
|
||||
+ {
|
||||
+ throw new StorageException("Test probe creation interrupted", exc);
|
||||
+ }
|
||||
+ }
|
||||
+ if (failCreateTimes > 0)
|
||||
+ {
|
||||
+ failCreateTimes--;
|
||||
+ throw new StorageException("Probe creation failed");
|
||||
+ }
|
||||
+ if (nullPath)
|
||||
+ {
|
||||
+ return null;
|
||||
+ }
|
||||
+ if (delayMillis == 0)
|
||||
+ {
|
||||
+ try
|
||||
+ {
|
||||
+ Files.createSymbolicLink(link, device);
|
||||
+ }
|
||||
+ catch (IOException exc)
|
||||
+ {
|
||||
+ throw new StorageException("Test device creation failed", exc);
|
||||
+ }
|
||||
+ }
|
||||
+ else if (delayMillis > 0)
|
||||
+ {
|
||||
+ create = executor.schedule(() -> Files.createSymbolicLink(link, device),
|
||||
+ delayMillis, TimeUnit.MILLISECONDS);
|
||||
+ }
|
||||
+ return link.toString();
|
||||
+ }
|
||||
+
|
||||
+ @Override
|
||||
+ public void deleteTmpProbeVlm(StorPool ignored) throws StorageException
|
||||
+ {
|
||||
+ deleteCalls++;
|
||||
+ if (interruptDelete)
|
||||
+ {
|
||||
+ throw new StorageException("Probe cleanup interrupted", new InterruptedException());
|
||||
+ }
|
||||
+ if (failDelete)
|
||||
+ {
|
||||
+ throw new StorageException("Probe cleanup failed");
|
||||
+ }
|
||||
+ try
|
||||
+ {
|
||||
+ if (create != null && !create.cancel(false))
|
||||
+ {
|
||||
+ create.get();
|
||||
+ }
|
||||
+ Files.deleteIfExists(link);
|
||||
+ activeProbes.decrementAndGet();
|
||||
+ }
|
||||
+ catch (Exception exc)
|
||||
+ {
|
||||
+ throw new StorageException("Test device cleanup failed", exc);
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+}
|
||||
@@ -1,8 +1,8 @@
|
||||
diff --git a/build.gradle b/build.gradle
|
||||
index 75b7f64..d2dfec9 100644
|
||||
index 9df6c840cdc0..9894a436712c 100644
|
||||
--- a/build.gradle
|
||||
+++ b/build.gradle
|
||||
@@ -37,6 +37,9 @@ configurations {
|
||||
@@ -34,6 +34,9 @@ configurations {
|
||||
}
|
||||
|
||||
def protobufVersion = '31.1'
|
||||
@@ -12,7 +12,7 @@ index 75b7f64..d2dfec9 100644
|
||||
def protoc = file("${projectDir}/tools/protoc-" + protobufVersion + '/bin/protoc')
|
||||
def logbackConfig = "logback.xml"
|
||||
def jvmOpts = ["-Xms32M"]
|
||||
@@ -577,7 +580,7 @@ tasks.register('buildTestCommands', Exec) {
|
||||
@@ -650,7 +653,7 @@ tasks.register('buildTestCommands', Exec) {
|
||||
tasks.register('unzipProtoc', Copy) {
|
||||
dependsOn(downloadProtoc)
|
||||
|
||||
@@ -21,7 +21,7 @@ index 75b7f64..d2dfec9 100644
|
||||
def outputDir = file("tools/protoc-" + protobufVersion)
|
||||
|
||||
from zipTree(zipFile)
|
||||
@@ -591,7 +594,7 @@ tasks.register('unzipProtoc', Copy) {
|
||||
@@ -664,7 +667,7 @@ tasks.register('unzipProtoc', Copy) {
|
||||
|
||||
tasks.register('downloadProtoc') {
|
||||
outputs.dir "tools"
|
||||
@@ -30,7 +30,7 @@ index 75b7f64..d2dfec9 100644
|
||||
outputs.file protozip
|
||||
|
||||
doLast {
|
||||
@@ -599,7 +602,7 @@ tasks.register('downloadProtoc') {
|
||||
@@ -672,7 +675,7 @@ tasks.register('downloadProtoc') {
|
||||
mkdir "tools"
|
||||
println "downloading protoc..."
|
||||
new URL('https://github.com/google/protobuf/releases/download/v'
|
||||
|
||||
+12
-13
@@ -1,28 +1,27 @@
|
||||
diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
|
||||
index 5627d1be8..ece191292 100644
|
||||
index 0a19f875b702..d44e19b70d88 100644
|
||||
--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
|
||||
+++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
|
||||
@@ -42,6 +42,9 @@ import java.util.Arrays;
|
||||
import java.util.List;
|
||||
@@ -45,6 +45,8 @@ import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
+import java.nio.charset.StandardCharsets;
|
||||
+import java.util.regex.Matcher;
|
||||
+import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Singleton
|
||||
@@ -56,6 +58,9 @@ public class DrbdAdm
|
||||
|
||||
public static final int WAIT_CONNECT_RES_TIME = 10;
|
||||
private static final long DOWN_WAIT_TIMEOUT_SEC = 5;
|
||||
@@ -68,6 +70,9 @@ public class DrbdAdm
|
||||
private static final long SECONDARY_RETRY_INITIAL_BACKOFF_MS = 250L;
|
||||
private static final long SECONDARY_RETRY_MAX_BACKOFF_MS = 2_000L;
|
||||
private static final String DEVICE_HELD_OPEN_ERR_MSG = "Device is held open by someone";
|
||||
+ private static final long FORCE_DETACH_RETRY_WAIT_MS = 250;
|
||||
+ private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen";
|
||||
+ private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b");
|
||||
|
||||
private final ExtCmdFactory extCmdFactory;
|
||||
private final AccessContext sysCtx;
|
||||
@@ -131,8 +136,38 @@ public class DrbdAdm
|
||||
private final StltConfigAccessor stltCfgAccessor;
|
||||
@@ -143,8 +148,38 @@ public class DrbdAdm
|
||||
// command.add(resName);
|
||||
command.add(drbdRscData.getSuffixedResourceName());
|
||||
// execute(Arrays.asList("drbdsetup", "show", drbdRscData.getSuffixedResourceName()));
|
||||
@@ -35,7 +34,7 @@ index 5627d1be8..ece191292 100644
|
||||
+ ExtCmd extCmd = extCmdFactory.create();
|
||||
+ if (Platform.isWindows())
|
||||
+ {
|
||||
+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000);
|
||||
+ extCmd.setTimeout(TimeoutType.WAIT, DFLT_WINDOWS_TIMEOUT_MS);
|
||||
+ }
|
||||
+
|
||||
+ OutputData outputData = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr);
|
||||
@@ -63,8 +62,8 @@ index 5627d1be8..ece191292 100644
|
||||
|
||||
drbdRscData.setAdjustRequired(false);
|
||||
}
|
||||
@@ -805,6 +840,75 @@ public class DrbdAdm
|
||||
}
|
||||
@@ -955,6 +990,75 @@ public class DrbdAdm
|
||||
return ret;
|
||||
}
|
||||
|
||||
+ static boolean isBitmapLeakOnAttach(OutputData outputData)
|
||||
|
||||
-104
@@ -1,104 +0,0 @@
|
||||
diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
|
||||
index 71c1df6..f91903b 100644
|
||||
--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
|
||||
+++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
|
||||
@@ -62,6 +62,10 @@ public class DrbdAdm
|
||||
private static final long FORCE_DETACH_RETRY_WAIT_MS = 250;
|
||||
private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen";
|
||||
private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b");
|
||||
+ private static final int SECONDARY_RETRY_MAX_ATTEMPTS = 7;
|
||||
+ private static final long SECONDARY_RETRY_INITIAL_BACKOFF_MS = 250L;
|
||||
+ private static final long SECONDARY_RETRY_MAX_BACKOFF_MS = 2000L;
|
||||
+ private static final String DEVICE_HELD_OPEN_ERR_MSG = "Device is held open by someone";
|
||||
|
||||
private final ExtCmdFactory extCmdFactory;
|
||||
private final AccessContext sysCtx;
|
||||
@@ -945,6 +949,79 @@ public class DrbdAdm
|
||||
return true;
|
||||
}
|
||||
|
||||
+ /**
|
||||
+ * Switches a resource to secondary mode after mkfs, retrying when DRBD reports the device as held
|
||||
+ * open by another process. On Talos and other systems with active block-device probing
|
||||
+ * (machined/udev), the kernel uevent for a freshly-promoted device may cause an external probe
|
||||
+ * to open the device just before LINSTOR demotes it back to Secondary, producing
|
||||
+ * "Device is held open by someone" (errno -12) and aborting resource initialization.
|
||||
+ * Probes typically release the fd within milliseconds, so a short retry with exponential backoff
|
||||
+ * is sufficient to win the race without changing the contract of {@link #secondary}.
|
||||
+ */
|
||||
+ public void secondaryAfterMkfs(DrbdRscData<Resource> drbdRscData) throws ExtCmdFailedException
|
||||
+ {
|
||||
+ String[] commandArr = new String[] {
|
||||
+ DRBDADM_UTIL, "-vvv", "secondary", drbdRscData.getSuffixedResourceName()
|
||||
+ };
|
||||
+
|
||||
+ File nullDevice = new File(Platform.nullDevice());
|
||||
+
|
||||
+ OutputData lastOutput = null;
|
||||
+ for (int attempt = 1; attempt <= SECONDARY_RETRY_MAX_ATTEMPTS; attempt++)
|
||||
+ {
|
||||
+ try
|
||||
+ {
|
||||
+ ExtCmd extCmd = extCmdFactory.create();
|
||||
+ if (Platform.isWindows())
|
||||
+ {
|
||||
+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000);
|
||||
+ }
|
||||
+
|
||||
+ lastOutput = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr);
|
||||
+ if (lastOutput.exitCode == 0)
|
||||
+ {
|
||||
+ return;
|
||||
+ }
|
||||
+ if (!isDeviceHeldOpenError(lastOutput) || attempt == SECONDARY_RETRY_MAX_ATTEMPTS)
|
||||
+ {
|
||||
+ throw new ExtCmdFailedException(commandArr, lastOutput);
|
||||
+ }
|
||||
+ }
|
||||
+ catch (ChildProcessTimeoutException timeoutExc)
|
||||
+ {
|
||||
+ throw new ExtCmdFailedException(commandArr, timeoutExc);
|
||||
+ }
|
||||
+ catch (IOException ioExc)
|
||||
+ {
|
||||
+ throw new ExtCmdFailedException(commandArr, ioExc);
|
||||
+ }
|
||||
+
|
||||
+ long sleepMs = Math.min(
|
||||
+ SECONDARY_RETRY_INITIAL_BACKOFF_MS * (1L << (attempt - 1)),
|
||||
+ SECONDARY_RETRY_MAX_BACKOFF_MS
|
||||
+ );
|
||||
+ try
|
||||
+ {
|
||||
+ Thread.sleep(sleepMs);
|
||||
+ }
|
||||
+ catch (InterruptedException ie)
|
||||
+ {
|
||||
+ Thread.currentThread().interrupt();
|
||||
+ throw new ExtCmdFailedException(commandArr, lastOutput);
|
||||
+ }
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ static boolean isDeviceHeldOpenError(OutputData outputData)
|
||||
+ {
|
||||
+ if (outputData == null || outputData.stderrData == null)
|
||||
+ {
|
||||
+ return false;
|
||||
+ }
|
||||
+ String stderr = new String(outputData.stderrData, StandardCharsets.UTF_8);
|
||||
+ return stderr.contains(DEVICE_HELD_OPEN_ERR_MSG);
|
||||
+ }
|
||||
+
|
||||
public static class DrbdPrimary implements AutoCloseable
|
||||
{
|
||||
private final DrbdAdm drbdAdm;
|
||||
@@ -963,7 +1040,7 @@ public class DrbdAdm
|
||||
@Override
|
||||
public void close() throws ExtCmdFailedException
|
||||
{
|
||||
- drbdAdm.secondary(drbdRscData);
|
||||
+ drbdAdm.secondaryAfterMkfs(drbdRscData);
|
||||
}
|
||||
}
|
||||
}
|
||||
-67
@@ -1,67 +0,0 @@
|
||||
diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java
|
||||
index a639c27cc..4dabc1a83 100644
|
||||
--- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java
|
||||
+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java
|
||||
@@ -434,26 +434,39 @@ public class CtrlSnapshotRestoreApiCallHandler
|
||||
}
|
||||
}
|
||||
|
||||
- /*
|
||||
- * When a snapshot was received as a backup, it is marked to re-create its local metadata. Before the
|
||||
- * snapshot was sent the new DRBD_INITIALIZED flag and the VlmDfn property InitialUpToDateOn were
|
||||
+ /**
|
||||
+ * <p>Sets the {@value InternalApiConsts#KEY_LINSTOR_DRBD_INITIAL_UPTODATE_ON} property to the nodeName
|
||||
+ * of the first Snapshot (which has to have a disk) of the given SnapshotDefinition.</p>
|
||||
+ *
|
||||
+ * <p>If this SnapshotDefinition is a backup, we also unset the {@link VolumeDefinition.Flags#DRBD_INITIALIZED}
|
||||
+ * flag. Otherwise we ensure the same flag is set.</p>
|
||||
+ *
|
||||
+ * <p>Backup path: When a snapshot was received as a backup, it is marked to re-create its local metadata.
|
||||
+ * Before the snapshot was sent the new DRBD_INITIALIZED flag and the VlmDfn property InitialUpToDateOn were
|
||||
* cleared. If DRBD_INITIALZED would be set, no peer would even read the property (on purpose).
|
||||
* Without the property, no peer would think of itself as a winner, thus the DRBD resource would never
|
||||
* become UpToDate.
|
||||
* To avoid this, this method sets the InitialUpToDateOn property to let the first resource win the
|
||||
- * race, so it can become UpToDate and therefore also pull the other peers eventually into UpToDate state.
|
||||
+ * race, so it can become UpToDate and therefore also pull the other peers eventually into UpToDate state. </p>
|
||||
*
|
||||
- * If the given SnapDfn had also non-backup snapshots, those should be able to keep their metadata.
|
||||
+ * <p>If the given SnapDfn had also non-backup snapshots, those should be able to keep their metadata.
|
||||
* In that case we do not have to do anything. If however all snapshots are descendants from backups
|
||||
* we must pick one and decide the UpToDate race upfront so that the winner can set its local DRBD
|
||||
- * as UpToDate, which also triggers the controller to set the DRBD_INITIALZED flag eventually.
|
||||
+ * as UpToDate, which also triggers the controller to set the DRBD_INITIALZED flag eventually.</p>
|
||||
+ *
|
||||
+ * <p>Non-Backup path: When a regular snapshot (i.e. not from a backup) is restored, we can or even should
|
||||
+ * assume that the backing snapshot LVs have correct metadata which do not need be be initialized. When
|
||||
+ * a SnapshotVolumeDefinition is created it unfortunately does not properly store the VolumeDefinition's flags,
|
||||
+ * therefore the restored VolumeDefinition will also not have the DRBD_INITIALIZED flag set, although it
|
||||
+ * refers to a device with healthy and UpToDate metadata. To fix this issue we simply set the flag in this case.</p>
|
||||
*/
|
||||
private void checkDrbdInitializedState(ResourceDefinition rscDfnRef) throws AccessDeniedException
|
||||
{
|
||||
AccessContext accCtx = peerAccCtx.get();
|
||||
- if (DrbdLayerUtils.isForceInitialSyncSet(accCtx, rscDfnRef) && rscDfnRef.getDiskfulCount(accCtx) > 0)
|
||||
+ if (rscDfnRef.getDiskfulCount(accCtx) > 0)
|
||||
{
|
||||
String winnerNodeName = rscDfnRef.getDiskfulResources(accCtx).get(0).getNode().getName().value;
|
||||
+ boolean uninitializeDrbd = DrbdLayerUtils.isForceInitialSyncSet(accCtx, rscDfnRef);
|
||||
Iterator<VolumeDefinition> vlmDfnIt = rscDfnRef.iterateVolumeDfn(accCtx);
|
||||
while (vlmDfnIt.hasNext())
|
||||
{
|
||||
@@ -462,7 +475,14 @@ public class CtrlSnapshotRestoreApiCallHandler
|
||||
{
|
||||
vlmDfn.getProps(accCtx)
|
||||
.setProp(InternalApiConsts.KEY_LINSTOR_DRBD_INITIAL_UPTODATE_ON, winnerNodeName);
|
||||
- vlmDfn.getFlags().disableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED);
|
||||
+ if (uninitializeDrbd)
|
||||
+ {
|
||||
+ vlmDfn.getFlags().disableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED);
|
||||
+ }
|
||||
+ else
|
||||
+ {
|
||||
+ vlmDfn.getFlags().enableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED);
|
||||
+ }
|
||||
}
|
||||
catch (InvalidKeyException | InvalidValueException exc)
|
||||
{
|
||||
@@ -36,6 +36,12 @@ spec:
|
||||
enabled: true
|
||||
podTemplate:
|
||||
spec:
|
||||
{{- /* run-migration upgrades the database in place: left on the
|
||||
operator's default image it migrates ahead of the controller,
|
||||
irreversibly. */}}
|
||||
initContainers:
|
||||
- name: run-migration
|
||||
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
|
||||
containers:
|
||||
- name: linstor-controller
|
||||
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
|
||||
|
||||
@@ -11,6 +11,9 @@ spec:
|
||||
spec:
|
||||
# host-network is recommended by Piraeus while it is not default in the upstream
|
||||
hostNetwork: true
|
||||
initContainers:
|
||||
- name: setup-lvm-configuration
|
||||
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
|
||||
containers:
|
||||
- name: linstor-satellite
|
||||
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
|
||||
|
||||
Loaded 100 of 129 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user