mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
feat(observability-proxy): add metrics store and cost limit flags
The store names and the short-term retention mirror metricsStorages in the monitoring package, which an operator can rename or resize; they are flags so the chart can follow such a change without a rebuild. The cost limit defaults are the ones the metrics proposal settles on: 30d range, 11000 points per series, 30s timeout. Store names become part of a Service host name, so they are checked as DNS labels at startup. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
47409db70c
commit
cd3d889ee8
2 files changed
+44
-3
No files matched your search
@@ -14,9 +14,10 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Command observability-proxy serves tenant logs from the VictoriaLogs
|
||||
// stores behind a Kubernetes TokenReview and SubjectAccessReview, with the
|
||||
// tenant's namespace scope forced into every upstream request.
|
||||
// Command observability-proxy serves tenant logs and metrics from the
|
||||
// VictoriaLogs and VictoriaMetrics stores behind a Kubernetes TokenReview
|
||||
// and SubjectAccessReview, with the tenant's namespace scope forced into
|
||||
// every upstream request.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -37,6 +38,7 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"k8s.io/apimachinery/pkg/util/validation"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
@@ -56,6 +58,7 @@ type options struct {
|
||||
tailIdleTimeout time.Duration
|
||||
shutdownTimeout time.Duration
|
||||
logs observabilityproxy.LogsLimits
|
||||
metrics observabilityproxy.MetricsConfig
|
||||
limiter observabilityproxy.LimiterConfig
|
||||
}
|
||||
|
||||
@@ -81,6 +84,14 @@ func parseFlags(args []string) (options, error) {
|
||||
fs.IntVar(&o.logs.DefaultLimit, "logs-default-limit", 1000, "entries returned when a request sets no limit")
|
||||
fs.IntVar(&o.logs.MaxLimit, "logs-max-limit", 10000, "largest limit a request may set")
|
||||
fs.DurationVar(&o.logs.MaxTailResume, "logs-max-tail-resume", 15*time.Minute, "how far back a tail may resume with since or start_offset")
|
||||
// Store names and retention follow metricsStorages in
|
||||
// packages/system/monitoring/values.yaml.
|
||||
fs.StringVar(&o.metrics.ShortTermStore, "metrics-shortterm-store", "shortterm", "VMCluster name of the short-term metrics store of the root tenant")
|
||||
fs.StringVar(&o.metrics.LongTermStore, "metrics-longterm-store", "longterm", "VMCluster name of the long-term metrics store of the root tenant")
|
||||
fs.DurationVar(&o.metrics.ShortTermRetention, "metrics-shortterm-retention", 72*time.Hour, "retention of the short-term store; older windows are read from the long-term one")
|
||||
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
|
||||
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
|
||||
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped")
|
||||
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
|
||||
fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables")
|
||||
fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user")
|
||||
@@ -94,6 +105,13 @@ func parseFlags(args []string) (options, error) {
|
||||
if fs.NArg() > 0 {
|
||||
return o, fmt.Errorf("unexpected argument %q", fs.Arg(0))
|
||||
}
|
||||
o.metrics.RootTenant = o.rootTenant
|
||||
for _, store := range []string{o.metrics.ShortTermStore, o.metrics.LongTermStore} {
|
||||
// The name becomes part of a Service host name.
|
||||
if errs := validation.IsDNS1123Label("vmselect-" + store); store == "" || len(errs) > 0 {
|
||||
return o, fmt.Errorf("invalid metrics store name %q", store)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case o.rootTenant == "":
|
||||
return o, errors.New("--root-tenant must not be empty")
|
||||
@@ -110,6 +128,10 @@ func parseFlags(args []string) (options, error) {
|
||||
return o, errors.New("--logs-max-range must be at least 3s")
|
||||
case o.logs.DefaultLimit <= 0 || o.logs.MaxLimit < o.logs.DefaultLimit:
|
||||
return o, errors.New("--logs-default-limit must be positive and not above --logs-max-limit")
|
||||
case o.metrics.ShortTermRetention <= 0 || o.metrics.Limits.MaxRange <= 0 || o.metrics.Limits.MaxTimeout <= 0:
|
||||
return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive")
|
||||
case o.metrics.Limits.MaxPoints < 2:
|
||||
return o, errors.New("--metrics-max-points must be at least 2")
|
||||
case o.logs.MaxTailResume < 0:
|
||||
return o, errors.New("--logs-max-tail-resume must not be negative")
|
||||
case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0:
|
||||
@@ -167,6 +189,7 @@ func run(o options) error {
|
||||
proxy := observabilityproxy.NewServer(observabilityproxy.Config{
|
||||
ClusterDomain: o.clusterDomain,
|
||||
Logs: o.logs,
|
||||
Metrics: o.metrics,
|
||||
Limiter: o.limiter,
|
||||
RequestTimeout: o.requestTimeout,
|
||||
TailIdleTimeout: o.tailIdleTimeout,
|
||||
|
||||
@@ -32,6 +32,18 @@ func TestParseFlagsDefaults(t *testing.T) {
|
||||
if o.authCacheTTL != 10*time.Second || o.logs.MaxTailResume != 15*time.Minute || o.logs.MaxRange != 7*24*time.Hour {
|
||||
t.Errorf("limit defaults changed: %+v", o)
|
||||
}
|
||||
// Retention and store names of packages/system/monitoring/values.yaml;
|
||||
// cost limits of the metrics proposal (Design 7).
|
||||
m := o.metrics
|
||||
if m.ShortTermStore != "shortterm" || m.LongTermStore != "longterm" || m.ShortTermRetention != 72*time.Hour {
|
||||
t.Errorf("metrics store defaults changed: %+v", m)
|
||||
}
|
||||
if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second {
|
||||
t.Errorf("metrics limit defaults changed: %+v", m.Limits)
|
||||
}
|
||||
if m.RootTenant != o.rootTenant {
|
||||
t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFlagsOverrides(t *testing.T) {
|
||||
@@ -69,6 +81,12 @@ func TestParseFlagsInvalid(t *testing.T) {
|
||||
{"--max-concurrent-per-user=-1"},
|
||||
{"--max-tails-per-user=-1"},
|
||||
{"--max-tails=-1"},
|
||||
{"--metrics-shortterm-store="},
|
||||
{"--metrics-longterm-store=Long.Term"},
|
||||
{"--metrics-shortterm-retention=0s"},
|
||||
{"--metrics-max-range=0s"},
|
||||
{"--metrics-max-points=1"},
|
||||
{"--metrics-max-timeout=0s"},
|
||||
{"--unknown"},
|
||||
{"unexpected", "--root-tenant=tenant-x"},
|
||||
} {
|
||||
|
||||
Reference in new issue
Block a user