Files
cozystack/hack/e2e-apps/monitoring-oidc-customconfig.bats
T
IvanHunters 7f9e1a8bf6 test(e2e): poll the reconciled Grafana CR field instead of racing on CR existence
CI run 28794778739 caught two race conditions in the render-side
Monitoring OIDC bats:

- `Grafana CR carries auth.generic_oauth pointing at the cozy realm`
  (monitoring-oidc-system.bats:86) failed with `enabled=` empty after
  a 0-second wait. The prior `until kubectl get grafana grafana`
  only proved the CR EXISTS — it did not prove grafana-operator had
  reconciled the mode=System values onto it. When the CR from the
  previous `Monitoring CR accepts spec.oidc.mode=System` test was
  already there but the operator hadn't yet flipped `.spec.config.auth
  .generic_oauth.enabled` to "true", the jsonpath returned empty and
  `[ "" = "true" ]` short-circuited the whole test.
- `secretRef variant mounts operator Secret under /etc/grafana/oidc`
  (monitoring-oidc-customconfig.bats:119) timed out at 180s waiting
  for the volumeMount swap from inline → ini. On a warm sandbox the
  full inner-HR upgrade + grafana-operator reconcile cycle can
  legitimately take >3m — the timeout was too tight.

Fix: replace `until kubectl get grafana` with `until [ "$(kubectl
... jsonpath=<target-field>)" = <expected> ]` on every subsequent
assertion, and bump the secretRef-swap window to 600s (well inside
the inner HR's 15m timeout). Same pattern used for the "no
role_attribute_path" and "operator payload verbatim" assertions so
the whole suite is race-free.

Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
2026-07-15 15:31:45 +03:00

203 lines
8.0 KiB
Bash
Executable File

#!/usr/bin/env bats
# Phase-1 OIDC CustomConfig selector e2e for the Grafana instance,
# render-side only.
#
# What is exercised on a live cluster: that the new
# `spec.oidc.mode: CustomConfig` field is admitted; that cozystack-api
# accepts the updated schema; that the HelmRelease renders; that the
# operator-supplied auth.generic_oauth payload lands on the Grafana
# CR; and, crucially, that NO Keycloak objects are created in the
# `cozy` realm when the tenant brings their own issuer.
# cozytest.sh (the e2e runner) is not real bats: it never invokes
# setup()/teardown(). Cleanup belongs in cozy_cleanup(), which runs at
# suite exit and on the first failing test. Per-test isolation is done
# inline at the top of each @test.
#
# Naming: the extra/monitoring chart is a singleton per tenant
# namespace and enforces `.Release.Name == "monitoring"` in
# templates/check-release-name.yaml. So the Monitoring CR (and its
# operator-produced outer HelmRelease) MUST be named `monitoring`;
# the inner packages/system/monitoring HelmRelease that carries the
# OIDC templates is then named `${outer}-system`, i.e. `monitoring-system`.
CR_NAME="monitoring"
INNER_REL="${CR_NAME}-system"
BYO_SECRET="mon-oidc-byo-oauth"
cleanup_mon() {
kubectl -n tenant-test delete monitoring.apps.cozystack.io "${CR_NAME}" \
--ignore-not-found --wait=false 2>/dev/null || true
kubectl -n tenant-test wait monitoring.apps.cozystack.io "${CR_NAME}" \
--for=delete --timeout=2m 2>/dev/null || true
kubectl -n tenant-test delete secret "${BYO_SECRET}" \
--ignore-not-found 2>/dev/null || true
}
cozy_cleanup() { cleanup_mon; }
@test "Monitoring CR accepts spec.oidc.mode=CustomConfig with inline config" {
cleanup_mon
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Monitoring
metadata:
name: ${CR_NAME}
namespace: tenant-test
spec:
host: ""
metricsStorages:
- name: shortterm
retentionPeriod: "3d"
deduplicationInterval: "15s"
storage: 10Gi
storageClassName: replicated
logsStorages:
- name: generic
retentionPeriod: "1"
storage: 10Gi
storageClassName: replicated
oidc:
mode: CustomConfig
customConfig:
config:
client_id: byo-monitoring
client_secret: byo-secret
auth_url: https://idp.example.test/auth
token_url: https://idp.example.test/token
api_url: https://idp.example.test/userinfo
users:
- email: e2e-byo@example.com
role: Viewer
EOF
# Outer HR (created by cozystack-operator from the CR) shares the CR name.
timeout 60 sh -ec 'until kubectl -n tenant-test get hr "'"${CR_NAME}"'" >/dev/null 2>&1; do sleep 2; done'
# Inner HR (rendered by extra/monitoring) is ${outer}-system.
timeout 120 sh -ec 'until kubectl -n tenant-test get hr "'"${INNER_REL}"'" >/dev/null 2>&1; do sleep 2; done'
timeout 180 sh -ec 'until kubectl -n tenant-test get grafana grafana >/dev/null 2>&1; do sleep 5; done'
}
@test "Grafana CR carries the operator's auth.generic_oauth payload verbatim" {
# Poll the specific field, not just CR existence — otherwise we may
# grab a stale Grafana CR that hasn't yet been reconciled with the
# inner HR's CustomConfig values.
timeout 180 sh -ec '
until [ "$(kubectl -n tenant-test get grafana grafana \
-o jsonpath="{.spec.config.auth\.generic_oauth.client_id}" 2>/dev/null)" = "byo-monitoring" ]; do
sleep 5
done
'
client_id=$(kubectl -n tenant-test get grafana grafana \
-o jsonpath='{.spec.config.auth\.generic_oauth.client_id}')
echo "client_id: ${client_id}"
[ "${client_id}" = "byo-monitoring" ]
auth_url=$(kubectl -n tenant-test get grafana grafana \
-o jsonpath='{.spec.config.auth\.generic_oauth.auth_url}')
echo "auth_url: ${auth_url}"
[ "${auth_url}" = "https://idp.example.test/auth" ]
# No cozy realm in the picture.
echo "${auth_url}" | grep -vq "realms/cozy" || {
echo "auth_url unexpectedly references the cozy realm"
return 1
}
}
@test "No KeycloakClient / Scope are created in the cozy realm" {
# Only meaningful when the EDP CRDs exist on the cluster.
if ! kubectl api-resources --api-group=v1.edp.epam.com >/dev/null 2>&1; then
skip "EDP Keycloak operator CRDs not present on this cluster"
fi
# clientId helper: printf "%s-%s" .Release.Namespace .Release.Name in
# the inner chart -> tenant-test-monitoring-system.
CID="tenant-test-${INNER_REL}"
# Give any reconciler a moment; then assert neither cozy-realm object
# exists. Chart-owned KeycloakRealmGroups were removed entirely (see
# design note in docs/oidc-grafana.md) so no group assertions here.
sleep 5
! kubectl -n tenant-test get keycloakclient.v1.edp.epam.com "${CID}" 2>/dev/null
! kubectl -n tenant-test get keycloakclientscope.v1.edp.epam.com "${CID}-audience" 2>/dev/null
}
@test "secretRef variant mounts operator Secret under /etc/grafana/oidc" {
# Operator-owned Secret carrying a ready-made ini fragment.
kubectl -n tenant-test create secret generic "${BYO_SECRET}" \
--from-literal=auth.ini='[auth.generic_oauth]
enabled = true
name = ByoIdp
client_id = byo-monitoring
client_secret = byo-secret
' --dry-run=client -o yaml | kubectl apply -f -
# Update the Monitoring CR in place: swap customConfig.config (inline)
# for customConfig.secretRef.name. Delete + immediate apply races
# kubectl's create-vs-patch decision against the API server's grace-
# period cache and can hit "NotFound" during patch, so let the operator
# reconcile the mode change on the same CR instead.
kubectl apply -f - <<EOF
apiVersion: apps.cozystack.io/v1alpha1
kind: Monitoring
metadata:
name: ${CR_NAME}
namespace: tenant-test
spec:
host: ""
metricsStorages:
- name: shortterm
retentionPeriod: "3d"
deduplicationInterval: "15s"
storage: 10Gi
storageClassName: replicated
logsStorages:
- name: generic
retentionPeriod: "1"
storage: 10Gi
storageClassName: replicated
oidc:
mode: CustomConfig
customConfig:
secretRef:
name: ${BYO_SECRET}
EOF
timeout 60 sh -ec 'until kubectl -n tenant-test get hr "'"${CR_NAME}"'" >/dev/null 2>&1; do sleep 2; done'
timeout 120 sh -ec 'until kubectl -n tenant-test get hr "'"${INNER_REL}"'" >/dev/null 2>&1; do sleep 2; done'
timeout 180 sh -ec 'until kubectl -n tenant-test get grafana grafana >/dev/null 2>&1; do sleep 5; done'
# The Grafana CR from @test 1 already exists and carries the inline
# auth.generic_oauth block; wait for the chart to reconcile the swap
# to secretRef by polling for the target state (volume mount present
# AND inline block absent) instead of asserting immediately. 600s
# window is sized for a full inner-HR upgrade cycle + grafana-
# operator reconcile — 180s is too tight when the sandbox is warm-
# loaded, cf. the intermittent CI failure.
timeout 600 sh -ec '
until [ -n "$(kubectl -n tenant-test get grafana grafana -o jsonpath="{.spec.deployment.spec.template.spec.containers[?(@.name==\"grafana\")].volumeMounts[?(@.name==\"oidc-custom-ini\")].mountPath}")" ]; do sleep 5; done
'
timeout 120 sh -ec '
until [ -z "$(kubectl -n tenant-test get grafana grafana -o jsonpath="{.spec.config.auth\.generic_oauth}")" ]; do sleep 5; done
'
# spec.config carries no auth.generic_oauth section in this branch.
section=$(kubectl -n tenant-test get grafana grafana \
-o jsonpath='{.spec.config.auth\.generic_oauth}')
echo "spec.config.auth.generic_oauth: ${section:-<empty>}"
[ -z "${section}" ]
# The volume + mount + env are on the Grafana CR's deployment override.
mount=$(kubectl -n tenant-test get grafana grafana \
-o jsonpath='{.spec.deployment.spec.template.spec.containers[?(@.name=="grafana")].volumeMounts[?(@.name=="oidc-custom-ini")].mountPath}')
echo "mount: ${mount}"
[ "${mount}" = "/etc/grafana/oidc" ]
vol=$(kubectl -n tenant-test get grafana grafana \
-o jsonpath='{.spec.deployment.spec.template.spec.volumes[?(@.name=="oidc-custom-ini")].secret.secretName}')
echo "secret volume: ${vol}"
[ "${vol}" = "${BYO_SECRET}" ]
}