mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
Private infrastructure names, a real hostname, a commented-out internal mirror block, an internal product name, and a reviewer's name in a comment do not belong in a public repository; drop them and replace what needs a stand-in with neutral placeholders. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
1774 lines
77 KiB
Go
1774 lines
77 KiB
Go
/*
|
|
Copyright 2025 The Cozystack Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package operator
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"reflect"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
|
|
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
|
sourcewatcherv1beta1 "github.com/fluxcd/source-watcher/api/v2/v1beta1"
|
|
"k8s.io/apimachinery/pkg/api/meta"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
|
)
|
|
|
|
// referenceTime is a fixed anchor for time-sensitive tests; every duration in
|
|
// the assertions is expressed as an offset from it so the tests read as
|
|
// "given the AG was last touched at reference-8s, and now is reference, ...".
|
|
var referenceTime = time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC)
|
|
|
|
// agFixture builds an ArtifactGenerator with a fixed .metadata.generation and
|
|
// creation timestamp so tests can exercise the Generation and grace-period
|
|
// branches of artifactGeneratorStuck without a full apiserver round-trip.
|
|
func agFixture(gen int64, createdOffset time.Duration, status sourcewatcherv1beta1.ArtifactGeneratorStatus) *sourcewatcherv1beta1.ArtifactGenerator {
|
|
return &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Generation: gen,
|
|
CreationTimestamp: metav1.NewTime(referenceTime.Add(createdOffset)),
|
|
},
|
|
Status: status,
|
|
}
|
|
}
|
|
|
|
// TestArtifactGeneratorStuck locks in detection of the fluxcd/pkg#934 split
|
|
// -write stall. The predicate must fire ONLY when artifacts are demonstrably
|
|
// produced on the current generation AND the Ready condition has been sitting
|
|
// in Unknown longer than the grace period (or is missing on an old-enough AG).
|
|
// Ready=True and Ready=False must both pass through untouched — the retry
|
|
// driver only runs for the stuck-Unknown case, and a real regeneration failure
|
|
// (Ready=False) must never be papered over.
|
|
func TestArtifactGeneratorStuck(t *testing.T) {
|
|
nonEmptyInventory := []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
}
|
|
const observedDigest = "sha256:0e7"
|
|
|
|
// beyondGrace: how far into Unknown the AG has been before we'll intervene.
|
|
beyondGrace := stuckGracePeriod + 5*time.Second
|
|
|
|
tests := []struct {
|
|
name string
|
|
ag *sourcewatcherv1beta1.ArtifactGenerator
|
|
want bool
|
|
}{
|
|
{
|
|
name: "fresh ArtifactGenerator with nothing observed yet",
|
|
ag: agFixture(1, -time.Hour, sourcewatcherv1beta1.ArtifactGeneratorStatus{}),
|
|
want: false,
|
|
},
|
|
{
|
|
name: "inventory present but sources not yet observed",
|
|
ag: agFixture(1, -time.Hour, sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: nonEmptyInventory,
|
|
}),
|
|
want: false,
|
|
},
|
|
{
|
|
name: "sources observed but inventory empty",
|
|
ag: agFixture(1, -time.Hour, sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
ObservedSourcesDigest: observedDigest,
|
|
}),
|
|
want: false,
|
|
},
|
|
{
|
|
name: "artifacts produced, Ready condition absent, AG within grace period — do NOT intervene yet",
|
|
ag: agFixture(1, -5*time.Second, sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: nonEmptyInventory,
|
|
ObservedSourcesDigest: observedDigest,
|
|
}),
|
|
want: false,
|
|
},
|
|
{
|
|
name: "artifacts produced, Ready condition absent, AG older than grace period — intervene",
|
|
ag: agFixture(1, -beyondGrace, sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: nonEmptyInventory,
|
|
ObservedSourcesDigest: observedDigest,
|
|
}),
|
|
want: true,
|
|
},
|
|
{
|
|
name: "artifacts produced, Ready=Unknown within grace period — legitimate in-flight rebuild, do NOT intervene",
|
|
ag: agFixture(1, -time.Hour, sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: nonEmptyInventory,
|
|
ObservedSourcesDigest: observedDigest,
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionUnknown,
|
|
Reason: "Progressing",
|
|
Message: "Reconciliation in progress",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(referenceTime.Add(-5 * time.Second)),
|
|
}},
|
|
}),
|
|
want: false,
|
|
},
|
|
{
|
|
name: "artifacts produced, Ready=Unknown beyond grace period on current Generation — the stuck-status case",
|
|
ag: agFixture(1, -time.Hour, sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: nonEmptyInventory,
|
|
ObservedSourcesDigest: observedDigest,
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionUnknown,
|
|
Reason: "Progressing",
|
|
Message: "Reconciliation in progress",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(referenceTime.Add(-beyondGrace)),
|
|
}},
|
|
}),
|
|
want: true,
|
|
},
|
|
{
|
|
name: "artifacts observed on a PRIOR Generation (spec was updated) — do NOT intervene, condition is legitimately stale",
|
|
ag: agFixture(2, -time.Hour, sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: nonEmptyInventory,
|
|
ObservedSourcesDigest: observedDigest,
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionUnknown,
|
|
Reason: "Progressing",
|
|
Message: "Reconciliation in progress",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(referenceTime.Add(-beyondGrace)),
|
|
}},
|
|
}),
|
|
want: false,
|
|
},
|
|
{
|
|
name: "upstream Ready=True — pass through the real condition, do NOT intervene",
|
|
ag: agFixture(1, -time.Hour, sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: nonEmptyInventory,
|
|
ObservedSourcesDigest: observedDigest,
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionTrue,
|
|
Reason: "Succeeded",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(referenceTime.Add(-beyondGrace)),
|
|
}},
|
|
}),
|
|
want: false,
|
|
},
|
|
{
|
|
name: "upstream Ready=False — real failure, must NOT be masked",
|
|
ag: agFixture(1, -time.Hour, sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: nonEmptyInventory,
|
|
ObservedSourcesDigest: observedDigest,
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionFalse,
|
|
Reason: "SourceRefFailed",
|
|
Message: "cannot fetch source",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(referenceTime.Add(-beyondGrace)),
|
|
}},
|
|
}),
|
|
want: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
ready := meta.FindStatusCondition(tt.ag.Status.Conditions, "Ready")
|
|
got := artifactGeneratorStuck(tt.ag, ready, referenceTime)
|
|
if got != tt.want {
|
|
t.Errorf("artifactGeneratorStuck() = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestDecideRecovery locks in the bounded-retry state machine that drives
|
|
// source-watcher out of the stuck window. First attempt forces immediately;
|
|
// the N-th subsequent attempt waits backoffFor(N) since the last force before
|
|
// firing again; after maxRecoveryAttempts fruitless attempts we surface
|
|
// Ready=False. Wait durations are asserted for exact equality because
|
|
// decideRecovery is deterministic (`needed - elapsed`) — a loose bound would
|
|
// silently miss a backoff-schedule regression.
|
|
func TestDecideRecovery(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
attempts int
|
|
lastRecoveryAt time.Time
|
|
now time.Time
|
|
wantAction recoveryAction
|
|
wantWait time.Duration // 0 means don't check (Force / GiveUp branches ignore wait)
|
|
}{
|
|
{
|
|
name: "first-ever detection — force immediately",
|
|
attempts: 0,
|
|
now: referenceTime,
|
|
wantAction: recoveryActionForce,
|
|
},
|
|
{
|
|
name: "backoff not yet elapsed after attempt 1 — wait",
|
|
attempts: 1,
|
|
lastRecoveryAt: referenceTime.Add(-10 * time.Second),
|
|
now: referenceTime,
|
|
wantAction: recoveryActionWait,
|
|
wantWait: 20 * time.Second, // initialBackoff (30s) minus elapsed 10s
|
|
},
|
|
{
|
|
name: "backoff elapsed after attempt 1 — force",
|
|
attempts: 1,
|
|
lastRecoveryAt: referenceTime.Add(-45 * time.Second),
|
|
now: referenceTime,
|
|
wantAction: recoveryActionForce,
|
|
},
|
|
{
|
|
name: "backoff after attempt 3 (2m) — wait 90s",
|
|
attempts: 3,
|
|
lastRecoveryAt: referenceTime.Add(-30 * time.Second),
|
|
now: referenceTime,
|
|
wantAction: recoveryActionWait,
|
|
wantWait: 90 * time.Second, // backoffFor(3)=2m minus elapsed 30s
|
|
},
|
|
{
|
|
name: "attempts exhausted with elapsed >> final backoff — give up",
|
|
attempts: maxRecoveryAttempts,
|
|
now: referenceTime,
|
|
wantAction: recoveryActionGiveUp,
|
|
},
|
|
{
|
|
name: "attempts exceeded — still give up",
|
|
attempts: maxRecoveryAttempts + 3,
|
|
now: referenceTime,
|
|
wantAction: recoveryActionGiveUp,
|
|
},
|
|
{
|
|
// lexfrei's follow-up: without holding the final backoff window,
|
|
// the Owns()-watch re-fire from the N-th force-drift's own
|
|
// writes preempts the response window and give-up fires within
|
|
// milliseconds of the last force. Budget collapses to ~7.5m
|
|
// instead of the documented ~11.5m. This case pins the fix:
|
|
// attempts=max but no time elapsed since the final force,
|
|
// expect Wait for the full backoffFor(maxRecoveryAttempts).
|
|
name: "attempts exhausted but final backoff not elapsed — wait, do not give up",
|
|
attempts: maxRecoveryAttempts,
|
|
lastRecoveryAt: referenceTime.Add(-time.Second),
|
|
now: referenceTime,
|
|
wantAction: recoveryActionWait,
|
|
wantWait: backoffFor(maxRecoveryAttempts) - time.Second,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := decideRecovery(tt.attempts, tt.lastRecoveryAt, tt.now)
|
|
if got.action != tt.wantAction {
|
|
t.Fatalf("action = %v, want %v", got.action, tt.wantAction)
|
|
}
|
|
if tt.wantWait > 0 && got.wait != tt.wantWait {
|
|
t.Errorf("wait = %v, want %v", got.wait, tt.wantWait)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestBackoffFor pins the exponential-with-cap schedule so a future edit that
|
|
// silently doubles the ceiling or drops the exponent gets caught here.
|
|
func TestBackoffFor(t *testing.T) {
|
|
// initialBackoff=30s, maxBackoff=4m. Expected: 30s, 60s, 120s, 240s, 240s, 240s, ...
|
|
tests := []struct {
|
|
attempt int
|
|
want time.Duration
|
|
}{
|
|
{0, 30 * time.Second}, // clamped to attempt=1
|
|
{1, 30 * time.Second},
|
|
{2, 60 * time.Second},
|
|
{3, 120 * time.Second},
|
|
{4, 240 * time.Second},
|
|
{5, 240 * time.Second},
|
|
{10, 240 * time.Second},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(strconv.Itoa(tt.attempt), func(t *testing.T) {
|
|
got := backoffFor(tt.attempt)
|
|
if got != tt.want {
|
|
t.Errorf("backoffFor(%d) = %v, want %v", tt.attempt, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestReadRecoveryTracking verifies that a corrupted or malformed retry-tracking
|
|
// annotation cannot wedge the retry loop by producing a nonsensical counter —
|
|
// missing, empty, non-numeric, or negative values must all read back as
|
|
// "no prior attempts".
|
|
func TestReadRecoveryTracking(t *testing.T) {
|
|
fixed := referenceTime.UTC().Format(time.RFC3339Nano)
|
|
|
|
tests := []struct {
|
|
name string
|
|
annotations map[string]string
|
|
wantAttempt int
|
|
wantTimeSet bool
|
|
}{
|
|
{
|
|
name: "no annotations",
|
|
annotations: nil,
|
|
wantAttempt: 0,
|
|
wantTimeSet: false,
|
|
},
|
|
{
|
|
name: "valid attempt + valid timestamp",
|
|
annotations: map[string]string{
|
|
annotationRecoveryAttempts: "3",
|
|
annotationLastRecoveryAt: fixed,
|
|
},
|
|
wantAttempt: 3,
|
|
wantTimeSet: true,
|
|
},
|
|
{
|
|
name: "corrupt attempt counter — read as zero",
|
|
annotations: map[string]string{
|
|
annotationRecoveryAttempts: "not-a-number",
|
|
annotationLastRecoveryAt: fixed,
|
|
},
|
|
wantAttempt: 0,
|
|
wantTimeSet: true,
|
|
},
|
|
{
|
|
name: "negative attempt — clamped to zero",
|
|
annotations: map[string]string{
|
|
annotationRecoveryAttempts: "-5",
|
|
},
|
|
wantAttempt: 0,
|
|
wantTimeSet: false,
|
|
},
|
|
{
|
|
name: "corrupt timestamp — read as zero time",
|
|
annotations: map[string]string{
|
|
annotationRecoveryAttempts: "2",
|
|
annotationLastRecoveryAt: "not-a-date",
|
|
},
|
|
wantAttempt: 2,
|
|
wantTimeSet: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{Annotations: tt.annotations},
|
|
}
|
|
attempt, ts := readRecoveryTracking(ag)
|
|
if attempt != tt.wantAttempt {
|
|
t.Errorf("attempt = %d, want %d", attempt, tt.wantAttempt)
|
|
}
|
|
if tt.wantTimeSet && ts.IsZero() {
|
|
t.Error("timestamp is zero, want set")
|
|
}
|
|
if !tt.wantTimeSet && !ts.IsZero() {
|
|
t.Errorf("timestamp = %v, want zero", ts)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestCloneAnnotations pins the "shallow copy, nil in / nil out" contract that
|
|
// the rollback path in forceArtifactGeneratorDrift / clearRecoveryTracking
|
|
// depends on. If a future refactor "normalises" nil to an empty map, the
|
|
// rollback would leave the caller with an empty map instead of the original
|
|
// nil, subtly changing observable state in downstream code that treats
|
|
// annotations == nil as a distinct signal.
|
|
func TestCloneAnnotations(t *testing.T) {
|
|
t.Run("nil in, nil out", func(t *testing.T) {
|
|
if got := cloneAnnotations(nil); got != nil {
|
|
t.Errorf("cloneAnnotations(nil) = %v, want nil", got)
|
|
}
|
|
})
|
|
t.Run("empty in, non-nil empty out", func(t *testing.T) {
|
|
got := cloneAnnotations(map[string]string{})
|
|
if got == nil {
|
|
t.Error("cloneAnnotations(empty) returned nil, want empty map")
|
|
}
|
|
if len(got) != 0 {
|
|
t.Errorf("cloneAnnotations(empty) len = %d, want 0", len(got))
|
|
}
|
|
})
|
|
t.Run("populated in, independent copy out", func(t *testing.T) {
|
|
src := map[string]string{"a": "1", "b": "2"}
|
|
got := cloneAnnotations(src)
|
|
if len(got) != 2 || got["a"] != "1" || got["b"] != "2" {
|
|
t.Errorf("cloneAnnotations(populated) = %v, want copy of %v", got, src)
|
|
}
|
|
// Mutation-independence: modifying the source must not touch the copy.
|
|
src["a"] = "changed"
|
|
if got["a"] != "1" {
|
|
t.Errorf("clone was aliased to source; got[a]=%q after src[a]=changed", got["a"])
|
|
}
|
|
})
|
|
}
|
|
|
|
// testScheme is a shared scheme with the CRDs both reconciler test suites need.
|
|
func testScheme(t *testing.T) *runtime.Scheme {
|
|
t.Helper()
|
|
s := runtime.NewScheme()
|
|
if err := cozyv1alpha1.AddToScheme(s); err != nil {
|
|
t.Fatalf("cozyv1alpha1.AddToScheme: %v", err)
|
|
}
|
|
if err := sourcewatcherv1beta1.AddToScheme(s); err != nil {
|
|
t.Fatalf("sourcewatcherv1beta1.AddToScheme: %v", err)
|
|
}
|
|
return s
|
|
}
|
|
|
|
// newAG builds a minimal ArtifactGenerator suitable for fake-client tests.
|
|
func newAG(annotations map[string]string) *sourcewatcherv1beta1.ArtifactGenerator {
|
|
return &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example",
|
|
Namespace: "cozy-system",
|
|
Annotations: annotations,
|
|
},
|
|
}
|
|
}
|
|
|
|
// TestForceArtifactGeneratorDrift_Success asserts that on a successful
|
|
// force-drift call:
|
|
// - the AG's status.conditions[Ready] is set to False in the apiserver (this
|
|
// is the signal source-watcher's detectDrift keys on),
|
|
// - the tracking annotations land on the caller's `ag` AND in the apiserver,
|
|
// - the same in-memory `ag` reflects both mutations (mutation contract).
|
|
//
|
|
// This locks in the two-subresource shape (status patch + metadata patch) and
|
|
// the mutation contract documented on the function.
|
|
func TestForceArtifactGeneratorDrift_Success(t *testing.T) {
|
|
ag := newAG(nil)
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ag).WithObjects(ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
if err := r.forceArtifactGeneratorDrift(context.Background(), ag, referenceTime, 2); err != nil {
|
|
t.Fatalf("forceArtifactGeneratorDrift: %v", err)
|
|
}
|
|
|
|
// In-memory ag: Ready=False + tracking annotations reflect the writes.
|
|
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
|
|
if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != reasonRecoveryForced {
|
|
t.Fatalf("ag.Status Ready condition = %+v, want False/%s", ready, reasonRecoveryForced)
|
|
}
|
|
if got := ag.Annotations[annotationRecoveryAttempts]; got != "2" {
|
|
t.Errorf("ag.Annotations[%s] = %q, want 2", annotationRecoveryAttempts, got)
|
|
}
|
|
if got := ag.Annotations[annotationLastRecoveryAt]; got == "" {
|
|
t.Errorf("ag.Annotations[%s] not set", annotationLastRecoveryAt)
|
|
}
|
|
// B1 (lexfrei): the requestedAt annotation MUST also be set — it is the
|
|
// signal source-watcher's ReconcileRequestedPredicate keys on to enqueue
|
|
// a reconcile. Without it the status patch alone is a no-op for up to
|
|
// the AG's 1h self-requeue interval.
|
|
if got := ag.Annotations[annotationFluxRequestedAt]; got == "" {
|
|
t.Errorf("ag.Annotations[%s] not set — source-watcher will not enqueue a reconcile", annotationFluxRequestedAt)
|
|
}
|
|
|
|
// apiserver reflects both writes.
|
|
persisted := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persisted); err != nil {
|
|
t.Fatalf("Get after force-drift: %v", err)
|
|
}
|
|
persistedReady := meta.FindStatusCondition(persisted.Status.Conditions, "Ready")
|
|
if persistedReady == nil || persistedReady.Status != metav1.ConditionFalse {
|
|
t.Errorf("persisted AG Ready condition = %+v, want False", persistedReady)
|
|
}
|
|
for _, k := range []string{annotationFluxRequestedAt, annotationRecoveryAttempts, annotationLastRecoveryAt} {
|
|
if _, ok := persisted.Annotations[k]; !ok {
|
|
t.Errorf("persisted AG missing annotation %s", k)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestForceArtifactGeneratorDrift_StatusPatchFailure_Rollback drives the
|
|
// status-patch failure branch: an interceptor errors on SubResourcePatch.
|
|
// The caller's `ag` must be untouched (both conditions and annotations) and
|
|
// no metadata patch should have been issued.
|
|
func TestForceArtifactGeneratorDrift_StatusPatchFailure_Rollback(t *testing.T) {
|
|
ag := newAG(map[string]string{"unrelated": "value"})
|
|
baseClient := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ag).WithObjects(ag).Build()
|
|
metadataPatches := 0
|
|
failing := interceptor.NewClient(baseClient, interceptor.Funcs{
|
|
Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
|
metadataPatches++
|
|
return c.Patch(ctx, obj, patch, opts...)
|
|
},
|
|
SubResourcePatch: func(_ context.Context, _ client.Client, _ string, _ client.Object, _ client.Patch, _ ...client.SubResourcePatchOption) error {
|
|
return errors.New("simulated status patch failure")
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: failing, Scheme: testScheme(t)}
|
|
|
|
if err := r.forceArtifactGeneratorDrift(context.Background(), ag, referenceTime, 3); err == nil {
|
|
t.Fatal("forceArtifactGeneratorDrift succeeded, want error")
|
|
}
|
|
if ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready"); ready != nil {
|
|
t.Errorf("ag.Status Ready condition = %+v after failed status patch, want unset (rolled back)", ready)
|
|
}
|
|
if _, ok := ag.Annotations[annotationRecoveryAttempts]; ok {
|
|
t.Errorf("tracking annotation leaked despite status-patch failure: %v", ag.Annotations)
|
|
}
|
|
if metadataPatches != 0 {
|
|
t.Errorf("metadata patch issued %d times despite status-patch failure, want 0", metadataPatches)
|
|
}
|
|
if ag.Annotations["unrelated"] != "value" {
|
|
t.Errorf("pre-existing annotation lost: %v", ag.Annotations)
|
|
}
|
|
}
|
|
|
|
// TestForceArtifactGeneratorDrift_MetadataPatchFailure_RollbackAnnotations
|
|
// drives the second-step failure: status patch succeeds, metadata patch fails.
|
|
// Ready=False must remain in the caller's `ag` (successfully persisted) but
|
|
// the tracking annotations must roll back to pre-call state so the counter
|
|
// doesn't advance without landing in the apiserver.
|
|
func TestForceArtifactGeneratorDrift_MetadataPatchFailure_RollbackAnnotations(t *testing.T) {
|
|
ag := newAG(map[string]string{"unrelated": "value"})
|
|
baseClient := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ag).WithObjects(ag).Build()
|
|
failing := interceptor.NewClient(baseClient, interceptor.Funcs{
|
|
Patch: func(_ context.Context, _ client.WithWatch, _ client.Object, _ client.Patch, _ ...client.PatchOption) error {
|
|
return errors.New("simulated metadata patch failure")
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: failing, Scheme: testScheme(t)}
|
|
|
|
if err := r.forceArtifactGeneratorDrift(context.Background(), ag, referenceTime, 4); err == nil {
|
|
t.Fatal("forceArtifactGeneratorDrift succeeded, want error from metadata patch")
|
|
}
|
|
// Status patch already landed, so ag.Status.Conditions carries Ready=False
|
|
// even though the operation as a whole errored.
|
|
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
|
|
if ready == nil || ready.Status != metav1.ConditionFalse {
|
|
t.Errorf("ag.Status Ready condition = %+v, want False (status patch succeeded)", ready)
|
|
}
|
|
// Annotations must roll back — the counter never advanced in apiserver so
|
|
// leaving it locally advanced would desync the caller's view. Both the
|
|
// tracking annotations AND the requestedAt bump (added for B1) must be
|
|
// symmetrically rolled back — otherwise source-watcher would see a bumped
|
|
// requestedAt with no matching backoff-state on our side.
|
|
if _, ok := ag.Annotations[annotationRecoveryAttempts]; ok {
|
|
t.Errorf("tracking annotation not rolled back after metadata patch failure: %v", ag.Annotations)
|
|
}
|
|
if _, ok := ag.Annotations[annotationFluxRequestedAt]; ok {
|
|
t.Errorf("requestedAt annotation not rolled back after metadata patch failure: %v", ag.Annotations)
|
|
}
|
|
if ag.Annotations["unrelated"] != "value" {
|
|
t.Errorf("pre-existing annotation lost: %v", ag.Annotations)
|
|
}
|
|
}
|
|
|
|
// TestForceArtifactGeneratorDrift_MetadataPatchFailure_RollbackRestoresPriorBump
|
|
// exercises the subtle case where prior tracking annotations were set by an
|
|
// EARLIER force-drift and the current metadata patch fails. Rollback must
|
|
// restore prior values, not delete them.
|
|
func TestForceArtifactGeneratorDrift_MetadataPatchFailure_RollbackRestoresPriorBump(t *testing.T) {
|
|
priorTimestamp := referenceTime.Add(-time.Minute).UTC().Format(time.RFC3339Nano)
|
|
ag := newAG(map[string]string{
|
|
annotationRecoveryAttempts: "1",
|
|
annotationLastRecoveryAt: priorTimestamp,
|
|
})
|
|
baseClient := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ag).WithObjects(ag).Build()
|
|
failing := interceptor.NewClient(baseClient, interceptor.Funcs{
|
|
Patch: func(_ context.Context, _ client.WithWatch, _ client.Object, _ client.Patch, _ ...client.PatchOption) error {
|
|
return errors.New("simulated metadata patch failure")
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: failing, Scheme: testScheme(t)}
|
|
|
|
if err := r.forceArtifactGeneratorDrift(context.Background(), ag, referenceTime, 2); err == nil {
|
|
t.Fatal("force-drift succeeded, want error")
|
|
}
|
|
if got := ag.Annotations[annotationRecoveryAttempts]; got != "1" {
|
|
t.Errorf("annotationRecoveryAttempts = %q after rollback, want prior value 1", got)
|
|
}
|
|
if got := ag.Annotations[annotationLastRecoveryAt]; got != priorTimestamp {
|
|
t.Errorf("annotationLastRecoveryAt = %q after rollback, want prior %q", got, priorTimestamp)
|
|
}
|
|
}
|
|
|
|
// TestClearRecoveryTracking_Noop asserts that if the AG has no tracking
|
|
// annotations, we don't issue a Patch at all — otherwise every healthy
|
|
// reconcile would generate a no-op write.
|
|
func TestClearRecoveryTracking_Noop(t *testing.T) {
|
|
ag := newAG(map[string]string{"unrelated": "value"})
|
|
baseClient := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(ag).Build()
|
|
patchCount := 0
|
|
watched := interceptor.NewClient(baseClient, interceptor.Funcs{
|
|
Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
|
patchCount++
|
|
return c.Patch(ctx, obj, patch, opts...)
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: watched, Scheme: testScheme(t)}
|
|
|
|
if err := r.clearRecoveryTracking(context.Background(), ag); err != nil {
|
|
t.Fatalf("clearRecoveryTracking: %v", err)
|
|
}
|
|
if patchCount != 0 {
|
|
t.Errorf("clearRecoveryTracking issued %d Patch calls, want 0", patchCount)
|
|
}
|
|
}
|
|
|
|
// TestClearRecoveryTracking_PatchFailure_Rollback drives the failure branch and
|
|
// asserts the tracking annotations are restored on the caller's `ag`.
|
|
func TestClearRecoveryTracking_PatchFailure_Rollback(t *testing.T) {
|
|
original := map[string]string{
|
|
annotationRecoveryAttempts: "3",
|
|
annotationLastRecoveryAt: referenceTime.UTC().Format(time.RFC3339Nano),
|
|
"unrelated": "value",
|
|
}
|
|
ag := newAG(cloneAnnotations(original))
|
|
baseClient := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(ag).Build()
|
|
failing := interceptor.NewClient(baseClient, interceptor.Funcs{
|
|
Patch: func(_ context.Context, _ client.WithWatch, _ client.Object, _ client.Patch, _ ...client.PatchOption) error {
|
|
return errors.New("simulated Patch failure")
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: failing, Scheme: testScheme(t)}
|
|
|
|
if err := r.clearRecoveryTracking(context.Background(), ag); err == nil {
|
|
t.Fatal("clearRecoveryTracking succeeded, want error")
|
|
}
|
|
if len(ag.Annotations) != len(original) {
|
|
t.Fatalf("ag.Annotations size = %d, want %d after rollback", len(ag.Annotations), len(original))
|
|
}
|
|
for k, v := range original {
|
|
if ag.Annotations[k] != v {
|
|
t.Errorf("ag.Annotations[%s] = %q, want %q after rollback", k, ag.Annotations[k], v)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestReconcile_DeletionTimestamp_EarlyReturn asserts that a PackageSource
|
|
// carrying a deletionTimestamp does not have its status touched and does not
|
|
// trigger any writes to its ArtifactGenerator's annotations — the retry driver
|
|
// must not keep chasing an object that's already on its way out.
|
|
func TestReconcile_DeletionTimestamp_EarlyReturn(t *testing.T) {
|
|
now := metav1.NewTime(referenceTime)
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "doomed",
|
|
Namespace: "cozy-system",
|
|
DeletionTimestamp: &now,
|
|
Finalizers: []string{"cozystack.io/pretend"},
|
|
},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{},
|
|
}
|
|
ag := newAG(map[string]string{annotationRecoveryAttempts: "2"})
|
|
|
|
writes := 0
|
|
base := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(ps, ag).Build()
|
|
watched := interceptor.NewClient(base, interceptor.Funcs{
|
|
Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
|
writes++
|
|
return c.Patch(ctx, obj, patch, opts...)
|
|
},
|
|
Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
|
|
writes++
|
|
return c.Update(ctx, obj, opts...)
|
|
},
|
|
SubResourcePatch: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error {
|
|
writes++
|
|
return c.SubResource(subResourceName).Patch(ctx, obj, patch, opts...)
|
|
},
|
|
SubResourceUpdate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
|
|
writes++
|
|
return c.SubResource(subResourceName).Update(ctx, obj, opts...)
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: watched, Scheme: testScheme(t)}
|
|
|
|
res, err := r.Reconcile(context.Background(), ctrl.Request{
|
|
NamespacedName: types.NamespacedName{Name: ps.Name, Namespace: ps.Namespace},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Reconcile: %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 {
|
|
t.Errorf("Reconcile scheduled RequeueAfter=%v on a deleted object, want 0", res.RequeueAfter)
|
|
}
|
|
if writes != 0 {
|
|
t.Errorf("Reconcile issued %d writes on a deleted object, want 0", writes)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_UnknownWithinGrace_SchedulesFollowUp locks in the fix for
|
|
// the dormancy bug caught on a test cluster: when Ready=Unknown but grace hasn't elapsed
|
|
// yet, artifactGeneratorStuck returns false, we fall through to the copy path
|
|
// and — without an explicit RequeueAfter — the reconciler would sleep until
|
|
// the AG's next status change (which may never come if it's exactly the
|
|
// split-patch race we exist to fix) or the 10h informer resync. This test
|
|
// asserts we schedule a follow-up reconcile at grace-period expiry so the
|
|
// recovery driver can take over.
|
|
func TestUpdateStatus_UnknownWithinGrace_SchedulesFollowUp(t *testing.T) {
|
|
// Ready=Unknown, transitioned 10s ago — grace is 30s, so 20s remaining.
|
|
transitionAt := referenceTime.Add(-10 * time.Second)
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{
|
|
Name: "src",
|
|
Kind: "OCIRepository",
|
|
Namespace: "cozy-system",
|
|
},
|
|
},
|
|
}
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example", Namespace: "cozy-system", Generation: 1,
|
|
CreationTimestamp: metav1.NewTime(referenceTime.Add(-time.Hour)),
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
},
|
|
ObservedSourcesDigest: "sha256:0e7",
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionUnknown,
|
|
Reason: "Progressing",
|
|
Message: "Reconciliation in progress",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(transitionAt),
|
|
}},
|
|
},
|
|
}
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
res, err := r.updateStatus(context.Background(), ps, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
// remaining = transitionAt + stuckGracePeriod - now = -10s + 30s = 20s
|
|
if res.RequeueAfter != 20*time.Second {
|
|
t.Errorf("RequeueAfter = %v, want 20s (remaining grace after 10s elapsed)", res.RequeueAfter)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_MissingReadyWithinGrace_SchedulesFollowUp — same dormancy
|
|
// fix but for the ready==nil branch: a fresh AG with no Ready condition yet
|
|
// must also schedule a follow-up so the operator notices when grace elapses.
|
|
func TestUpdateStatus_MissingReadyWithinGrace_SchedulesFollowUp(t *testing.T) {
|
|
createdAt := referenceTime.Add(-5 * time.Second) // 25s remaining on grace
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{
|
|
Name: "src",
|
|
Kind: "OCIRepository",
|
|
Namespace: "cozy-system",
|
|
},
|
|
},
|
|
}
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example", Namespace: "cozy-system", Generation: 1,
|
|
CreationTimestamp: metav1.NewTime(createdAt),
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{}, // no Ready
|
|
}
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
res, err := r.updateStatus(context.Background(), ps, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
if res.RequeueAfter != 25*time.Second {
|
|
t.Errorf("RequeueAfter = %v, want 25s", res.RequeueAfter)
|
|
}
|
|
}
|
|
|
|
// TestAgFollowUpDelay pins the follow-up-delay computation, including the
|
|
// one-second floor that prevents a zero-delay tight loop when the anchor is
|
|
// right at or beyond grace expiry.
|
|
func TestAgFollowUpDelay(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
anchorOffset time.Duration
|
|
want time.Duration
|
|
}{
|
|
{"anchor 10s ago — grace has 20s left", -10 * time.Second, 20 * time.Second},
|
|
{"anchor 5s ago — grace has 25s left", -5 * time.Second, 25 * time.Second},
|
|
{"anchor right at grace expiry — floor to 1s", -stuckGracePeriod, time.Second},
|
|
{"anchor past grace expiry — floor to 1s", -2 * stuckGracePeriod, time.Second},
|
|
{"anchor now — full grace remaining", 0, stuckGracePeriod},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := agFollowUpDelay(referenceTime.Add(tt.anchorOffset), referenceTime)
|
|
if got != tt.want {
|
|
t.Errorf("agFollowUpDelay(offset=%v) = %v, want %v", tt.anchorOffset, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// stuckAG builds an ArtifactGenerator in the fluxcd/pkg#934 stall signature:
|
|
// Inventory populated, ObservedSourcesDigest set, Ready=Unknown on the current
|
|
// generation, LastTransitionTime old enough to clear the grace period. Used by
|
|
// the maybeRecoverArtifactGenerator tests below.
|
|
func stuckAG(t *testing.T, annotations map[string]string, readyTransition time.Time) *sourcewatcherv1beta1.ArtifactGenerator {
|
|
t.Helper()
|
|
return &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example",
|
|
Namespace: "cozy-system",
|
|
Generation: 1,
|
|
Annotations: annotations,
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
},
|
|
ObservedSourcesDigest: "sha256:0e7",
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionUnknown,
|
|
Reason: "Progressing",
|
|
Message: "Reconciliation in progress",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(readyTransition),
|
|
}},
|
|
},
|
|
}
|
|
}
|
|
|
|
// TestMaybeRecoverArtifactGenerator_ForceBranch pins the first-time-stuck path:
|
|
// no prior tracking annotations, so decideRecovery returns Force, we patch
|
|
// Ready=False on the AG's status, update tracking annotations, and set the
|
|
// PackageSource to Unknown/AwaitingSourceWatcherRecovery with a RequeueAfter
|
|
// equal to backoffFor(1).
|
|
func TestMaybeRecoverArtifactGenerator_ForceBranch(t *testing.T) {
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
}
|
|
ag := stuckAG(t, nil, referenceTime.Add(-2*time.Minute))
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
|
|
res, err := r.maybeRecoverArtifactGenerator(context.Background(), ps, ag, ready, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("maybeRecoverArtifactGenerator: %v", err)
|
|
}
|
|
if res.RequeueAfter != backoffFor(1) {
|
|
t.Errorf("RequeueAfter = %v, want %v", res.RequeueAfter, backoffFor(1))
|
|
}
|
|
|
|
// AG status.Ready must be False in apiserver.
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
agReady := meta.FindStatusCondition(persistedAG.Status.Conditions, "Ready")
|
|
if agReady == nil || agReady.Status != metav1.ConditionFalse {
|
|
t.Errorf("AG Ready in apiserver = %+v, want False", agReady)
|
|
}
|
|
if persistedAG.Annotations[annotationRecoveryAttempts] != "1" {
|
|
t.Errorf("AG %s annotation = %q, want 1", annotationRecoveryAttempts, persistedAG.Annotations[annotationRecoveryAttempts])
|
|
}
|
|
|
|
// PackageSource must be Unknown/AwaitingSourceWatcherRecovery in apiserver.
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Status != metav1.ConditionUnknown || psReady.Reason != reasonAwaitingRecovery {
|
|
t.Errorf("PS Ready = %+v, want Unknown/%s", psReady, reasonAwaitingRecovery)
|
|
}
|
|
}
|
|
|
|
// TestMaybeRecoverArtifactGenerator_WaitBranch pins the in-backoff-window path:
|
|
// prior attempt exists, backoff not yet elapsed, so decideRecovery returns
|
|
// Wait and we do NOT touch the AG's status/annotations — only the PackageSource
|
|
// gets Unknown/AwaitingSourceWatcherRecovery and RequeueAfter set to the
|
|
// remaining backoff.
|
|
func TestMaybeRecoverArtifactGenerator_WaitBranch(t *testing.T) {
|
|
priorForceAt := referenceTime.Add(-10 * time.Second) // 20s remaining on initialBackoff=30s
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
}
|
|
ag := stuckAG(t, map[string]string{
|
|
annotationRecoveryAttempts: "1",
|
|
annotationLastRecoveryAt: priorForceAt.UTC().Format(time.RFC3339Nano),
|
|
}, referenceTime.Add(-2*time.Minute))
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
agWriteCount := 0
|
|
watched := interceptor.NewClient(c, interceptor.Funcs{
|
|
Patch: func(ctx context.Context, cli client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
|
if _, ok := obj.(*sourcewatcherv1beta1.ArtifactGenerator); ok {
|
|
agWriteCount++
|
|
}
|
|
return cli.Patch(ctx, obj, patch, opts...)
|
|
},
|
|
SubResourcePatch: func(ctx context.Context, cli client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error {
|
|
if _, ok := obj.(*sourcewatcherv1beta1.ArtifactGenerator); ok {
|
|
agWriteCount++
|
|
}
|
|
return cli.SubResource(sub).Patch(ctx, obj, patch, opts...)
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: watched, Scheme: testScheme(t)}
|
|
|
|
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
|
|
res, err := r.maybeRecoverArtifactGenerator(context.Background(), ps, ag, ready, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("maybeRecoverArtifactGenerator: %v", err)
|
|
}
|
|
if res.RequeueAfter != 20*time.Second {
|
|
t.Errorf("RequeueAfter = %v, want 20s", res.RequeueAfter)
|
|
}
|
|
if agWriteCount != 0 {
|
|
t.Errorf("wait branch wrote %d times to AG, want 0", agWriteCount)
|
|
}
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Reason != reasonAwaitingRecovery {
|
|
t.Errorf("PS Ready = %+v, want reason %s", psReady, reasonAwaitingRecovery)
|
|
}
|
|
}
|
|
|
|
// TestMaybeRecoverArtifactGenerator_GiveUpBranch pins the exhaustion path:
|
|
// attempts equals maxRecoveryAttempts, so we surface the failure as
|
|
// PackageSource Ready=False/SourceWatcherStalled and schedule no follow-up
|
|
// reconcile (an operator must intervene). The AG is left untouched.
|
|
func TestMaybeRecoverArtifactGenerator_GiveUpBranch(t *testing.T) {
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
}
|
|
priorForceAt := referenceTime.Add(-time.Hour).UTC().Format(time.RFC3339Nano)
|
|
ag := stuckAG(t, map[string]string{
|
|
annotationRecoveryAttempts: strconv.Itoa(maxRecoveryAttempts),
|
|
annotationLastRecoveryAt: priorForceAt,
|
|
}, referenceTime.Add(-time.Hour)) // last transition BEFORE lastRecoveryAt so reset-counter doesn't fire
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
|
|
res, err := r.maybeRecoverArtifactGenerator(context.Background(), ps, ag, ready, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("maybeRecoverArtifactGenerator: %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 {
|
|
t.Errorf("RequeueAfter = %v, want 0 (give-up path stops rescheduling)", res.RequeueAfter)
|
|
}
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Status != metav1.ConditionFalse || psReady.Reason != reasonSourceWatcherBad {
|
|
t.Errorf("PS Ready = %+v, want False/%s", psReady, reasonSourceWatcherBad)
|
|
}
|
|
}
|
|
|
|
// TestMaybeRecoverArtifactGenerator_DoesNotResetCounterOnFreshTransition pins
|
|
// the fix for lexfrei's B3: source-watcher writes Ready=Unknown (Progressing)
|
|
// as it takes the drifted branch after our force, and that transition's
|
|
// LastTransitionTime is always newer than lastRecoveryAt. A reset-counter
|
|
// heuristic that keys on that comparison would restart the budget on every
|
|
// Progressing write, making the SourceWatcherStalled give-up unreachable
|
|
// under the very race this driver targets. Assertion: with attempts already
|
|
// at maxRecoveryAttempts, we go straight to GiveUp even when Ready has just
|
|
// been touched — the natural clearRecoveryTracking on the not-stuck path is
|
|
// the only counter reset, and the unified own-marker / progressing-in-recovery
|
|
// gate in updateStatus holds tracking through Progressing by routing back
|
|
// through this same maybeRecoverArtifactGenerator's Wait branch.
|
|
func TestMaybeRecoverArtifactGenerator_DoesNotResetCounterOnFreshTransition(t *testing.T) {
|
|
priorForceAt := referenceTime.Add(-time.Hour)
|
|
freshTransitionAt := referenceTime.Add(-2 * time.Minute) // AFTER priorForceAt — must NOT reset
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
}
|
|
ag := stuckAG(t, map[string]string{
|
|
annotationRecoveryAttempts: strconv.Itoa(maxRecoveryAttempts),
|
|
annotationLastRecoveryAt: priorForceAt.UTC().Format(time.RFC3339Nano),
|
|
}, freshTransitionAt)
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
ready := meta.FindStatusCondition(ag.Status.Conditions, "Ready")
|
|
res, err := r.maybeRecoverArtifactGenerator(context.Background(), ps, ag, ready, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("maybeRecoverArtifactGenerator: %v", err)
|
|
}
|
|
// Give-up path returns no RequeueAfter (operator must intervene).
|
|
if res.RequeueAfter != 0 {
|
|
t.Errorf("RequeueAfter = %v, want 0 — give-up should not schedule a follow-up", res.RequeueAfter)
|
|
}
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Status != metav1.ConditionFalse || psReady.Reason != reasonSourceWatcherBad {
|
|
t.Errorf("PS Ready = %+v, want False/%s — reset would have taken the Force branch instead", psReady, reasonSourceWatcherBad)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_OwnMarkerHoldsTrackingViaStateMachine pins lexfrei's B2
|
|
// fix: when the AG's Ready condition carries our own reasonRecoveryForced
|
|
// marker (our previous forceArtifactGeneratorDrift write reflecting back
|
|
// through the Owns() watch), updateStatus must (a) NOT clear the
|
|
// recovery-tracking annotations we just wrote, and (b) NOT copy the
|
|
// synthetic Ready=False through to the PackageSource. It routes through the
|
|
// unified own-marker / progressing-in-recovery / stuck gate into
|
|
// maybeRecoverArtifactGenerator, and — because we are inside the backoff
|
|
// window — the state machine's Wait branch keeps the PS in
|
|
// Unknown/AwaitingSourceWatcherRecovery with a follow-up requeue.
|
|
func TestUpdateStatus_OwnMarkerHoldsTrackingViaStateMachine(t *testing.T) {
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{Name: "src", Kind: "OCIRepository", Namespace: "cozy-system"},
|
|
},
|
|
}
|
|
forceAt := referenceTime.Add(-5 * time.Second)
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example", Namespace: "cozy-system", Generation: 1,
|
|
CreationTimestamp: metav1.NewTime(referenceTime.Add(-time.Hour)),
|
|
Annotations: map[string]string{
|
|
annotationRecoveryAttempts: "1",
|
|
annotationLastRecoveryAt: forceAt.UTC().Format(time.RFC3339Nano),
|
|
},
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
},
|
|
ObservedSourcesDigest: "sha256:0e7",
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionFalse,
|
|
Reason: reasonRecoveryForced,
|
|
Message: "cozystack-operator forced drift",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(forceAt),
|
|
}},
|
|
},
|
|
}
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
agWrites := 0
|
|
watched := interceptor.NewClient(c, interceptor.Funcs{
|
|
Patch: func(ctx context.Context, cli client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
|
if _, ok := obj.(*sourcewatcherv1beta1.ArtifactGenerator); ok {
|
|
agWrites++
|
|
}
|
|
return cli.Patch(ctx, obj, patch, opts...)
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: watched, Scheme: testScheme(t)}
|
|
|
|
res, err := r.updateStatus(context.Background(), ps, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
// The routing goes through maybeRecoverArtifactGenerator's Wait branch:
|
|
// decideRecovery(attempts=1, lastRecoveryAt=referenceTime-5s, now=referenceTime)
|
|
// → elapsed 5s, backoffFor(1)=30s → wait 25s. Exact equality catches
|
|
// silent backoff-schedule regressions.
|
|
if res.RequeueAfter != 25*time.Second {
|
|
t.Errorf("RequeueAfter = %v, want 25s (backoffFor(1)=30s minus 5s elapsed)", res.RequeueAfter)
|
|
}
|
|
if agWrites != 0 {
|
|
t.Errorf("clearRecoveryTracking wrote to AG %d times — tracking must be held while our own marker is visible", agWrites)
|
|
}
|
|
// AG's recovery-tracking annotations must still be intact.
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if got := persistedAG.Annotations[annotationRecoveryAttempts]; got != "1" {
|
|
t.Errorf("AG %s = %q after own-marker reconcile, want 1 (held)", annotationRecoveryAttempts, got)
|
|
}
|
|
// PS must be Unknown/AwaitingSourceWatcherRecovery — NOT False/SourceWatcherRecoveryForced.
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Status != metav1.ConditionUnknown || psReady.Reason != reasonAwaitingRecovery {
|
|
t.Errorf("PS Ready = %+v, want Unknown/%s — synthetic marker must not leak to PS status", psReady, reasonAwaitingRecovery)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_ProgressingDuringRecoveryHoldsTracking pins lexfrei's B2
|
|
// fix for the second Owns()-re-entrancy path: source-watcher writes
|
|
// Ready=Unknown/Progressing after taking the drifted branch and before
|
|
// finalising the rebuild. That transition arrives via the Owns() watch, and
|
|
// if we cleared tracking on it the attempts counter would never accumulate.
|
|
// Assertion: with tracking annotations present (attempts>0) and Ready=Unknown
|
|
// visible on the current generation, updateStatus routes through the unified
|
|
// own-marker / progressing-in-recovery gate into maybeRecoverArtifactGenerator
|
|
// so the state machine holds tracking through Wait instead of clearing it.
|
|
func TestUpdateStatus_ProgressingDuringRecoveryHoldsTracking(t *testing.T) {
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{Name: "src", Kind: "OCIRepository", Namespace: "cozy-system"},
|
|
},
|
|
}
|
|
forceAt := referenceTime.Add(-10 * time.Second)
|
|
progressingAt := referenceTime.Add(-3 * time.Second) // Newer than forceAt
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example", Namespace: "cozy-system", Generation: 1,
|
|
CreationTimestamp: metav1.NewTime(referenceTime.Add(-time.Hour)),
|
|
Annotations: map[string]string{
|
|
annotationRecoveryAttempts: "2",
|
|
annotationLastRecoveryAt: forceAt.UTC().Format(time.RFC3339Nano),
|
|
},
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
},
|
|
ObservedSourcesDigest: "sha256:0e7",
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionUnknown,
|
|
Reason: "Progressing",
|
|
Message: "Reconciliation in progress",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(progressingAt),
|
|
}},
|
|
},
|
|
}
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
agWrites := 0
|
|
watched := interceptor.NewClient(c, interceptor.Funcs{
|
|
Patch: func(ctx context.Context, cli client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
|
if _, ok := obj.(*sourcewatcherv1beta1.ArtifactGenerator); ok {
|
|
agWrites++
|
|
}
|
|
return cli.Patch(ctx, obj, patch, opts...)
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: watched, Scheme: testScheme(t)}
|
|
|
|
if _, err := r.updateStatus(context.Background(), ps, referenceTime); err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
if agWrites != 0 {
|
|
t.Errorf("tracking cleared during recovery Progressing — attempts=2 wrote to AG %d times, want 0", agWrites)
|
|
}
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if got := persistedAG.Annotations[annotationRecoveryAttempts]; got != "2" {
|
|
t.Errorf("AG %s = %q, want 2 (held through Progressing)", annotationRecoveryAttempts, got)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_ReadyTrueClearsTracking pins the canonical reset: once
|
|
// source-watcher writes Ready=True (recovery succeeded), the not-stuck path
|
|
// clears the recovery-tracking annotations and copies the real condition
|
|
// through. This is the only counter reset that remains after B3.
|
|
func TestUpdateStatus_ReadyTrueClearsTracking(t *testing.T) {
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{Name: "src", Kind: "OCIRepository", Namespace: "cozy-system"},
|
|
},
|
|
}
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example", Namespace: "cozy-system", Generation: 1,
|
|
CreationTimestamp: metav1.NewTime(referenceTime.Add(-time.Hour)),
|
|
Annotations: map[string]string{
|
|
annotationRecoveryAttempts: "3",
|
|
annotationLastRecoveryAt: referenceTime.Add(-time.Minute).UTC().Format(time.RFC3339Nano),
|
|
},
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
},
|
|
ObservedSourcesDigest: "sha256:0e7",
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionTrue,
|
|
Reason: "Succeeded",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(referenceTime.Add(-time.Second)),
|
|
}},
|
|
},
|
|
}
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
if _, err := r.updateStatus(context.Background(), ps, referenceTime); err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if _, ok := persistedAG.Annotations[annotationRecoveryAttempts]; ok {
|
|
t.Errorf("recovery-attempts annotation still present after Ready=True: %v", persistedAG.Annotations)
|
|
}
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Status != metav1.ConditionTrue {
|
|
t.Errorf("PS Ready = %+v, want True (copied through from AG)", psReady)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_OwnMarkerStale_TriggersRetry regression-tests the
|
|
// source-watcher-unresponsive path that an earlier iteration of this driver
|
|
// tripped on: after we force-drift, source-watcher never enqueues (network
|
|
// partition, missing predicate, pod dead), and our previous force's marker
|
|
// keeps reflecting back via Owns(). If updateStatus routed that into a
|
|
// perpetual "await" helper the recovery would tight-loop forever and
|
|
// SourceWatcherStalled would be unreachable. Asserted here: past
|
|
// backoffFor(attempts), the state machine issues another force (bumps
|
|
// attempts, re-stamps requestedAt) rather than idling.
|
|
func TestUpdateStatus_OwnMarkerStale_TriggersRetry(t *testing.T) {
|
|
// attempts=1, lastRecoveryAt=referenceTime-60s. backoffFor(1)=30s, so
|
|
// 60s > 30s → decideRecovery must Force again.
|
|
forceAt := referenceTime.Add(-60 * time.Second)
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{Name: "src", Kind: "OCIRepository", Namespace: "cozy-system"},
|
|
},
|
|
}
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example", Namespace: "cozy-system", Generation: 1,
|
|
CreationTimestamp: metav1.NewTime(referenceTime.Add(-time.Hour)),
|
|
Annotations: map[string]string{
|
|
annotationRecoveryAttempts: "1",
|
|
annotationLastRecoveryAt: forceAt.UTC().Format(time.RFC3339Nano),
|
|
annotationFluxRequestedAt: forceAt.UTC().Format(time.RFC3339Nano),
|
|
},
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
},
|
|
ObservedSourcesDigest: "sha256:0e7",
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionFalse,
|
|
Reason: reasonRecoveryForced,
|
|
Message: "cozystack-operator forced drift (source-watcher never responded)",
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(forceAt),
|
|
}},
|
|
},
|
|
}
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
if _, err := r.updateStatus(context.Background(), ps, referenceTime); err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
// attempts must have incremented from 1 → 2.
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if got := persistedAG.Annotations[annotationRecoveryAttempts]; got != "2" {
|
|
t.Errorf("AG %s = %q, want 2 — stale own-marker must escalate through Force, not idle", annotationRecoveryAttempts, got)
|
|
}
|
|
// requestedAt must have been re-stamped to a fresh timestamp so
|
|
// source-watcher's next reconcile-request predicate fires.
|
|
if got := persistedAG.Annotations[annotationFluxRequestedAt]; got == forceAt.UTC().Format(time.RFC3339Nano) {
|
|
t.Errorf("AG %s = %q — must be re-stamped on retry", annotationFluxRequestedAt, got)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_OwnMarkerAttemptsExhausted_GivesUp regression-tests the
|
|
// endpoint of the retry loop: with attempts=maxRecoveryAttempts and our own
|
|
// marker still reflecting back (source-watcher never reacted through the
|
|
// whole budget), updateStatus must surface Ready=False /
|
|
// SourceWatcherStalled on the PackageSource instead of looping forever.
|
|
func TestUpdateStatus_OwnMarkerAttemptsExhausted_GivesUp(t *testing.T) {
|
|
forceAt := referenceTime.Add(-10 * time.Minute) // well past any backoff
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{Name: "src", Kind: "OCIRepository", Namespace: "cozy-system"},
|
|
},
|
|
}
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example", Namespace: "cozy-system", Generation: 1,
|
|
CreationTimestamp: metav1.NewTime(referenceTime.Add(-time.Hour)),
|
|
Annotations: map[string]string{
|
|
annotationRecoveryAttempts: strconv.Itoa(maxRecoveryAttempts),
|
|
annotationLastRecoveryAt: forceAt.UTC().Format(time.RFC3339Nano),
|
|
},
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
},
|
|
ObservedSourcesDigest: "sha256:0e7",
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionFalse,
|
|
Reason: reasonRecoveryForced,
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(forceAt),
|
|
}},
|
|
},
|
|
}
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
res, err := r.updateStatus(context.Background(), ps, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 {
|
|
t.Errorf("RequeueAfter = %v on give-up, want 0 (operator must intervene)", res.RequeueAfter)
|
|
}
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Status != metav1.ConditionFalse || psReady.Reason != reasonSourceWatcherBad {
|
|
t.Errorf("PS Ready = %+v, want False/%s — unresponsive source-watcher must eventually surface as stalled", psReady, reasonSourceWatcherBad)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_OwnMarkerAttemptsExhaustedButFinalBackoffPending pins
|
|
// lexfrei's follow-up on the give-up path: the Owns()-watch re-fire from the
|
|
// 5th force-drift's own writes triggers a reconcile milliseconds after the
|
|
// force. Without the "hold final backoff window" gate in decideRecovery,
|
|
// that re-fire would see `attempts >= maxRecoveryAttempts` and immediately
|
|
// return GiveUp, preempting the 4-minute response window the last nudge
|
|
// was supposed to grant source-watcher — effective time-to-give-up
|
|
// collapses from the documented ~11.5m to ~7.5m. Assertion: with
|
|
// attempts=maxRecoveryAttempts AND lastRecoveryAt=now-eps (the exact shape
|
|
// of the re-fire on the 5th force's own writes), updateStatus must return
|
|
// Wait with RequeueAfter equal to the remaining backoffFor(maxRecoveryAttempts),
|
|
// NOT GiveUp.
|
|
func TestUpdateStatus_OwnMarkerAttemptsExhaustedButFinalBackoffPending(t *testing.T) {
|
|
// Simulate the moment right after the 5th force landed: attempts=max,
|
|
// lastRecoveryAt=now-eps, marker still on the AG.
|
|
forceAt := referenceTime.Add(-time.Second)
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{Name: "src", Kind: "OCIRepository", Namespace: "cozy-system"},
|
|
},
|
|
}
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example", Namespace: "cozy-system", Generation: 1,
|
|
CreationTimestamp: metav1.NewTime(referenceTime.Add(-time.Hour)),
|
|
Annotations: map[string]string{
|
|
annotationRecoveryAttempts: strconv.Itoa(maxRecoveryAttempts),
|
|
annotationLastRecoveryAt: forceAt.UTC().Format(time.RFC3339Nano),
|
|
},
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
},
|
|
ObservedSourcesDigest: "sha256:0e7",
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: metav1.ConditionFalse,
|
|
Reason: reasonRecoveryForced,
|
|
ObservedGeneration: 1,
|
|
LastTransitionTime: metav1.NewTime(forceAt),
|
|
}},
|
|
},
|
|
}
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
res, err := r.updateStatus(context.Background(), ps, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
wantWait := backoffFor(maxRecoveryAttempts) - time.Second
|
|
if res.RequeueAfter != wantWait {
|
|
t.Errorf("RequeueAfter = %v, want %v — final backoff window must not be preempted", res.RequeueAfter, wantWait)
|
|
}
|
|
// PS must still be Unknown/AwaitingSourceWatcherRecovery, NOT False/SourceWatcherStalled.
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Status != metav1.ConditionUnknown || psReady.Reason != reasonAwaitingRecovery {
|
|
t.Errorf("PS Ready = %+v, want Unknown/%s — give-up must wait for final backoff to elapse", psReady, reasonAwaitingRecovery)
|
|
}
|
|
}
|
|
|
|
// TestReconcileArtifactGeneratorsLayersValuesFiles pins how a component's
|
|
// valuesFiles become copy operations: every file lands on the same
|
|
// values.yaml, the first one replacing the chart's own values and each later
|
|
// one merged on top in declaration order, so a later file wins on conflicting
|
|
// keys.
|
|
func TestReconcileArtifactGeneratorsLayersValuesFiles(t *testing.T) {
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "demo.app"},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{Kind: "OCIRepository", Name: "demo-app", Namespace: "cozy-system"},
|
|
Variants: []cozyv1alpha1.Variant{{
|
|
Name: "default",
|
|
Components: []cozyv1alpha1.Component{{
|
|
Name: "web",
|
|
Path: "apps/web",
|
|
ValuesFiles: []string{"values.yaml", "values-ha.yaml", "values-site.yaml"},
|
|
}},
|
|
}},
|
|
},
|
|
}
|
|
cl := fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(ps.DeepCopy()).Build()
|
|
r := &PackageSourceReconciler{Client: cl, Scheme: testScheme(t)}
|
|
if err := r.reconcileArtifactGenerators(context.Background(), ps); err != nil {
|
|
t.Fatalf("reconcileArtifactGenerators: %v", err)
|
|
}
|
|
|
|
var ag sourcewatcherv1beta1.ArtifactGenerator
|
|
if err := cl.Get(context.Background(), types.NamespacedName{Name: "demo.app", Namespace: "cozy-system"}, &ag); err != nil {
|
|
t.Fatalf("get ArtifactGenerator: %v", err)
|
|
}
|
|
if len(ag.Spec.OutputArtifacts) != 1 {
|
|
t.Fatalf("OutputArtifacts = %d, want 1", len(ag.Spec.OutputArtifacts))
|
|
}
|
|
want := []sourcewatcherv1beta1.CopyOperation{
|
|
{From: "@demo-app/apps/web/**", To: "@artifact/web/"},
|
|
{From: "@demo-app/apps/web/values.yaml", To: "@artifact/web/values.yaml", Strategy: "Overwrite"},
|
|
{From: "@demo-app/apps/web/values-ha.yaml", To: "@artifact/web/values.yaml", Strategy: "Merge"},
|
|
{From: "@demo-app/apps/web/values-site.yaml", To: "@artifact/web/values.yaml", Strategy: "Merge"},
|
|
}
|
|
if got := ag.Spec.OutputArtifacts[0].Copy; !reflect.DeepEqual(got, want) {
|
|
t.Errorf("Copy =\n%+v\nwant\n%+v", got, want)
|
|
}
|
|
}
|
|
|
|
// builtAG returns a PackageSource and an ArtifactGenerator on the given
|
|
// generation whose Ready condition reports a finished build.
|
|
func builtAG(gen int64, readyStatus metav1.ConditionStatus, readyGen int64, annotations map[string]string) (*cozyv1alpha1.PackageSource, *sourcewatcherv1beta1.ArtifactGenerator) {
|
|
ps := &cozyv1alpha1.PackageSource{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "cozy-system", Generation: 1},
|
|
Spec: cozyv1alpha1.PackageSourceSpec{
|
|
SourceRef: &cozyv1alpha1.PackageSourceRef{Name: "src", Kind: "OCIRepository", Namespace: "cozy-system"},
|
|
},
|
|
}
|
|
ag := &sourcewatcherv1beta1.ArtifactGenerator{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "example", Namespace: "cozy-system", Generation: gen,
|
|
CreationTimestamp: metav1.NewTime(referenceTime.Add(-time.Hour)),
|
|
Annotations: annotations,
|
|
},
|
|
Status: sourcewatcherv1beta1.ArtifactGeneratorStatus{
|
|
Inventory: []sourcewatcherv1beta1.ExternalArtifactReference{
|
|
{Name: "one", Namespace: "cozy-system", Digest: "sha256:aaa"},
|
|
},
|
|
ObservedSourcesDigest: "sha256:0e7",
|
|
Conditions: []metav1.Condition{{
|
|
Type: "Ready",
|
|
Status: readyStatus,
|
|
Reason: "Succeeded",
|
|
ObservedGeneration: readyGen,
|
|
LastTransitionTime: metav1.NewTime(referenceTime.Add(-time.Second)),
|
|
}},
|
|
},
|
|
}
|
|
return ps, ag
|
|
}
|
|
|
|
// TestUpdateStatus_ReadyOnUnverifiedGeneration_ForcesRebuild pins the guard
|
|
// against the source-watcher race where a build started on generation N-1
|
|
// finishes after the spec moved to N and still records N as observed: the
|
|
// first Ready=True on a spec generation the operator changed forces one
|
|
// rebuild and records that generation as verified.
|
|
func TestUpdateStatus_ReadyOnUnverifiedGeneration_ForcesRebuild(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
gen int64
|
|
annotations map[string]string
|
|
}{
|
|
{"first spec change", 2, nil},
|
|
{"later spec change", 3, map[string]string{annotationVerifiedGeneration: "2"}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
testReadyOnUnverifiedGenerationForcesRebuild(t, tc.gen, tc.annotations)
|
|
})
|
|
}
|
|
}
|
|
|
|
func testReadyOnUnverifiedGenerationForcesRebuild(t *testing.T, gen int64, annotations map[string]string) {
|
|
ps, ag := builtAG(gen, metav1.ConditionTrue, gen, annotations)
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
res, err := r.updateStatus(context.Background(), ps, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
if res.RequeueAfter != backoffFor(1) {
|
|
t.Errorf("RequeueAfter = %v, want %v", res.RequeueAfter, backoffFor(1))
|
|
}
|
|
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
agReady := meta.FindStatusCondition(persistedAG.Status.Conditions, "Ready")
|
|
if agReady == nil || agReady.Status != metav1.ConditionFalse || agReady.Reason != reasonRecoveryForced {
|
|
t.Errorf("AG Ready = %+v, want False/%s", agReady, reasonRecoveryForced)
|
|
}
|
|
if persistedAG.Annotations[annotationFluxRequestedAt] == "" {
|
|
t.Errorf("requestedAt not bumped: %v", persistedAG.Annotations)
|
|
}
|
|
if got, want := persistedAG.Annotations[annotationVerifiedGeneration], strconv.FormatInt(gen, 10); got != want {
|
|
t.Errorf("verified generation = %q, want %q", got, want)
|
|
}
|
|
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Status != metav1.ConditionUnknown || psReady.Reason != reasonAwaitingRecovery {
|
|
t.Errorf("PS Ready = %+v, want Unknown/%s", psReady, reasonAwaitingRecovery)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_VerificationSkipped covers every state where the forced
|
|
// rebuild must not happen, so a verified or never-changed generator is not
|
|
// rebuilt on every reconcile and a failed or in-flight build is left to
|
|
// source-watcher.
|
|
func TestUpdateStatus_VerificationSkipped(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
gen int64
|
|
status metav1.ConditionStatus
|
|
readyGen int64
|
|
annotations map[string]string
|
|
}{
|
|
{"generation already verified", 2, metav1.ConditionTrue, 2, map[string]string{annotationVerifiedGeneration: "2"}},
|
|
{"spec never changed after creation", 1, metav1.ConditionTrue, 1, nil},
|
|
{"build failed", 2, metav1.ConditionFalse, 2, nil},
|
|
{"source-watcher has not caught up with the spec", 3, metav1.ConditionTrue, 2, map[string]string{annotationVerifiedGeneration: "2"}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ps, ag := builtAG(tc.gen, tc.status, tc.readyGen, tc.annotations)
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
if _, err := r.updateStatus(context.Background(), ps, referenceTime); err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if _, ok := persistedAG.Annotations[annotationFluxRequestedAt]; ok {
|
|
t.Errorf("rebuild forced: %v", persistedAG.Annotations)
|
|
}
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready")
|
|
if psReady == nil || psReady.Status != tc.status {
|
|
t.Errorf("PS Ready = %+v, want %s copied through", psReady, tc.status)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_VerifiedRebuildSettles walks the forced rebuild to its end:
|
|
// once source-watcher reports Ready=True again, the recovery bookkeeping is
|
|
// cleared and the condition is copied through without another rebuild.
|
|
func TestUpdateStatus_VerifiedRebuildSettles(t *testing.T) {
|
|
forcedAt := referenceTime.Add(-10 * time.Second).UTC().Format(time.RFC3339Nano)
|
|
ps, ag := builtAG(2, metav1.ConditionTrue, 2, map[string]string{
|
|
annotationVerifiedGeneration: "2",
|
|
annotationRecoveryAttempts: "1",
|
|
annotationLastRecoveryAt: forcedAt,
|
|
annotationFluxRequestedAt: forcedAt,
|
|
})
|
|
c := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
r := &PackageSourceReconciler{Client: c, Scheme: testScheme(t)}
|
|
|
|
res, err := r.updateStatus(context.Background(), ps, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
if res.RequeueAfter != 0 {
|
|
t.Errorf("RequeueAfter = %v, want none", res.RequeueAfter)
|
|
}
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := c.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if _, ok := persistedAG.Annotations[annotationRecoveryAttempts]; ok {
|
|
t.Errorf("recovery tracking not cleared: %v", persistedAG.Annotations)
|
|
}
|
|
if got := persistedAG.Annotations[annotationFluxRequestedAt]; got != forcedAt {
|
|
t.Errorf("requestedAt = %q, want %q (no second rebuild)", got, forcedAt)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_VerificationForceFailure_LeavesGenerationUnverified pins
|
|
// the ordering the verification relies on: a generation is marked verified
|
|
// only after the rebuild was forced, so a failed force is retried instead of
|
|
// being recorded as done.
|
|
func TestUpdateStatus_VerificationForceFailure_LeavesGenerationUnverified(t *testing.T) {
|
|
ps, ag := builtAG(2, metav1.ConditionTrue, 2, nil)
|
|
baseClient := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
failing := interceptor.NewClient(baseClient, interceptor.Funcs{
|
|
SubResourcePatch: func(_ context.Context, _ client.Client, _ string, _ client.Object, _ client.Patch, _ ...client.SubResourcePatchOption) error {
|
|
return errors.New("simulated status patch failure")
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: failing, Scheme: testScheme(t)}
|
|
|
|
if _, err := r.updateStatus(context.Background(), ps, referenceTime); err == nil {
|
|
t.Fatal("updateStatus succeeded, want error from the forced rebuild")
|
|
}
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := baseClient.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if _, ok := persistedAG.Annotations[annotationVerifiedGeneration]; ok {
|
|
t.Errorf("generation marked verified although the rebuild was never forced: %v", persistedAG.Annotations)
|
|
}
|
|
persistedPS := &cozyv1alpha1.PackageSource{}
|
|
if err := baseClient.Get(context.Background(), client.ObjectKeyFromObject(ps), persistedPS); err != nil {
|
|
t.Fatalf("Get PS: %v", err)
|
|
}
|
|
if psReady := meta.FindStatusCondition(persistedPS.Status.Conditions, "Ready"); psReady != nil {
|
|
t.Errorf("PS Ready = %+v, want unset: the unverified Ready=True must not be copied through", psReady)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_VerificationIgnoresStaleRecoveryCounter pins that the
|
|
// verification rebuild starts its own attempt budget: clearing a counter left
|
|
// by an earlier stall is best-effort, and inheriting it could leave the
|
|
// verification a single backoff window before SourceWatcherStalled.
|
|
func TestUpdateStatus_VerificationIgnoresStaleRecoveryCounter(t *testing.T) {
|
|
ps, ag := builtAG(2, metav1.ConditionTrue, 2, map[string]string{
|
|
annotationRecoveryAttempts: "4",
|
|
annotationLastRecoveryAt: referenceTime.Add(-time.Hour).UTC().Format(time.RFC3339Nano),
|
|
})
|
|
baseClient := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
patches := 0
|
|
failing := interceptor.NewClient(baseClient, interceptor.Funcs{
|
|
Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
|
patches++
|
|
if patches == 1 {
|
|
return errors.New("simulated failure clearing recovery tracking")
|
|
}
|
|
return c.Patch(ctx, obj, patch, opts...)
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: failing, Scheme: testScheme(t)}
|
|
|
|
res, err := r.updateStatus(context.Background(), ps, referenceTime)
|
|
if err != nil {
|
|
t.Fatalf("updateStatus: %v", err)
|
|
}
|
|
if res.RequeueAfter != backoffFor(1) {
|
|
t.Errorf("RequeueAfter = %v, want %v", res.RequeueAfter, backoffFor(1))
|
|
}
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := baseClient.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if got := persistedAG.Annotations[annotationRecoveryAttempts]; got != "1" {
|
|
t.Errorf("recovery attempts = %q, want \"1\"", got)
|
|
}
|
|
}
|
|
|
|
// TestUpdateStatus_VerificationMarkFailure_RebuildsAgain covers the other
|
|
// half of that ordering: when the rebuild was forced but recording the
|
|
// generation failed, the next Ready=True forces the rebuild again.
|
|
func TestUpdateStatus_VerificationMarkFailure_RebuildsAgain(t *testing.T) {
|
|
ps, ag := builtAG(2, metav1.ConditionTrue, 2, nil)
|
|
baseClient := fake.NewClientBuilder().WithScheme(testScheme(t)).WithStatusSubresource(ps, ag).WithObjects(ps, ag).Build()
|
|
failMark := true
|
|
failing := interceptor.NewClient(baseClient, interceptor.Funcs{
|
|
Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
|
|
if _, marking := obj.GetAnnotations()[annotationVerifiedGeneration]; marking && failMark {
|
|
return errors.New("simulated metadata patch failure")
|
|
}
|
|
return c.Patch(ctx, obj, patch, opts...)
|
|
},
|
|
})
|
|
r := &PackageSourceReconciler{Client: failing, Scheme: testScheme(t)}
|
|
|
|
if _, err := r.updateStatus(context.Background(), ps, referenceTime); err == nil {
|
|
t.Fatal("updateStatus succeeded, want error from recording the verified generation")
|
|
}
|
|
persistedAG := &sourcewatcherv1beta1.ArtifactGenerator{}
|
|
if err := baseClient.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if _, ok := persistedAG.Annotations[annotationVerifiedGeneration]; ok {
|
|
t.Fatalf("verified generation persisted despite the failed patch: %v", persistedAG.Annotations)
|
|
}
|
|
firstForce := persistedAG.Annotations[annotationFluxRequestedAt]
|
|
if firstForce == "" {
|
|
t.Fatalf("rebuild was not forced before the mark: %v", persistedAG.Annotations)
|
|
}
|
|
|
|
// source-watcher finishes the forced rebuild and reports Ready=True again.
|
|
meta.SetStatusCondition(&persistedAG.Status.Conditions, metav1.Condition{
|
|
Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", ObservedGeneration: 2,
|
|
})
|
|
if err := baseClient.Status().Update(context.Background(), persistedAG); err != nil {
|
|
t.Fatalf("Update AG status: %v", err)
|
|
}
|
|
failMark = false
|
|
if _, err := r.updateStatus(context.Background(), ps, referenceTime.Add(time.Minute)); err != nil {
|
|
t.Fatalf("second updateStatus: %v", err)
|
|
}
|
|
if err := baseClient.Get(context.Background(), client.ObjectKeyFromObject(ag), persistedAG); err != nil {
|
|
t.Fatalf("Get AG: %v", err)
|
|
}
|
|
if got := persistedAG.Annotations[annotationFluxRequestedAt]; got == firstForce {
|
|
t.Errorf("requestedAt = %q unchanged, want a second forced rebuild", got)
|
|
}
|
|
if got := persistedAG.Annotations[annotationVerifiedGeneration]; got != "2" {
|
|
t.Errorf("verified generation = %q, want \"2\"", got)
|
|
}
|
|
}
|