fix(workloadmonitor): keep a bucket not ready until its claim is provisioned

A bucket WorkloadMonitor has no pods and minReplicas 0, so it stayed
operational, and the Bucket app stayed Ready, while its BucketClaim was
never provisioned. A tenant whose nearest S3 ancestor runs no SeaweedFS
gets a claim for a bucket class that does not exist, and the dashboard
showed a ready bucket that was never created.

Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
This commit is contained in:
Yan Bondarenko committed 2026-11-01 11:42:17 +03:00
1 parent 94ada11eef
commit 9d33037566
3 files changed
+168 -3

No files matched your search

+4
View File
@@ -39,6 +39,10 @@ type WorkloadMonitorSpec struct {
// that is not operational because a DataVolume it selects is not ready.
const WorkloadMonitorReasonDataVolumeNotReady = "DataVolumeNotReady"
// WorkloadMonitorReasonBucketClaimNotReady is the status Reason of a monitor
// that is not operational because a BucketClaim it selects is not provisioned.
const WorkloadMonitorReasonBucketClaimNotReady = "BucketClaimNotReady"
// WorkloadMonitorStatus defines the observed state of WorkloadMonitor
type WorkloadMonitorStatus struct {
// Operational indicates if the workload meets all operational requirements
@@ -374,6 +374,25 @@ func (r *WorkloadMonitorReconciler) isBucketClaimReady(bc *cosiv1alpha1.BucketCl
return bc.Status.BucketReady
}
// bucketClaimsMessage names every BucketClaim that has not been provisioned,
// with its bucket class, and is empty when there is none.
func (r *WorkloadMonitorReconciler) bucketClaimsMessage(claims []cosiv1alpha1.BucketClaim) string {
var pending []string
for i := range claims {
bc := &claims[i]
if r.isBucketClaimReady(bc) {
continue
}
entry := fmt.Sprintf("BucketClaim %s is not provisioned", bc.Name)
if bc.Spec.BucketClassName != "" {
entry = fmt.Sprintf("BucketClaim %s of bucket class %s is not provisioned", bc.Name, bc.Spec.BucketClassName)
}
pending = append(pending, entry)
}
sort.Strings(pending)
return strings.Join(pending, "; ")
}
// isServiceReady checks if the service has an external IP bound
func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool {
return len(svc.Status.LoadBalancer.Ingress) > 0
@@ -1157,6 +1176,8 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}
}
bucketsMessage := r.bucketClaimsMessage(bucketClaimList.Items)
// Update WorkloadMonitor status based on observed pods
monitor.Status.ObservedReplicas = observedReplicas
monitor.Status.AvailableReplicas = availableReplicas
@@ -1178,15 +1199,23 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas {
fresh.Status.Operational = pointer.Bool(false)
}
// Only the DataVolume check writes Message and Reason, so the stored ones
// are the last DataVolume verdict: kept while no reader is installed yet, so a
// controller restart does not report a stuck disk as ready.
// A stored DataVolume verdict is the last one: kept while no reader is
// installed yet, so a controller restart does not report a stuck disk as
// ready. BucketClaims are listed on every pass, so their verdict is
// recomputed and only fills fields no DataVolume verdict holds.
if dataVolumesRead {
fresh.Status.Message = dataVolumesMessage
fresh.Status.Reason = ""
if dataVolumesMessage != "" {
fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady
}
} else if fresh.Status.Reason == cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady {
fresh.Status.Message = ""
fresh.Status.Reason = ""
}
if fresh.Status.Message == "" && bucketsMessage != "" {
fresh.Status.Message = bucketsMessage
fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady
}
if fresh.Status.Message != "" {
fresh.Status.Operational = pointer.Bool(false)
@@ -2975,3 +2975,135 @@ func TestReconcileBucketApp_LegacySizeWinsOverOwnerlessNewWorkload(t *testing.T)
t.Errorf("size = %s, want the legacy 689Gi over the stale 10Gi", got.String())
}
}
func newBucketClaim(name string, labels map[string]string, class string, ready bool) *cosiv1alpha1.BucketClaim {
return &cosiv1alpha1.BucketClaim{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Labels: labels},
Spec: cosiv1alpha1.BucketClaimSpec{
BucketClassName: class,
Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3},
},
Status: cosiv1alpha1.BucketClaimStatus{BucketReady: ready},
}
}
func reconcileBucketMonitor(t *testing.T, c client.Client, s *runtime.Scheme, withReader bool) *cozyv1alpha1.WorkloadMonitor {
t.Helper()
reconciler := &WorkloadMonitorReconciler{Client: c, Scheme: s}
if withReader {
reconciler.DataVolumeReader = c
}
req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-bucket", Namespace: "default"}}
if _, err := reconciler.Reconcile(context.TODO(), req); err != nil {
t.Fatalf("Reconcile returned error: %v", err)
}
updated := &cozyv1alpha1.WorkloadMonitor{}
if err := c.Get(context.TODO(), req.NamespacedName, updated); err != nil {
t.Fatalf("Failed to get updated WorkloadMonitor: %v", err)
}
if updated.Status.Operational == nil {
t.Fatal("Expected Operational to be set, got nil")
}
return updated
}
func newBucketMonitorClient(s *runtime.Scheme, objs ...client.Object) client.Client {
monitor := &cozyv1alpha1.WorkloadMonitor{
ObjectMeta: metav1.ObjectMeta{Name: "my-bucket", Namespace: "default"},
Spec: cozyv1alpha1.WorkloadMonitorSpec{
Kind: "bucket",
Type: "s3",
Selector: map[string]string{"app.kubernetes.io/instance": "my-bucket"},
Replicas: ptr.To[int32](0),
MinReplicas: ptr.To[int32](0),
},
}
return fake.NewClientBuilder().
WithScheme(s).
WithObjects(append([]client.Object{monitor}, objs...)...).
WithStatusSubresource(monitor).
Build()
}
func TestReconcile_BucketClaimDecidesOperational(t *testing.T) {
selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"}
cases := []struct {
name string
claim *cosiv1alpha1.BucketClaim
operational bool
message string
}{
{"provisioned", newBucketClaim("my-bucket", selected, "tenant-root", true), true, ""},
{"not provisioned", newBucketClaim("my-bucket", selected, "tenant-root", false), false, "BucketClaim my-bucket of bucket class tenant-root is not provisioned"},
{"not provisioned without a class", newBucketClaim("my-bucket", selected, "", false), false, "BucketClaim my-bucket is not provisioned"},
{"outside the selector", newBucketClaim("other", map[string]string{"app.kubernetes.io/instance": "other"}, "tenant-root", false), true, ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s := newTestScheme()
got := reconcileBucketMonitor(t, newBucketMonitorClient(s, tc.claim), s, true)
if *got.Status.Operational != tc.operational {
t.Errorf("Operational = %v, want %v", *got.Status.Operational, tc.operational)
}
if got.Status.Message != tc.message {
t.Errorf("Message = %q, want %q", got.Status.Message, tc.message)
}
wantReason := ""
if tc.message != "" {
wantReason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady
}
if got.Status.Reason != wantReason {
t.Errorf("Reason = %q, want %q", got.Status.Reason, wantReason)
}
})
}
}
func TestReconcile_ProvisionedBucketClaimClearsTheVerdict(t *testing.T) {
for _, withReader := range []bool{true, false} {
t.Run(fmt.Sprintf("dataVolumeReader=%v", withReader), func(t *testing.T) {
s := newTestScheme()
claim := newBucketClaim("my-bucket", map[string]string{"app.kubernetes.io/instance": "my-bucket"}, "tenant-root", false)
c := newBucketMonitorClient(s, claim)
got := reconcileBucketMonitor(t, c, s, withReader)
if *got.Status.Operational || got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady {
t.Fatalf("before provisioning: Operational = %v, Reason = %q", *got.Status.Operational, got.Status.Reason)
}
stored := &cosiv1alpha1.BucketClaim{}
if err := c.Get(context.TODO(), types.NamespacedName{Name: "my-bucket", Namespace: "default"}, stored); err != nil {
t.Fatalf("Failed to get BucketClaim: %v", err)
}
stored.Status.BucketReady = true
if err := c.Update(context.TODO(), stored); err != nil {
t.Fatalf("Failed to update BucketClaim: %v", err)
}
got = reconcileBucketMonitor(t, c, s, withReader)
if !*got.Status.Operational || got.Status.Message != "" || got.Status.Reason != "" {
t.Errorf("after provisioning: Operational = %v, Message = %q, Reason = %q", *got.Status.Operational, got.Status.Message, got.Status.Reason)
}
})
}
}
func TestReconcile_DataVolumeVerdictOutranksBucketClaim(t *testing.T) {
s := newTestScheme()
selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"}
phase := "ImportInProgress"
c := newBucketMonitorClient(s,
newBucketClaim("my-bucket", selected, "tenant-root", false),
newDataVolume("my-bucket", selected, &phase),
)
got := reconcileBucketMonitor(t, c, s, true)
if *got.Status.Operational {
t.Error("Operational = true, want false")
}
if got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady {
t.Errorf("Reason = %q, want %q", got.Status.Reason, cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady)
}
if got.Status.Message != "DataVolume my-bucket is ImportInProgress" {
t.Errorf("Message = %q", got.Status.Message)
}
}