mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(workloadmonitor): keep a bucket not ready until its claim is provisioned
A bucket WorkloadMonitor has no pods and minReplicas 0, so it stayed operational, and the Bucket app stayed Ready, while its BucketClaim was never provisioned. A tenant whose nearest S3 ancestor runs no SeaweedFS gets a claim for a bucket class that does not exist, and the dashboard showed a ready bucket that was never created. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
This commit is contained in:
1 parent
94ada11eef
commit
9d33037566
3 files changed
+168
-3
No files matched your search
@@ -39,6 +39,10 @@ type WorkloadMonitorSpec struct {
|
||||
// that is not operational because a DataVolume it selects is not ready.
|
||||
const WorkloadMonitorReasonDataVolumeNotReady = "DataVolumeNotReady"
|
||||
|
||||
// WorkloadMonitorReasonBucketClaimNotReady is the status Reason of a monitor
|
||||
// that is not operational because a BucketClaim it selects is not provisioned.
|
||||
const WorkloadMonitorReasonBucketClaimNotReady = "BucketClaimNotReady"
|
||||
|
||||
// WorkloadMonitorStatus defines the observed state of WorkloadMonitor
|
||||
type WorkloadMonitorStatus struct {
|
||||
// Operational indicates if the workload meets all operational requirements
|
||||
|
||||
@@ -374,6 +374,25 @@ func (r *WorkloadMonitorReconciler) isBucketClaimReady(bc *cosiv1alpha1.BucketCl
|
||||
return bc.Status.BucketReady
|
||||
}
|
||||
|
||||
// bucketClaimsMessage names every BucketClaim that has not been provisioned,
|
||||
// with its bucket class, and is empty when there is none.
|
||||
func (r *WorkloadMonitorReconciler) bucketClaimsMessage(claims []cosiv1alpha1.BucketClaim) string {
|
||||
var pending []string
|
||||
for i := range claims {
|
||||
bc := &claims[i]
|
||||
if r.isBucketClaimReady(bc) {
|
||||
continue
|
||||
}
|
||||
entry := fmt.Sprintf("BucketClaim %s is not provisioned", bc.Name)
|
||||
if bc.Spec.BucketClassName != "" {
|
||||
entry = fmt.Sprintf("BucketClaim %s of bucket class %s is not provisioned", bc.Name, bc.Spec.BucketClassName)
|
||||
}
|
||||
pending = append(pending, entry)
|
||||
}
|
||||
sort.Strings(pending)
|
||||
return strings.Join(pending, "; ")
|
||||
}
|
||||
|
||||
// isServiceReady checks if the service has an external IP bound
|
||||
func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool {
|
||||
return len(svc.Status.LoadBalancer.Ingress) > 0
|
||||
@@ -1157,6 +1176,8 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
|
||||
}
|
||||
}
|
||||
|
||||
bucketsMessage := r.bucketClaimsMessage(bucketClaimList.Items)
|
||||
|
||||
// Update WorkloadMonitor status based on observed pods
|
||||
monitor.Status.ObservedReplicas = observedReplicas
|
||||
monitor.Status.AvailableReplicas = availableReplicas
|
||||
@@ -1178,15 +1199,23 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
|
||||
if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas {
|
||||
fresh.Status.Operational = pointer.Bool(false)
|
||||
}
|
||||
// Only the DataVolume check writes Message and Reason, so the stored ones
|
||||
// are the last DataVolume verdict: kept while no reader is installed yet, so a
|
||||
// controller restart does not report a stuck disk as ready.
|
||||
// A stored DataVolume verdict is the last one: kept while no reader is
|
||||
// installed yet, so a controller restart does not report a stuck disk as
|
||||
// ready. BucketClaims are listed on every pass, so their verdict is
|
||||
// recomputed and only fills fields no DataVolume verdict holds.
|
||||
if dataVolumesRead {
|
||||
fresh.Status.Message = dataVolumesMessage
|
||||
fresh.Status.Reason = ""
|
||||
if dataVolumesMessage != "" {
|
||||
fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady
|
||||
}
|
||||
} else if fresh.Status.Reason == cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady {
|
||||
fresh.Status.Message = ""
|
||||
fresh.Status.Reason = ""
|
||||
}
|
||||
if fresh.Status.Message == "" && bucketsMessage != "" {
|
||||
fresh.Status.Message = bucketsMessage
|
||||
fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady
|
||||
}
|
||||
if fresh.Status.Message != "" {
|
||||
fresh.Status.Operational = pointer.Bool(false)
|
||||
|
||||
@@ -2975,3 +2975,135 @@ func TestReconcileBucketApp_LegacySizeWinsOverOwnerlessNewWorkload(t *testing.T)
|
||||
t.Errorf("size = %s, want the legacy 689Gi over the stale 10Gi", got.String())
|
||||
}
|
||||
}
|
||||
|
||||
func newBucketClaim(name string, labels map[string]string, class string, ready bool) *cosiv1alpha1.BucketClaim {
|
||||
return &cosiv1alpha1.BucketClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Labels: labels},
|
||||
Spec: cosiv1alpha1.BucketClaimSpec{
|
||||
BucketClassName: class,
|
||||
Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3},
|
||||
},
|
||||
Status: cosiv1alpha1.BucketClaimStatus{BucketReady: ready},
|
||||
}
|
||||
}
|
||||
|
||||
func reconcileBucketMonitor(t *testing.T, c client.Client, s *runtime.Scheme, withReader bool) *cozyv1alpha1.WorkloadMonitor {
|
||||
t.Helper()
|
||||
reconciler := &WorkloadMonitorReconciler{Client: c, Scheme: s}
|
||||
if withReader {
|
||||
reconciler.DataVolumeReader = c
|
||||
}
|
||||
req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-bucket", Namespace: "default"}}
|
||||
if _, err := reconciler.Reconcile(context.TODO(), req); err != nil {
|
||||
t.Fatalf("Reconcile returned error: %v", err)
|
||||
}
|
||||
updated := &cozyv1alpha1.WorkloadMonitor{}
|
||||
if err := c.Get(context.TODO(), req.NamespacedName, updated); err != nil {
|
||||
t.Fatalf("Failed to get updated WorkloadMonitor: %v", err)
|
||||
}
|
||||
if updated.Status.Operational == nil {
|
||||
t.Fatal("Expected Operational to be set, got nil")
|
||||
}
|
||||
return updated
|
||||
}
|
||||
|
||||
func newBucketMonitorClient(s *runtime.Scheme, objs ...client.Object) client.Client {
|
||||
monitor := &cozyv1alpha1.WorkloadMonitor{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "my-bucket", Namespace: "default"},
|
||||
Spec: cozyv1alpha1.WorkloadMonitorSpec{
|
||||
Kind: "bucket",
|
||||
Type: "s3",
|
||||
Selector: map[string]string{"app.kubernetes.io/instance": "my-bucket"},
|
||||
Replicas: ptr.To[int32](0),
|
||||
MinReplicas: ptr.To[int32](0),
|
||||
},
|
||||
}
|
||||
return fake.NewClientBuilder().
|
||||
WithScheme(s).
|
||||
WithObjects(append([]client.Object{monitor}, objs...)...).
|
||||
WithStatusSubresource(monitor).
|
||||
Build()
|
||||
}
|
||||
|
||||
func TestReconcile_BucketClaimDecidesOperational(t *testing.T) {
|
||||
selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"}
|
||||
cases := []struct {
|
||||
name string
|
||||
claim *cosiv1alpha1.BucketClaim
|
||||
operational bool
|
||||
message string
|
||||
}{
|
||||
{"provisioned", newBucketClaim("my-bucket", selected, "tenant-root", true), true, ""},
|
||||
{"not provisioned", newBucketClaim("my-bucket", selected, "tenant-root", false), false, "BucketClaim my-bucket of bucket class tenant-root is not provisioned"},
|
||||
{"not provisioned without a class", newBucketClaim("my-bucket", selected, "", false), false, "BucketClaim my-bucket is not provisioned"},
|
||||
{"outside the selector", newBucketClaim("other", map[string]string{"app.kubernetes.io/instance": "other"}, "tenant-root", false), true, ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := newTestScheme()
|
||||
got := reconcileBucketMonitor(t, newBucketMonitorClient(s, tc.claim), s, true)
|
||||
if *got.Status.Operational != tc.operational {
|
||||
t.Errorf("Operational = %v, want %v", *got.Status.Operational, tc.operational)
|
||||
}
|
||||
if got.Status.Message != tc.message {
|
||||
t.Errorf("Message = %q, want %q", got.Status.Message, tc.message)
|
||||
}
|
||||
wantReason := ""
|
||||
if tc.message != "" {
|
||||
wantReason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady
|
||||
}
|
||||
if got.Status.Reason != wantReason {
|
||||
t.Errorf("Reason = %q, want %q", got.Status.Reason, wantReason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcile_ProvisionedBucketClaimClearsTheVerdict(t *testing.T) {
|
||||
for _, withReader := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("dataVolumeReader=%v", withReader), func(t *testing.T) {
|
||||
s := newTestScheme()
|
||||
claim := newBucketClaim("my-bucket", map[string]string{"app.kubernetes.io/instance": "my-bucket"}, "tenant-root", false)
|
||||
c := newBucketMonitorClient(s, claim)
|
||||
|
||||
got := reconcileBucketMonitor(t, c, s, withReader)
|
||||
if *got.Status.Operational || got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady {
|
||||
t.Fatalf("before provisioning: Operational = %v, Reason = %q", *got.Status.Operational, got.Status.Reason)
|
||||
}
|
||||
|
||||
stored := &cosiv1alpha1.BucketClaim{}
|
||||
if err := c.Get(context.TODO(), types.NamespacedName{Name: "my-bucket", Namespace: "default"}, stored); err != nil {
|
||||
t.Fatalf("Failed to get BucketClaim: %v", err)
|
||||
}
|
||||
stored.Status.BucketReady = true
|
||||
if err := c.Update(context.TODO(), stored); err != nil {
|
||||
t.Fatalf("Failed to update BucketClaim: %v", err)
|
||||
}
|
||||
|
||||
got = reconcileBucketMonitor(t, c, s, withReader)
|
||||
if !*got.Status.Operational || got.Status.Message != "" || got.Status.Reason != "" {
|
||||
t.Errorf("after provisioning: Operational = %v, Message = %q, Reason = %q", *got.Status.Operational, got.Status.Message, got.Status.Reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcile_DataVolumeVerdictOutranksBucketClaim(t *testing.T) {
|
||||
s := newTestScheme()
|
||||
selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"}
|
||||
phase := "ImportInProgress"
|
||||
c := newBucketMonitorClient(s,
|
||||
newBucketClaim("my-bucket", selected, "tenant-root", false),
|
||||
newDataVolume("my-bucket", selected, &phase),
|
||||
)
|
||||
got := reconcileBucketMonitor(t, c, s, true)
|
||||
if *got.Status.Operational {
|
||||
t.Error("Operational = true, want false")
|
||||
}
|
||||
if got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady {
|
||||
t.Errorf("Reason = %q, want %q", got.Status.Reason, cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady)
|
||||
}
|
||||
if got.Status.Message != "DataVolume my-bucket is ImportInProgress" {
|
||||
t.Errorf("Message = %q", got.Status.Message)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user