mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-12 23:54:01 +09:00
fix(mariadb): keep the operator's message in deadline failures
The operator sets Complete=False with reason Failed and its own cause
in the message ("Error creating Job", "Backup not complete") when a
run fails before or outside its Job. Such a run waits out the
deadline, and the deadline failure kept only the status and reason,
so the BackupJob or RestoreJob ended without the one line that says
why.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
This commit is contained in:
1 parent
9f51b9db04
commit
69cea9ccbd
2 files changed
+33
-10
No files matched your search
@@ -242,12 +242,8 @@ func (r *BackupJobReconciler) reconcileMariaDB(ctx context.Context, j *backupsv1
|
||||
// Backup eventually fails the BackupJob instead of pinning it Running
|
||||
// forever.
|
||||
if mariadbBackupDeadlineExceeded(j.Status.StartedAt) {
|
||||
detail := "no Complete condition observed"
|
||||
if cond != nil {
|
||||
detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason)
|
||||
}
|
||||
return r.markBackupJobFailed(ctx, j, fmt.Sprintf(
|
||||
"k8s.mariadb.com Backup did not complete within %s (%s)", mariadbDefaultBackupDeadline, detail))
|
||||
"k8s.mariadb.com Backup did not complete within %s (%s)", mariadbDefaultBackupDeadline, mariadbCompleteDetail(cond)))
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
|
||||
}
|
||||
@@ -273,6 +269,20 @@ func mariadbJobFailedMessage(ctx context.Context, c client.Reader, kind, namespa
|
||||
return fmt.Sprintf("%s; Job %s/%s reports no Failed condition", message, namespace, name)
|
||||
}
|
||||
|
||||
// mariadbCompleteDetail describes the last Complete condition for a deadline
|
||||
// failure. The message matters for reason Failed, which the operator sets
|
||||
// with its own cause ("Error creating Job", "Backup not complete") rather
|
||||
// than from a Job.
|
||||
func mariadbCompleteDetail(cond *metav1.Condition) string {
|
||||
if cond == nil {
|
||||
return "no Complete condition observed"
|
||||
}
|
||||
if cond.Message == "" {
|
||||
return fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason)
|
||||
}
|
||||
return fmt.Sprintf("Complete=%s/%s: %s", cond.Status, cond.Reason, cond.Message)
|
||||
}
|
||||
|
||||
// mariadbBackupDeadlineExceeded reports whether enough wall-clock time has
|
||||
// elapsed since the BackupJob started that we should give up on a stuck
|
||||
// operator Backup. Returns false when StartedAt is nil so the very first
|
||||
@@ -605,13 +615,9 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest
|
||||
default:
|
||||
deadline := options.effectiveRestoreDeadline()
|
||||
if restoreJob.Status.StartedAt != nil && time.Since(restoreJob.Status.StartedAt.Time) > deadline {
|
||||
detail := "no Complete condition observed"
|
||||
if cond != nil {
|
||||
detail = fmt.Sprintf("Complete=%s/%s", cond.Status, cond.Reason)
|
||||
}
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf(
|
||||
"k8s.mariadb.com/Restore did not complete within %s (%s; override via spec.options.restoreTimeoutSeconds)",
|
||||
deadline, detail))
|
||||
deadline, mariadbCompleteDetail(cond)))
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: mariadbPollInterval}, nil
|
||||
}
|
||||
|
||||
@@ -901,6 +901,8 @@ var mariadbCompleteOutcomeCases = []struct {
|
||||
wantMessage: []string{"no Complete condition observed"}},
|
||||
{name: "Complete=True with an unknown reason past the deadline fails", status: metav1.ConditionTrue, reason: "SomethingElse", message: "Something", expired: true, wantFailed: true,
|
||||
wantMessage: []string{"Complete=True/SomethingElse"}},
|
||||
{name: "Complete=False/Failed past the deadline carries the operator's message", status: metav1.ConditionFalse, reason: "Failed", message: "Error creating Job", expired: true, wantFailed: true,
|
||||
wantMessage: []string{"Complete=False/Failed: Error creating Job"}},
|
||||
}
|
||||
|
||||
func mariadbCompleteConditions(noCondition bool, status metav1.ConditionStatus, reason, message string) []metav1.Condition {
|
||||
@@ -1078,6 +1080,21 @@ func TestMariaDBJobFailedMessage_JobWithoutFailedCondition(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMariaDBCompleteDetail(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
cond *metav1.Condition
|
||||
want string
|
||||
}{
|
||||
{nil, "no Complete condition observed"},
|
||||
{&metav1.Condition{Status: metav1.ConditionFalse, Reason: "Failed", Message: "Backup not complete"}, "Complete=False/Failed: Backup not complete"},
|
||||
{&metav1.Condition{Status: metav1.ConditionFalse, Reason: "Failed"}, "Complete=False/Failed"},
|
||||
} {
|
||||
if got := mariadbCompleteDetail(tc.cond); got != tc.want {
|
||||
t.Errorf("mariadbCompleteDetail(%+v) = %q, want %q", tc.cond, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileMariaDBRestore_CompleteConditionOutcome(t *testing.T) {
|
||||
apps := mariadbapp.GroupName
|
||||
|
||||
|
||||
Reference in new issue
Block a user