cozystack/hack/cozyreport-summary.sh

#!/bin/sh
# Emit a human-readable summary of "what is broken" to a single file.
# Reads the live cluster (not the report dir) so it can use kubectl JSONPath.
# Usage: cozyreport-summary.sh > summary.txt
#
# Environment:
#   COZY_REPORT_PODS_MAX  how many not-Ready pods this summary lists before it
#                         reports the rest as an overflow (default 40, shared with
#                         the collector so the two never start from different
#                         numbers; the counts can still differ when the collector
#                         stops on its time budget, and its
#                         COLLECTION-TRUNCATED.txt explains that). A malformed value
#                         falls back to the default and says so in the section.
#                         Note it does not bound the HelmRelease listing, which has
#                         its own cap at the same default.
#   COZY_REPORT_PREFER_NS namespace prefix listed first (default `tenant-`; empty
#                         disables the ordering). Reached through the selection
#                         helpers sourced from hack/cozyreport.sh rather than read
#                         by name here, so it appears nowhere in this file's CODE
#                         (only in this block) -- which is exactly why it went
#                         undocumented:
#                         a knob arriving by `.` is invisible to a guard that greps
#                         for its name. It changes which pods this summary lists
#                         first, and the collector uses the same value, so the two
#                         orderings stay in step.
set -eu

# Share the pod-selection helper with cozyreport.sh rather than restating it:
# the summary and the collected evidence disagreeing about which pods are broken
# is worse than either being wrong on its own. COZYREPORT_LIB makes the source
# return before it starts a report of its own.
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# shellcheck disable=SC2034  # read by the sourcing guard in cozyreport.sh
COZYREPORT_LIB=1
# shellcheck source=hack/cozyreport.sh
. "$SCRIPT_DIR/cozyreport.sh"

# The wall-clock wrapper, taken from cozyreport.sh rather than resolved again so
# the two cannot hold different values. Empty when `timeout` is absent; the reads
# then run unbounded and the header says so.
BOUND=$COZYREPORT_BOUND

# How many rows any one section lists before it reports the rest as an overflow.
# Shares the collector's default so the two never START from different numbers.
# The counts can still differ, and legitimately: the collector also stops on its
# time budget, so it may hold fewer pods than this lists. Its own
# COLLECTION-TRUNCATED.txt says so; see the note on the pod listing below.
SUMMARY_CAP=$COZYREPORT_PODS_DEFAULT

# summary_read <what> <kubectl args...>
#
# One bounded read into READ_OUT. On failure it prints why the section is empty,
# because a killed read otherwise renders the section byte-identical to a healthy
# cluster -- and an empty "## HelmReleases not Ready" is the most misleading line
# this file can carry. CRD probes go through summary_crd, where a non-zero status
# has two meanings.
summary_read() {
  _sr_what=$1
  shift
  # kubectl's own reason is kept rather than sent to /dev/null, for the reason the
  # pod list below already keeps its own: "did not return (exit 1)" names a symptom
  # and no cause, and the difference between an apiserver that is too slow and one
  # answering "forbidden" is the whole finding. Discarding it here while the pod-list
  # read below keeps its own would have two reads in one file answering the same
  # question differently.
  _sr_err=$(mktemp "${TMPDIR:-/tmp}/cozyreport-summary.XXXXXX" 2>/dev/null) || _sr_err=""
  # The status is taken in an explicit `else`, not after the `if`: a compound `if`
  # whose condition fails and which has no else leaves `$?` at 0, so reading it
  # afterwards reports "exit 0" for every failure.
  # shellcheck disable=SC2086  # empty BOUND must vanish, not become ""
  if READ_OUT=$($BOUND "$@" 2>"${_sr_err:-/dev/null}"); then
    [ -z "$_sr_err" ] || rm -f "$_sr_err"
    return 0
  else
    _sr_rc=$?
  fi
  # The rows kubectl printed before it died are kept: they are the only ones it
  # named, and before this refactor they flowed down the pipe and were processed.
  # Discarding them turns a partial answer into no answer and then says nothing
  # was readable, which is false in the most useful direction.
  if [ -n "$READ_OUT" ]; then
    echo "  the $_sr_what read did not return (exit $_sr_rc); the rows below are the ones it printed before it stopped, and there may be others it never reached"
  else
    echo "  the $_sr_what read did not return (exit $_sr_rc), so this section is empty because nothing could be read, NOT because there was nothing to report"
  fi
  # LAST line, not the first, matching the pod list: a failing read is preceded by
  # one klog discovery-retry line per attempt and kubectl's own message comes after
  # them. A read killed by its own timeout is killed before kubectl writes anything,
  # so the file is empty and there is nothing to quote -- which is itself the
  # difference between "too slow to answer" and "answered, and the answer was no".
  if [ -n "$_sr_err" ] && [ -s "$_sr_err" ]; then
    # printf, not echo: kubectl's text is untrusted and dash's echo expands
    # backslash escapes, so a message carrying one could forge a line in a file a
    # triager reads as fact.
    printf '  kubectl said: %s\n' "$(tail -n 1 "$_sr_err" | cut -c1-200)"
  fi
  [ -z "$_sr_err" ] || rm -f "$_sr_err"
  return 0
}

# summary_crd <crd>: true when the CRD is present, false otherwise. Absence is the
# only outcome that answers false in silence; a killed read, a refusal with a
# message, a refusal without one, and a message that could not be captured each
# say which of them happened before answering false.
#
# `if $BOUND kubectl get crd X` collapses those last two into one, and a slow
# aggregated discovery layer -- the state that made the bound necessary -- then
# skips the whole section as though the CRD were not installed.
summary_crd() {
  # stderr kept for the same reason summary_read keeps it, but reported only on a
  # killed read: a CRD that is simply absent also exits non-zero, with kubectl
  # writing a NotFound to stderr, and quoting that would turn "this cluster does not
  # run cert-manager" into a line that reads like a failure. The killed case is the
  # one where the cause is not already implied by the outcome.
  _sc_err=$(mktemp "${TMPDIR:-/tmp}/cozyreport-summary.XXXXXX" 2>/dev/null) || _sc_err=""
  # shellcheck disable=SC2086
  if $BOUND kubectl get crd "$1" >/dev/null 2>"${_sc_err:-/dev/null}"; then
    [ -z "$_sc_err" ] || rm -f "$_sc_err"
    return 0
  else
    _sc_rc=$?
  fi
  if cozyreport_timed_out "$_sc_rc"; then
    echo "  the $1 presence check did not return (exit $_sc_rc), so this section was skipped without knowing whether the CRD is installed"
  elif [ -z "$_sc_err" ]; then
    # No scratch file means no writable temp dir here, so kubectl's message went to
    # /dev/null and is gone. Silence at this point would be the report's own local
    # condition rendered as a cluster fact, and rendered as the most common one:
    # every section this gate opens simply disappears, exactly as it does on a
    # cluster that does not run the operator behind the CRD.
    echo "  the $1 presence check failed (exit $_sc_rc) and its message could not be captured (no writable temp dir on the machine producing this report), so this section was skipped without knowing whether the CRD is installed"
  elif [ -s "$_sc_err" ]; then
    # Not a timeout and not a plain NotFound: an RBAC refusal on the CRD read skips
    # the section exactly like an absent CRD would, and only kubectl's message
    # separates "not installed" from "not allowed to look".
    if ! grep -qi 'not found' "$_sc_err"; then
      printf '  the %s presence check failed, so this section was skipped without knowing whether the CRD is installed; kubectl said: %s\n' \
        "$1" "$(tail -n 1 "$_sc_err" | cut -c1-200)"
    fi
  else
    # Exited non-zero having said nothing. Not the same as NotFound, which kubectl
    # is explicit about, so it cannot be filed under "the CRD is absent".
    echo "  the $1 presence check failed (exit $_sc_rc) without a message, so this section was skipped without knowing whether the CRD is installed"
  fi
  [ -z "$_sc_err" ] || rm -f "$_sc_err"
  return 1
}

# Printed before the first section, not inside one: every read in this file goes
# through the same empty $BOUND, so a notice sitting under "## Pods not Ready"
# leaves the reader of "## HelmReleases not Ready" -- which is above it and is the
# first section anyone opens -- with no indication at all. The collector states the
# same fact for its whole tree in COLLECTION-UNBOUNDED.txt rather than per section.
if [ -z "$BOUND" ]; then
  echo "(timeout is not on this host, so every read below is unbounded; this is a local dependency, not a cluster symptom)"
  echo
fi

echo "# Cozystack E2E Diagnostic Summary"
echo "Generated: $(date -Iseconds)"
echo

echo "## HelmReleases not Ready"
echo
if summary_crd helmreleases.helm.toolkit.fluxcd.io; then
  summary_read "HelmRelease" kubectl get hr -A --no-headers
  # The whole STATUS field, not $5. `kubectl get hr` prints NAMESPACE NAME AGE
  # READY STATUS, and STATUS is the Ready condition's *message* -- see the printer
  # columns on the CRD this repo ships, internal/fluxinstall/manifests/fluxcd.yaml
  # ("jsonPath: .status.conditions[?(@.type==\"Ready\")].message"). It is a whole
  # sentence, so $5 rendered every failed release as "Helm" or "Reconciliation",
  # dropping the part that says what broke, in the first section of the first file
  # a triager opens. Take everything from the fifth field on.
  # Rebuilt field by field rather than with a `{4}` interval expression: mawk
  # 1.3.4, the awk on the CI image, mis-applies an exact interval to a
  # variable-width group and strips one repetition instead of four. `sub()` still
  # returns 1, so there is no error and no signal -- the row simply arrives with
  # most of itself still in the message. Verified on ubuntu:24.04; gawk,
  # one-true-awk and busybox awk all handle it, which is how it would have reached
  # CI unnoticed.
  HR_ROWS=$(printf '%s\n' "$READ_OUT" | awk 'NF && $4 != "True" {
    msg = ""
    for (i = 5; i <= NF; i++) msg = msg (i > 5 ? " " : "") $i
    printf "  %s/%s — %s\n", $1, $2, msg
  }')
  HR_TOTAL=$(printf '%s\n' "$HR_ROWS" | grep -c . || true)
  if [ "$HR_TOTAL" -gt 0 ]; then
    printf '%s\n' "$HR_ROWS" | head -n "$SUMMARY_CAP"
  fi
  # The cap is reported for the same reason the pod cap is: an install that fails
  # early leaves well over 40 HelmReleases not Ready, and a listing that stops at 40
  # without saying so reads as the complete set.
  if [ "$HR_TOTAL" -gt "$SUMMARY_CAP" ]; then
    echo "  ... $((HR_TOTAL - SUMMARY_CAP)) more not-Ready HelmRelease(s) not listed here; the full set is in the report's kubernetes/helmreleases.txt"
  fi
fi
echo

echo "## Pods not Ready"
echo
# AGE is read as $NF, not $6: a pod that has restarted prints RESTARTS as
# `7 (4m12s ago)`, three awk fields, which shifts every later column.
# A restarted pod is the normal case in a failed run, so $6 printed `age=(4m12s`
# for exactly the rows worth reading. AGE is last whichever form RESTARTS takes.
#
# The row cap is reported rather than applied in silence: this section is a
# reader's first answer to "what is broken", and a cap that hides the rest reads
# like there is no rest. The ordering is shared with the collector, so the rows
# this cap keeps are the rows whose evidence the collector reached for first --
# the same rows, not necessarily the same count, since the collector can also stop
# on its time budget, and then it holds fewer pods than are listed here. Its own
# COLLECTION-TRUNCATED.txt says so.
#
# The pod list is read on its own, keeping its exit status, before anything is
# selected from it. Piping it straight into the selection discards that status,
# and then a list the apiserver refused renders byte-identical to a cluster where
# every pod is Ready -- in the file a triager opens first, and on exactly the run
# where the summary must not be the thing that says nothing is wrong.
POD_LIST_ERR=$(mktemp "${TMPDIR:-/tmp}/cozyreport-summary.XXXXXX" 2>/dev/null) || POD_LIST_ERR=""
# Bounded like the collector's copy of this read. The summary runs immediately
# before the tarball is written, so a hang here costs the whole artifact rather
# than this one section, and `|| true` on the caller does not survive a hang.
#
# `timeout` is used when it is there and skipped when it is not, rather than being
# required: without the fallback every read here exits 127 and this section would
# report that the cluster did not answer, on a host where the only thing missing is
# a local coreutils binary. That is the same false claim the collector's
# COLLECTION-UNBOUNDED.txt exists to avoid, and the two must not disagree inside
# one tarball. It also keeps working where the old unbounded pipeline did.
# Unquoted on purpose: an empty BOUND must expand to no argument at all.
# shellcheck disable=SC2086
if POD_LIST=$($BOUND kubectl get pod -A --no-headers 2>"${POD_LIST_ERR:-/dev/null}"); then
  POD_LIST_RC=0
else
  POD_LIST_RC=$?
fi
if [ "$POD_LIST_RC" -ne 0 ]; then
  # Two different failures, and the sentence has to match which one happened. A
  # read that produced nothing leaves an empty section; a read that printed rows
  # and then failed leaves the rows, and calling that section "empty" while the
  # rows are listed three lines below puts two contradictory statements about the
  # same read in one file. The partial rows are kept, as everywhere else in this
  # tree, because they are the only evidence there is -- they are just not all of
  # it, and the difference has to be said rather than implied.
  if [ -n "$POD_LIST" ]; then
    echo "  the cluster-wide pod list did not return (kubectl exit $POD_LIST_RC); the pods listed below are the ones it printed before it stopped, and there may be others it never reached"
  else
    echo "  the cluster-wide pod list did not return (kubectl exit $POD_LIST_RC), so this section is empty because nothing could be read, NOT because every pod was Ready"
  fi
  if [ -n "$POD_LIST_ERR" ] && [ -s "$POD_LIST_ERR" ]; then
    # Last line: a failing cluster-wide list is preceded by one klog discovery
    # retry line per attempt and kubectl's own message comes after them.
    printf '  kubectl said: %s\n' "$(tail -n 1 "$POD_LIST_ERR" | cut -c1-200)"
  fi
fi
[ -z "$POD_LIST_ERR" ] || rm -f "$POD_LIST_ERR"
POD_ROWS=$(printf '%s\n' "$POD_LIST" | cozyreport_pods_not_ready | cozyreport_pods_prioritize)
POD_TOTAL=$(printf '%s\n' "$POD_ROWS" | grep -c . || true)
POD_CAP=${COZY_REPORT_PODS_MAX:-$COZYREPORT_PODS_DEFAULT}
if ! cozyreport_is_count "$POD_CAP"; then
  echo "  (ignored a malformed COZY_REPORT_PODS_MAX='${COZY_REPORT_PODS_MAX:-}'; listing at most $COZYREPORT_PODS_DEFAULT)"
  POD_CAP=$COZYREPORT_PODS_DEFAULT
fi
# `NF` guard: an empty selection arrives as one empty line, and an unguarded action
# would print a row of blanks -- a nameless broken pod on every green run.
POD_LISTING=$(printf '%s\n' "$POD_ROWS" \
  | awk 'NF {printf "  %s/%s — %s ready=%s (restarts=%s, age=%s)\n", $1, $2, $4, $3, $5, $NF}')
# A zero cap is branched on rather than written as `A && B || C`, which would send
# `head -n 0` the value this guard keeps away from it (GNU head returns nothing,
# BSD/macOS head errors out).
if [ "$POD_CAP" -gt 0 ] && [ -n "$POD_LISTING" ]; then
  printf '%s\n' "$POD_LISTING" | head -n "$POD_CAP"
fi
if [ "${POD_TOTAL:-0}" -gt "$POD_CAP" ]; then
  echo "  ... $((POD_TOTAL - POD_CAP)) more not-Ready pod(s) not listed here; the full set is in kubernetes/pods.txt"
fi
echo

echo "## ImagePullBackOff / ErrImagePull"
echo
# Reuses the list already read above rather than calling the apiserver a second
# time for the same rows: one fewer cluster-wide read on the path that runs
# immediately before the tarball is written, and the two sections cannot disagree
# about what the cluster looked like.
#
# Which means it inherits that read's failure too. Without this guard the section
# is byte-identical between a healthy cluster and one whose pod list was refused,
# and a triager who jumps straight to this heading reads "nothing is failing to
# pull" off a list that never arrived.
if [ "$POD_LIST_RC" -ne 0 ]; then
  echo "  not known: this section is derived from the cluster-wide pod list above, which did not return"
else
  printf '%s\n' "$POD_LIST" \
    | awk 'NF && $4 ~ /ImagePullBackOff|ErrImagePull/ {printf "  %s/%s — %s\n", $1, $2, $4}'
fi
echo

echo "## Recent OOMKilled events (last 20)"
echo
summary_read "OOMKilling event" kubectl get events -A --field-selector reason=OOMKilling --sort-by=.lastTimestamp
printf '%s\n' "$READ_OUT" | awk 'NF' | tail -20
echo

echo "## Recent Warning events (last 30)"
echo
summary_read "Warning event" kubectl get events -A --field-selector type=Warning --sort-by=.lastTimestamp
printf '%s\n' "$READ_OUT" | awk 'NF' | tail -30
echo

echo "## cert-manager: Certificates not Ready"
echo
if summary_crd certificates.cert-manager.io; then
  summary_read "Certificate" kubectl get certificates.cert-manager.io -A --no-headers
  printf '%s\n' "$READ_OUT" | awk 'NF && $3 != "True" {printf "  %s/%s — Ready=%s\n", $1, $2, $3}'
fi
echo

echo "## Flux Sources not Ready"
echo
for kind in helmrepositories.source.toolkit.fluxcd.io ocirepositories.source.toolkit.fluxcd.io gitrepositories.source.toolkit.fluxcd.io externalartifacts.source.toolkit.fluxcd.io; do
  summary_crd "$kind" || continue
  short=${kind%%.*}
  # custom-columns, not jsonpath. The jsonpath this replaces nested one filter
  # inside another -- `.items[?(...conditions[?(@.type=="Ready")]...)]` -- which
  # kubectl rejects outright with "unterminated filter", so this section printed
  # nothing on every report ever taken and read as "all Flux sources are Ready".
  # A form that sends stderr to /dev/null and drops the status hides that
  # completely: the section renders empty and reads as "all sources Ready".
  #
  # Not the printer columns either: READY sits at a different index per kind, and
  # STATUS carries spaces, so column counting is wrong for at least one of the four.
  # custom-columns takes a single filter, the same shape the node section below uses.
  summary_read "$short" kubectl get "$kind" -A --no-headers \
    -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,READY:.status.conditions[?(@.type=="Ready")].status'
  printf '%s\n' "$READ_OUT" | awk -v k="$short" 'NF && $3 != "True" {printf "  %s %s/%s — Ready=%s\n", k, $1, $2, $3}'
done
echo

echo "## Storage: PVCs not Bound, PVs not Bound"
echo
summary_read "PVC" kubectl get pvc -A --no-headers
printf '%s\n' "$READ_OUT" | awk 'NF && $3 != "Bound" {printf "  PVC %s/%s — %s\n", $1, $2, $3}'
summary_read "PV" kubectl get pv --no-headers
printf '%s\n' "$READ_OUT" | awk 'NF && $5 != "Bound" {printf "  PV %s — %s\n", $1, $5}'
echo

echo "## Node Conditions"
summary_read "node condition" kubectl get nodes -o 'custom-columns=NAME:.metadata.name,READY:.status.conditions[?(@.type=="Ready")].status,DISK:.status.conditions[?(@.type=="DiskPressure")].status,MEM:.status.conditions[?(@.type=="MemoryPressure")].status'
[ -z "$READ_OUT" ] || printf '%s\n' "$READ_OUT"