fix(seaweedfs): remove identities left behind by deleted BucketAccesses

Since SeaweedFS 4.08 the S3 gateways move every identity from
/etc/iam/identity.json into /etc/iam/identities/<name>.json, and the
COSI driver up to v0.3.1 revokes only from identity.json. Every
BucketAccess deleted on such a filer left its identity file behind,
with its keys still accepted (seaweedfs/seaweedfs-cosi-driver#11). A
driver that deletes the file on revoke does nothing for the files
already left.

A post-install/post-upgrade hook removes a ba-<uid>.json file when no
BucketAccess in any namespace names it, by uid or by
status.accountID. The uid covers a BucketAccess still being granted,
the accountID one restored with its status. The identity directory is
listed before the BucketAccesses: a BucketAccess exists before its
grant writes a file, so a file seen in the first listing whose
BucketAccess still exists is always named by the second. File ages are
not used, because grants and gateway migrations rewrite left-over
files as well. A failed or empty listing removes nothing. Each removal
is logged and recorded as an Event on the HelmRelease. The hook runs
before the Object Lock hook, so in delete mode a left-over identity is
removed before that hook grants it anything.

A filer published for Client-topology clusters holds identities whose
BucketAccesses live in those clusters, so cosi.orphanIdentities
defaults to auto: report there, delete otherwise. An operator can
force either.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
Aleksei Sviridkin committed 2026-11-17 22:07:25 +03:00
1 parent c49f58c4a4
commit 045f1a9bbc
11 files changed
+970 -2

No files matched your search

+336
View File
@@ -0,0 +1,336 @@
#!/usr/bin/env bats
# Unit tests for packages/system/seaweedfs/files/orphan-identities.sh, the
# post-install/post-upgrade hook that removes the per-identity IAM files of BucketAccesses
# that no longer exist. Drives it against a fake `weed shell` that keeps the
# identity files in a directory, and a fake kubectl.
load test_helper
SCRIPT="$PWD/packages/system/seaweedfs/files/orphan-identities.sh"
OTHER=0b5e1a7e-0000-4000-8000-000000000009
LIVE=0b5e1a7e-0000-4000-8000-00000000000a
GONE=0b5e1a7e-0000-4000-8000-00000000000b
GONE2=0b5e1a7e-0000-4000-8000-00000000000c
setup() {
strict_setup
BIN="$BATS_TEST_TMPDIR/bin"
mkdir -p "$BIN"
export IDS="$BATS_TEST_TMPDIR/identities"
export BA_JSON="$BATS_TEST_TMPDIR/bucketaccesses.json"
export KUBECTL_LOG="$BATS_TEST_TMPDIR/kubectl"
export WEED_LOG="$BATS_TEST_TMPDIR/weed"
mkdir -p "$IDS"
: > "$KUBECTL_LOG"
: > "$WEED_LOG"
# One BucketAccess with no file, so the listing is never empty by accident.
echo '{"apiVersion":"v1","kind":"List","items":[{"metadata":{"uid":"'"$OTHER"'"},"status":{"accountID":"ba-'"$OTHER"'"}}]}' > "$BA_JSON"
cat > "$BIN/weed" <<'EOF'
#!/bin/bash
# A weed shell session over $IDS, one file per identity.
# Like the real one it reports a failed command on stderr and exits 0.
# WEED_SLEEP hangs every session. WEED_LS_FAIL fails a listing (WEED_LS_FAIL_FROM: from that listing on),
# WEED_RM_FAIL makes every removal fail.
set -eu
[ "$1" = shell ] || exit 2
echo session >> "$WEED_LOG"
[ -z "${WEED_SLEEP:-}" ] || sleep "$WEED_SLEEP"
while read -r cmd path; do
echo "$cmd $path" >> "$WEED_LOG"
case $cmd in
fs.ls)
[ "$path" = /etc/iam/identities ] || exit 3
n=$(($(cat "$WEED_LOG.ls" 2>/dev/null || echo 0) + 1)); echo "$n" > "$WEED_LOG.ls"
[ "$n" -ge "${WEED_LS_FAIL_FROM:-1}" ] || { ls -1 "$IDS"; continue; }
case ${WEED_LS_FAIL:-} in
exit) exit 1 ;;
print) echo "error: list $path: connection refused" >&2; continue ;;
esac
ls -1 "$IDS"
;;
fs.rm)
[ -z "${WEED_RM_FAIL:-}" ] || { echo "error: rm $path: permission denied" >&2; continue; }
rm "$IDS/${path##*/}"
;;
*)
exit 4
;;
esac
done
EOF
cat > "$BIN/kubectl" <<'EOF'
#!/bin/sh
echo "ARGS: $*" >> "$KUBECTL_LOG"
# Like the real one: any --request-timeout drops the in-cluster credentials.
case " $* " in
*" --request-timeout"*) echo 'The connection to the server localhost:8080 was refused' >&2; exit 1 ;;
esac
[ -z "${KUBECTL_SLEEP:-}" ] || exec sleep "$KUBECTL_SLEEP"
case $1 in
get)
case $2 in
helmrelease) printf 0b5e1a7e-0000-4000-8000-000000000001 ;;
*)
[ -z "${KUBECTL_BA_FAIL:-}" ] || { echo 'Error from server (Forbidden): bucketaccesses is forbidden' >&2; exit 1; }
# Child tenants keep their BucketAccesses in their own namespaces.
case " $* " in
*" --all-namespaces "*)
cat "$BA_JSON"
# A BucketAccess created and granted right after this listing.
[ -z "${KUBECTL_GRANT_AFTER:-}" ] || echo SECRET > "$IDS/ba-$KUBECTL_GRANT_AFTER.json"
;;
*) echo '{"kind":"List","items":[]}' ;;
esac
;;
esac
;;
*) cat >> "$KUBECTL_LOG" ;;
esac
EOF
chmod +x "$BIN/weed" "$BIN/kubectl"
export PATH="$BIN:$PATH"
export MASTER=seaweedfs-master:9333 FILER=seaweedfs-filer-client:8888
export NAMESPACE=tenant-foo RELEASE=seaweedfs-system JOB=seaweedfs-system-orphan-identities
export MODE=delete SHELL_TIMEOUT=5
}
# identity <file name>
identity() {
echo SECRET > "$IDS/$1"
}
# bucketaccess <uid> [accountID]
bucketaccess() {
jq --arg uid "$1" --arg acct "${2:-}" \
'.items += [{"metadata": {"uid": $uid}, "status": (if $acct == "" then {} else {"accountID": $acct} end)}]' \
"$BA_JSON" > "$BA_JSON.new"
mv "$BA_JSON.new" "$BA_JSON"
}
kept() {
[ -f "$IDS/$1" ] || { echo "FAIL: $1 was removed"; false; }
}
removed() {
if [ -f "$IDS/$1" ]; then echo "FAIL: $1 is still there"; false; fi
}
@test "an old identity file with no BucketAccess behind it is removed" {
identity "ba-$GONE.json"
identity "ba-$LIVE.json"
bucketaccess "$LIVE" "ba-$LIVE"
"$SCRIPT"
removed "ba-$GONE.json"
kept "ba-$LIVE.json"
}
@test "a BucketAccess still being granted protects its file by uid alone" {
identity "ba-$LIVE.json"
bucketaccess "$LIVE"
"$SCRIPT"
kept "ba-$LIVE.json"
}
@test "a BucketAccess restored under a new uid protects the file its accountID names" {
identity "ba-$GONE.json"
bucketaccess "$LIVE" "ba-$GONE"
"$SCRIPT"
kept "ba-$GONE.json"
}
@test "an empty BucketAccess listing removes nothing" {
identity "ba-$GONE.json"
echo '{"kind":"List","items":[]}' > "$BA_JSON"
"$SCRIPT"
kept "ba-$GONE.json"
grep -q '^type: Warning$' "$KUBECTL_LOG"
grep -q '^reason: OrphanIdentitiesFound$' "$KUBECTL_LOG"
}
@test "the identity directory is listed before the BucketAccesses" {
KUBECTL_GRANT_AFTER=$LIVE "$SCRIPT"
kept "ba-$LIVE.json"
}
@test "a file rewritten by a grant a moment ago is removed once its BucketAccess is gone" {
identity "ba-$GONE.json"
touch "$IDS/ba-$GONE.json"
"$SCRIPT"
removed "ba-$GONE.json"
}
@test "listing, removals and the check listing take one weed shell session each" {
identity "ba-$GONE.json"
identity "ba-$GONE2.json"
"$SCRIPT"
[ "$(grep -c '^session' "$WEED_LOG")" -eq 3 ]
}
@test "identities the COSI driver did not name are never touched" {
identity admin.json
identity ba-backup.json
identity "ba-$GONE.json.old"
identity "BA-$GONE.json"
identity "xba-$GONE.json"
identity "ba-${GONE}0.json"
identity ba-1234.json
"$SCRIPT"
kept "xba-$GONE.json"
kept "ba-${GONE}0.json"
kept ba-1234.json
kept admin.json
kept ba-backup.json
kept "ba-$GONE.json.old"
kept "BA-$GONE.json"
if grep -q '^fs.rm' "$WEED_LOG"; then echo "FAIL: removed something"; cat "$WEED_LOG"; false; fi
}
@test "report mode names the orphan and removes nothing" {
identity "ba-$GONE.json"
out=$(MODE=report "$SCRIPT" 2>&1)
kept "ba-$GONE.json"
if grep -q '^fs.rm' "$WEED_LOG"; then echo "FAIL: report mode removed a file"; false; fi
case $out in *"ba-$GONE"*) ;; *) echo "FAIL: orphan not named: $out"; false ;; esac
grep -q '^reason: OrphanIdentitiesFound$' "$KUBECTL_LOG"
grep -q "ba-$GONE" "$KUBECTL_LOG"
grep -q 'not removed (mode report)' "$KUBECTL_LOG"
grep -q 'cosi.orphanIdentities to delete' "$KUBECTL_LOG"
if grep -q 'other clusters' "$KUBECTL_LOG"; then echo "FAIL: Event claims a reason the script cannot know"; false; fi
}
@test "an unknown mode only reports" {
identity "ba-$GONE.json"
MODE=Delete "$SCRIPT"
kept "ba-$GONE.json"
grep -q '^reason: OrphanIdentitiesFound$' "$KUBECTL_LOG"
grep -q 'not removed (mode Delete)' "$KUBECTL_LOG"
}
@test "a BucketAccess listing that fails removes nothing and ends in a Warning Event" {
identity "ba-$GONE.json"
KUBECTL_BA_FAIL=1 "$SCRIPT"
kept "ba-$GONE.json"
if grep -q '^fs.rm' "$WEED_LOG"; then echo "FAIL: removed a file without a BucketAccess listing"; false; fi
grep -q '^type: Warning$' "$KUBECTL_LOG"
grep -q '^reason: OrphanIdentitiesNotChecked$' "$KUBECTL_LOG"
}
@test "a BucketAccess listing that is not a list removes nothing" {
identity "ba-$GONE.json"
echo '{"kind":"Status","status":"Failure"}' > "$BA_JSON"
"$SCRIPT"
kept "ba-$GONE.json"
grep -q '^reason: OrphanIdentitiesNotChecked$' "$KUBECTL_LOG"
}
@test "a listing item without a uid removes nothing" {
identity "ba-$GONE.json"
echo '{"kind":"List","items":[{"metadata":{}}]}' > "$BA_JSON"
"$SCRIPT"
kept "ba-$GONE.json"
grep -q '^reason: OrphanIdentitiesNotChecked$' "$KUBECTL_LOG"
}
@test "a listing of the identity directory that exits non-zero removes nothing and ends in a Warning Event" {
identity "ba-$GONE.json"
WEED_LS_FAIL="exit" "$SCRIPT"
kept "ba-$GONE.json"
if grep -q '^fs.rm' "$WEED_LOG"; then echo "FAIL: removed a file"; false; fi
grep -q '^reason: OrphanIdentitiesNotChecked$' "$KUBECTL_LOG"
}
@test "a listing of the identity directory that reports an error removes nothing" {
identity "ba-$GONE.json"
WEED_LS_FAIL="print" "$SCRIPT"
kept "ba-$GONE.json"
grep -q '^reason: OrphanIdentitiesNotChecked$' "$KUBECTL_LOG"
}
@test "every removal is logged and recorded as an Event on the HelmRelease" {
identity "ba-$GONE.json"
identity "ba-$GONE2.json"
out=$("$SCRIPT" 2>&1)
case $out in *"removed ba-$GONE"*) ;; *) echo "FAIL: removal not logged: $out"; false ;; esac
case $out in *"removed ba-$GONE2"*) ;; *) echo "FAIL: removal not logged: $out"; false ;; esac
[ "$(grep -c '^reason: OrphanIdentityRemoved$' "$KUBECTL_LOG")" -eq 2 ]
grep -q '^ kind: HelmRelease$' "$KUBECTL_LOG"
grep -q '^ name: seaweedfs-system$' "$KUBECTL_LOG"
grep -q '^ namespace: tenant-foo$' "$KUBECTL_LOG"
grep -q '^ uid: "0b5e1a7e-0000-4000-8000-000000000001"$' "$KUBECTL_LOG"
}
@test "a listing that fails after the removal ends in a Warning Event" {
identity "ba-$GONE.json"
WEED_LS_FAIL="exit" WEED_LS_FAIL_FROM=2 "$SCRIPT"
grep -q '^type: Warning$' "$KUBECTL_LOG"
grep -q '^reason: OrphanIdentityNotRemoved$' "$KUBECTL_LOG"
if grep -q '^reason: OrphanIdentityRemoved$' "$KUBECTL_LOG"; then echo "FAIL: claimed a removal it could not confirm"; false; fi
}
@test "a listing after the removal that reports an error claims no removal" {
identity "ba-$GONE.json"
WEED_LS_FAIL="print" WEED_LS_FAIL_FROM=2 "$SCRIPT"
grep -q '^reason: OrphanIdentityNotRemoved$' "$KUBECTL_LOG"
if grep -q '^reason: OrphanIdentityRemoved$' "$KUBECTL_LOG"; then echo "FAIL: claimed a removal it could not confirm"; false; fi
}
@test "a removal that fails ends in a Warning Event and exit 0" {
identity "ba-$GONE.json"
WEED_RM_FAIL=1 "$SCRIPT"
kept "ba-$GONE.json"
grep -q '^type: Warning$' "$KUBECTL_LOG"
grep -q '^reason: OrphanIdentityNotRemoved$' "$KUBECTL_LOG"
}
@test "nothing to do records no Event" {
identity "ba-$LIVE.json"
bucketaccess "$LIVE"
"$SCRIPT"
if grep -q '^ARGS: create' "$KUBECTL_LOG"; then echo "FAIL: recorded an Event"; false; fi
}
@test "a weed shell that never returns is cut off and removes nothing" {
identity "ba-$GONE.json"
start=$(date +%s)
SHELL_TIMEOUT=3 WEED_SLEEP=60 "$SCRIPT"
[ $(($(date +%s) - start)) -lt 60 ]
kept "ba-$GONE.json"
grep -q '^reason: OrphanIdentitiesNotChecked$' "$KUBECTL_LOG"
}
@test "secret content of an identity file never reaches the output" {
identity "ba-$GONE.json"
out=$("$SCRIPT" 2>&1)
case $out in *SECRET*) echo "FAIL: file content printed: $out"; false ;; esac
if grep -q SECRET "$KUBECTL_LOG"; then echo "FAIL: file content in an Event"; false; fi
}
# The Object Lock hook grants every identity with Read or Write, left-over
# ones included; removal has to come first.
@test "orphan removal runs before the Object Lock grant" {
helm template seaweedfs-system packages/system/seaweedfs --namespace tenant-foo > "$BATS_TEST_TMPDIR/manifest"
weight() {
yq eval-all "select(.kind == \"$1\" and .metadata.name == \"$2\") | .metadata.annotations[\"helm.sh/hook-weight\"]" "$BATS_TEST_TMPDIR/manifest"
}
orphan=$(weight Job seaweedfs-system-orphan-identities)
grant=$(weight Job seaweedfs-system-object-lock-actions)
[ -n "$orphan" ] && [ -n "$grant" ]
[ "$orphan" -lt "$grant" ]
for kind in ConfigMap ServiceAccount Role RoleBinding; do
[ "$(weight "$kind" seaweedfs-system-orphan-identities)" -lt "$orphan" ]
done
}
# The stub logs its call before it hangs, so SHELL_TIMEOUT has to leave a
# loaded runner time to start it.
@test "a kubectl that never returns is cut off and removes nothing" {
identity "ba-$GONE.json"
start=$SECONDS
SHELL_TIMEOUT=3 KUBECTL_SLEEP=60 "$SCRIPT"
[ $((SECONDS - start)) -lt 60 ]
kept "ba-$GONE.json"
grep -q '^ARGS: get bucketaccesses' "$KUBECTL_LOG"
grep -q '^ARGS: get helmrelease' "$KUBECTL_LOG"
grep -q '^ARGS: create' "$KUBECTL_LOG"
}