mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
Address review of the version generator (hack/update-versions.sh): - Fetch tags with `skopeo list-tags` (registry v2 /tags/list) instead of the Docker Hub v2 API, whose anonymous pagination caps at 1000 entries and then returns an error page with no `.results`, silently truncating the list so the frozen 24.9.2.42 fell outside the window and the run failed. skopeo is already in build-deps and is how the sibling generators (postgres, redis, mongodb, opensearch, kubernetes) fetch. - Make a frozen default tag missing from the registry a hard error instead of silently falling back to a newer patch, which would move the default image for existing installs. - Restore 0644 on the regenerated files (mktemp's 0600 survived the mv). - Exclude /hack from the chart tarball via .helmignore. - Refresh files/versions.yaml: v25.8 -> 25.8.32.4, the true latest 25.8 patch now visible with the complete tag list. - bats: add a missing-frozen-tag hard-error case and make the atomicity test reach the resolution phase after a valid default rather than failing at the default-in-set check. Signed-off-by: Alexey Artamonov <aleksei.artamonov@aenix.io>
174 lines
7.8 KiB
Bash
Executable File
174 lines
7.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Regenerate the ClickHouse version map (files/versions.yaml) and the version
|
|
# enum in values.yaml from the Docker Hub tags of clickhouse/clickhouse-server
|
|
# and clickhouse/clickhouse-keeper. Run via `make update`.
|
|
#
|
|
# Portable to stock macOS (bash 3.2, BSD awk/sort): no associative arrays and
|
|
# no `awk -v` with embedded newlines. For tests, CH_SERVER_TAGS_FILE /
|
|
# CH_KEEPER_TAGS_FILE inject the tag lists offline and VALUES_FILE /
|
|
# VERSIONS_FILE redirect the outputs.
|
|
|
|
set -o errexit
|
|
set -o nounset
|
|
set -o pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
CLICKHOUSE_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
VALUES_FILE="${VALUES_FILE:-${CLICKHOUSE_DIR}/values.yaml}"
|
|
VERSIONS_FILE="${VERSIONS_FILE:-${CLICKHOUSE_DIR}/files/versions.yaml}"
|
|
SERVER_REPO="clickhouse/clickhouse-server"
|
|
KEEPER_REPO="clickhouse/clickhouse-keeper"
|
|
|
|
# Major.minor lines to publish, newest first. Overridable for tests.
|
|
: "${CH_SUPPORTED_MAJORS:=25.8 25.3 24.9}"
|
|
|
|
# Numeric sort of X.Y.Z.W tags — portable across BSD and GNU sort (no -V).
|
|
version_sort() { sort -t. -k1,1n -k2,2n -k3,3n -k4,4n; }
|
|
|
|
# Emit every X.Y.Z.W tag of a Docker Hub repo, byte-collated and unique.
|
|
# Overridable offline via a file named by $2 (a variable name).
|
|
fetch_tags() {
|
|
local repo="$1" file_var="$2" file
|
|
file="${!file_var:-}"
|
|
if [ -n "$file" ]; then
|
|
grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' "$file" | LC_ALL=C sort -u
|
|
return
|
|
fi
|
|
# skopeo reads the registry v2 /tags/list, which returns the full tag set.
|
|
# (Docker Hub's own v2 API caps anonymous pagination at 1000 entries and then
|
|
# returns an error page with no `.results`, silently truncating the list.)
|
|
# skopeo is already required by the repo's build-deps and is how the sibling
|
|
# generators (postgres, redis, mongodb, opensearch, kubernetes) fetch tags.
|
|
command -v skopeo >/dev/null 2>&1 || { echo "Error: skopeo is required to fetch tags" >&2; exit 1; }
|
|
command -v jq >/dev/null 2>&1 || { echo "Error: jq is required to fetch tags" >&2; exit 1; }
|
|
skopeo list-tags "docker://docker.io/${repo}" \
|
|
| jq -r '.Tags[]' \
|
|
| grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | LC_ALL=C sort -u
|
|
}
|
|
|
|
echo "Resolving ClickHouse versions..." >&2
|
|
SERVER_TAGS="$(fetch_tags "$SERVER_REPO" CH_SERVER_TAGS_FILE)"
|
|
KEEPER_TAGS="$(fetch_tags "$KEEPER_REPO" CH_KEEPER_TAGS_FILE)"
|
|
|
|
# Tags present in BOTH images. comm walks its inputs in byte order, so they
|
|
# must be byte-collated (fetch_tags emits `LC_ALL=C sort -u`, and comm itself
|
|
# runs under LC_ALL=C); a version collation would silently drop common tags at
|
|
# every X.9 -> X.10 boundary.
|
|
COMMON_TAGS="$(LC_ALL=C comm -12 <(printf '%s\n' "$SERVER_TAGS") <(printf '%s\n' "$KEEPER_TAGS"))"
|
|
[ -n "$COMMON_TAGS" ] || { echo "Error: no tag common to ${SERVER_REPO} and ${KEEPER_REPO}" >&2; exit 1; }
|
|
|
|
# Determine the default major from the current values.yaml. Dropping the
|
|
# current default's major from CH_SUPPORTED_MAJORS would silently change the
|
|
# default image, so it is a hard error rather than a fallback to the newest.
|
|
# With no current default (first-time bootstrap) the newest major is used.
|
|
CURRENT_DEFAULT="$(awk '/^version:[[:space:]]/ {print $2; exit}' "$VALUES_FILE" 2>/dev/null || true)"
|
|
if [ -n "$CURRENT_DEFAULT" ]; then
|
|
case " $CH_SUPPORTED_MAJORS " in
|
|
*" ${CURRENT_DEFAULT#v} "*) DEFAULT_VERSION="$CURRENT_DEFAULT" ;;
|
|
*) echo "Error: current default ${CURRENT_DEFAULT} is not in CH_SUPPORTED_MAJORS (${CH_SUPPORTED_MAJORS}); refusing to silently change the default" >&2; exit 1 ;;
|
|
esac
|
|
else
|
|
DEFAULT_VERSION="v${CH_SUPPORTED_MAJORS%% *}"
|
|
fi
|
|
|
|
# The default major is frozen to the exact tag it already ships, read from the
|
|
# committed versions.yaml, so a regeneration never changes the default image.
|
|
# The freeze follows whichever major is the default — there is no hardcoded
|
|
# major. An empty result (bootstrap, or the default not yet mapped) means the
|
|
# default takes the latest common patch like any other line.
|
|
FROZEN_DEFAULT_TAG="$(grep -F "\"${DEFAULT_VERSION}\": " "$VERSIONS_FILE" 2>/dev/null | sed -E 's/.*: *"([^"]+)".*/\1/' | head -n1 || true)"
|
|
|
|
# Resolve each configured major to one tag: the default freezes to its shipped
|
|
# tag (if still available), every other line takes its latest common patch. A
|
|
# configured major with no common tag is a hard error raised BEFORE any file is
|
|
# written, so a partial Docker Hub result never drops a supported version or
|
|
# leaves a half-updated tree.
|
|
MAJORS=()
|
|
TAGS=()
|
|
for mm in $CH_SUPPORTED_MAJORS; do
|
|
key="v${mm}"
|
|
mm_re="${mm//./\\.}"
|
|
latest="$(printf '%s\n' "$COMMON_TAGS" | grep -E "^${mm_re}\.[0-9]+\.[0-9]+$" | version_sort | tail -n1 || true)"
|
|
if [ "$key" = "$DEFAULT_VERSION" ] && [ -n "$FROZEN_DEFAULT_TAG" ]; then
|
|
if printf '%s\n' "$COMMON_TAGS" | grep -qx "$FROZEN_DEFAULT_TAG"; then
|
|
tag="$FROZEN_DEFAULT_TAG"
|
|
else
|
|
# Never silently move the default image. If the frozen tag has genuinely
|
|
# left the registry, fail and let a maintainer re-pin files/versions.yaml
|
|
# deliberately.
|
|
echo "Error: frozen default tag ${FROZEN_DEFAULT_TAG} for ${key} is not available in both images; refusing to move the default. Re-pin files/versions.yaml deliberately if the default must change." >&2
|
|
exit 1
|
|
fi
|
|
elif [ -n "$latest" ]; then
|
|
tag="$latest"
|
|
else
|
|
echo "Error: no tag found for configured version ${mm} in both images" >&2
|
|
exit 1
|
|
fi
|
|
MAJORS+=("$key")
|
|
TAGS+=("$tag")
|
|
echo "Resolved ${key} -> ${tag}" >&2
|
|
done
|
|
|
|
# Build both artifacts in temp files and move them into place only once both
|
|
# are ready, so a failure mid-run leaves the committed files untouched.
|
|
TMP_VERSIONS="$(mktemp)"
|
|
TMP_BLOCK="$(mktemp)"
|
|
TMP_VALUES="$(mktemp)"
|
|
trap 'rm -f "$TMP_VERSIONS" "$TMP_BLOCK" "$TMP_VALUES"' EXIT
|
|
|
|
i=0
|
|
while [ "$i" -lt "${#MAJORS[@]}" ]; do
|
|
printf '"%s": "%s"\n' "${MAJORS[$i]}" "${TAGS[$i]}" >> "$TMP_VERSIONS"
|
|
i=$((i + 1))
|
|
done
|
|
|
|
# The values.yaml @enum/@param block, built in a file (avoids `awk -v` with
|
|
# embedded newlines, which BSD awk rejects).
|
|
{
|
|
echo "## @enum {string} Version"
|
|
i=0
|
|
while [ "$i" -lt "${#MAJORS[@]}" ]; do
|
|
echo "## @value ${MAJORS[$i]}"
|
|
i=$((i + 1))
|
|
done
|
|
echo ""
|
|
echo "## @param {Version} version - ClickHouse major.minor version to deploy. Applies to both the ClickHouse server and ClickHouse Keeper images. Downgrading to an older major is unsafe (ClickHouse cannot read data written by a newer server and Keeper snapshots are not backward compatible) — only increase this value."
|
|
echo "version: ${DEFAULT_VERSION}"
|
|
} > "$TMP_BLOCK"
|
|
|
|
# Splice the block into values.yaml with pure bash: replace an existing version
|
|
# section, or insert before the Application-specific section. The section is
|
|
# bounded — it ends at the `version:` line, or at the first line that is not a
|
|
# recognised section member — so a malformed input (enum header without a
|
|
# `version:` line) does not swallow the rest of the file.
|
|
replaced=0
|
|
in_section=0
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
case "$line" in
|
|
"## @enum {string} Version")
|
|
cat "$TMP_BLOCK" >> "$TMP_VALUES"; in_section=1; replaced=1; continue ;;
|
|
esac
|
|
if [ "$in_section" -eq 1 ]; then
|
|
case "$line" in
|
|
version:*) in_section=0; continue ;;
|
|
"## @value"*|"## @param"*|"") continue ;;
|
|
*) in_section=0 ;;
|
|
esac
|
|
fi
|
|
if [ "$replaced" -eq 0 ] && [ "$line" = "## @section Application-specific parameters" ]; then
|
|
cat "$TMP_BLOCK" >> "$TMP_VALUES"; echo "" >> "$TMP_VALUES"; replaced=1
|
|
fi
|
|
printf '%s\n' "$line" >> "$TMP_VALUES"
|
|
done < "$VALUES_FILE"
|
|
|
|
[ "$replaced" -eq 1 ] || { echo "Error: could not place the version block in $VALUES_FILE" >&2; exit 1; }
|
|
|
|
mv "$TMP_VERSIONS" "$VERSIONS_FILE"
|
|
mv "$TMP_VALUES" "$VALUES_FILE"
|
|
# mktemp creates 0600 files and mv preserves that mode; restore the normal
|
|
# world-readable mode so the committed files are not left at 0600.
|
|
chmod 0644 "$VERSIONS_FILE" "$VALUES_FILE"
|
|
echo "Updated $VERSIONS_FILE and $VALUES_FILE (default ${DEFAULT_VERSION})" >&2
|