mirror of
https://github.com/FOSSBilling/FOSSBilling.git
synced 2026-10-23 03:37:01 +09:00
Add open-source gettext extractor and translation source check (#4353)
* Add open-source gettext extractor and translation source check Replaces the manual Poedit Pro step for regenerating messages.pot. The extractor covers Twig |trans literals and PHP __trans / __pluralTrans / exception keywords, mirroring the historical Poedit runs (validated against the May 2026 messages.pot: only obsolete fossils differ). Also adds an informational per-PR check reporting msgid changes, as gettext matching is case-sensitive and casing-only changes silently invalidate translations. * Remove AI slop from extractor and workflow Drop the unused function, unused parameters, unreachable branches, restating comments and oversized docstring. No behavior change: output verified byte-identical. * Fix actions/github-script pin to a resolvable SHA * Refresh i18n POT header and check label * Rename translation workflow title * Run translation source check on ubuntu-slim * Ignore Python bytecode cache in scripts dir * Address review: plural edge case and workflow injection Skip __pluralTrans calls without a complete static plural instead of emitting a misleading singular entry. Pass the pot diff to the comment script via env to close the template-injection vector. * Address review: one-to-one casing match and comment pagination Consume each removed msgid at most once when pairing casing-only changes, and paginate issue comments when locating the sticky check comment. * Address review: string-content calls, concat fragments, plural diff, credentials Ignore keyword-like text inside PHP string/heredoc content, skip concatenated first arguments instead of recording untranslatable fragments (13 such fragments dropped from current output), report plural changes across casing pairs, and set persist-credentials: false with a non-failing comment step for fork PRs. * Skip PHP string-content matches and fix heredoc terminators Track string/heredoc spans during comment masking and ignore keyword-like text inside them; also accept comma-terminated flexible heredocs. Removes phantom refs and entries, keeps real calls with correct line numbers (verified: timeago entries intact, May-validation residuals all still excluded-path fossils or untranslatable fragments).
This commit is contained in:
Adam Daley authored and GitHub committed
1 parent
0d3b0dd40b
commit
c6992931bc
3 files changed
+603
No files matched your search
Executable
+519
@@ -0,0 +1,519 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract gettext msgids from FOSSBilling sources into a messages.pot file.
|
||||
|
||||
Open-source replacement for the manual Poedit Pro step. Scans Twig
|
||||
``|trans`` literals and PHP ``__trans`` / ``__pluralTrans`` / exception
|
||||
keywords under ``src/`` (minus vendor, install, data, load.php, tests and
|
||||
web-asset extensions); obsolete msgids are dropped.
|
||||
|
||||
Usage:
|
||||
extract_pot.py <src_dir> <out.pot>
|
||||
extract_pot.py --diff <old.pot> <new.pot> # msgid-set comparison,
|
||||
# flagging casing-only changes (gettext matches case-sensitively,
|
||||
# so those silently invalidate existing translations)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import bisect
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import NamedTuple
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration (mirrors the historical Poedit runs + project decisions)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
EXCLUDED_DIR_NAMES = {"vendor", "install", "data", "tests"}
|
||||
EXCLUDED_FILES = {"load.php"}
|
||||
EXCLUDED_EXTENSIONS = {".js", ".html", ".css", ".scss", ".md"}
|
||||
SCANNED_EXTENSIONS = {".php", ".twig"}
|
||||
|
||||
PHP_KEYWORDS_SINGLE = {
|
||||
"__trans",
|
||||
"Exception",
|
||||
"InformationException",
|
||||
"Server_Exception",
|
||||
"Registrar_Exception",
|
||||
"Payment_Exception",
|
||||
}
|
||||
PHP_KEYWORDS_PLURAL = {"__pluralTrans"}
|
||||
|
||||
HEADER_TEMPLATE = """\
|
||||
# en_US translation of FOSSBilling Application
|
||||
# Copyright 2022-2026 FOSSBilling
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: 4\\n"
|
||||
"POT-Creation-Date: {date}\\n"
|
||||
"PO-Revision-Date: YYYY-mm-DD HH:MM+ZZZZ\\n"
|
||||
"Last-Translator: FOSSBilling TM <info@fossbilling.org>\\n"
|
||||
"Language-Team: FOSSBilling TM <info@fossbilling.org>\\n"
|
||||
"MIME-Version: 1.0\\n"
|
||||
"Content-Type: text/plain; charset=UTF-8\\n"
|
||||
"Content-Transfer-Encoding: 8bit\\n"
|
||||
"Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\\n"
|
||||
"X-Generator: FOSSBilling extract_pot.py\\n"
|
||||
"""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# String decoding / encoding helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PHP_DOUBLE_ESCAPES = {
|
||||
"n": "\n",
|
||||
"r": "\r",
|
||||
"t": "\t",
|
||||
"v": "\v",
|
||||
"e": "\x1b",
|
||||
"f": "\f",
|
||||
"\\": "\\",
|
||||
"$": "$",
|
||||
'"': '"',
|
||||
}
|
||||
|
||||
|
||||
def _decode_php_double(body: str) -> str:
|
||||
"""Decode a PHP double-quoted string body (no surrounding quotes)."""
|
||||
|
||||
def repl(match: re.Match[str]) -> str:
|
||||
seq = match.group(1)
|
||||
if seq in _PHP_DOUBLE_ESCAPES:
|
||||
return _PHP_DOUBLE_ESCAPES[seq]
|
||||
if seq.startswith("x"):
|
||||
return chr(int(seq[1:] or "0", 16))
|
||||
if seq[0].isdigit():
|
||||
return chr(int(seq, 8) & 0xFF)
|
||||
return match.group(0)
|
||||
|
||||
return re.sub(r"\\(x[0-9A-Fa-f]{1,2}|[0-7]{1,3}|.)", repl, body, flags=re.DOTALL)
|
||||
|
||||
|
||||
def _decode_single(body: str) -> str:
|
||||
"""Decode a PHP/Twig single-quoted string body (only \\\\ and \\' are special)."""
|
||||
return body.replace("\\\\", "\x00").replace("\\'", "'").replace("\x00", "\\")
|
||||
|
||||
|
||||
def po_escape(text: str) -> str:
|
||||
out = text.replace("\\", "\\\\").replace('"', '\\"')
|
||||
out = out.replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t")
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Comment masking (replaced with spaces, newlines preserved -> offsets stable)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PHP_TOKEN = re.compile(
|
||||
r"(?P<heredoc><<<\s*['\"]?(?P<htag>[A-Za-z_][A-Za-z0-9_]*)['\"]?[^\n]*\n)"
|
||||
r"|(?P<dstr>\"(?:\\.|[^\"\\])*\")"
|
||||
r"|(?P<sstr>'(?:\\.|[^'\\])*')"
|
||||
r"|(?P<linec>//[^\n]*|#[^\n]*)"
|
||||
r"|(?P<blockc>/\*.*?\*/)",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def _mask_php_comments(text: str) -> tuple[str, list[tuple[int, int]]]:
|
||||
"""Blank out PHP comments, honouring strings and heredocs (offsets stable).
|
||||
|
||||
Also returns the spans of ordinary string literals and heredoc bodies so
|
||||
callers can ignore keyword-like text inside string content (which real
|
||||
xgettext never extracts).
|
||||
"""
|
||||
out: list[str] = []
|
||||
spans: list[tuple[int, int]] = []
|
||||
pos = 0
|
||||
for match in _PHP_TOKEN.finditer(text):
|
||||
kind = match.lastgroup
|
||||
if match.start() < pos:
|
||||
# Inside an already-consumed heredoc body; it was copied verbatim.
|
||||
continue
|
||||
if kind == "heredoc":
|
||||
tag = match.group("htag")
|
||||
# Flexible heredoc (PHP 7.3+): indented marker optionally
|
||||
# followed by `;` or `,` when used inside an expression.
|
||||
end = re.search(
|
||||
rf"^[ \t]*{re.escape(tag)}[ \t]*[;,]?[ \t]*$",
|
||||
text[match.end():],
|
||||
re.MULTILINE,
|
||||
)
|
||||
stop = match.end() + end.end() if end else len(text)
|
||||
spans.append((match.start(), stop))
|
||||
out.append(text[pos:stop])
|
||||
pos = stop
|
||||
continue
|
||||
out.append(text[pos : match.start()])
|
||||
token = match.group(0)
|
||||
if kind in ("linec", "blockc"):
|
||||
out.append(re.sub(r"[^\n]", " ", token))
|
||||
else:
|
||||
spans.append((match.start(), match.end()))
|
||||
out.append(token)
|
||||
pos = match.end()
|
||||
out.append(text[pos:])
|
||||
return "".join(out), spans
|
||||
|
||||
|
||||
def _in_spans(spans: list[tuple[int, int]], starts: list[int], offset: int) -> bool:
|
||||
"""Whether offset falls inside a string/heredoc span (spans are sorted)."""
|
||||
idx = bisect.bisect_right(starts, offset) - 1
|
||||
return idx >= 0 and spans[idx][0] <= offset < spans[idx][1]
|
||||
|
||||
|
||||
_TWIG_COMMENT = re.compile(r"\{#.*?#\}", re.DOTALL)
|
||||
|
||||
|
||||
def _mask_twig_comments(text: str) -> str:
|
||||
return _TWIG_COMMENT.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), text)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# A quoted literal immediately feeding the |trans filter, e.g.
|
||||
# {{ 'Order status'|trans }} {{ 'Hello'|trans({...}) }} {{ 'Guest' |trans }}
|
||||
_TWIG_TRANS = re.compile(
|
||||
r"""'(?P<sbody>(?:\\.|[^'\\])*)'\s*\|\s*trans(?![A-Za-z0-9_])"""
|
||||
r"""|"(?P<dbody>(?:\\.|[^"\\])*)"\s*\|\s*trans(?![A-Za-z0-9_])"""
|
||||
)
|
||||
|
||||
_PHP_STR = r"""'(?P<sbody>(?:\\.|[^'\\])*)'|"(?P<dbody>(?:\\.|[^"\\])*)\""""
|
||||
|
||||
_KEYWORD_ALTS = "|".join(
|
||||
sorted(PHP_KEYWORDS_SINGLE | PHP_KEYWORDS_PLURAL, key=len, reverse=True)
|
||||
)
|
||||
# Matches bare calls AND `new [Namespace\]Keyword(`. The lookbehind rejects
|
||||
# method/static calls (->, ::), variables and longer identifiers
|
||||
# (e.g. the `Exception` in `InformationException`).
|
||||
_PHP_CALL = re.compile(
|
||||
rf"(?:new\s+)?(?<![\w$>:\\])(?:\\)?(?:[A-Za-z_][\w]*\\)*(?P<kw>{_KEYWORD_ALTS})\s*\(\s*"
|
||||
rf"(?P<arg>{_PHP_STR})?"
|
||||
rf"(?P<rest>\s*,)?",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# xgettext treats a double-quoted PHP literal as non-constant (and skips the
|
||||
# call) when it contains an unescaped `$` starting an interpolation
|
||||
# (`$var`, `${var}`, `{$var}` …). A `$` followed by anything else
|
||||
# (e.g. `$5`) is literal and kept. Single-quoted strings never interpolate.
|
||||
_PHP_DYNAMIC_DQOUTE = re.compile(r"(?<!\\)\$(?=[A-Za-z_{\x80-\xff])")
|
||||
|
||||
class Occurrence(NamedTuple):
|
||||
msgid: str
|
||||
plural: str | None
|
||||
ref: str
|
||||
|
||||
|
||||
def _line_no(text: str, offset: int) -> int:
|
||||
return text.count("\n", 0, offset) + 1
|
||||
|
||||
|
||||
def extract_twig(rel: str, text: str, occurrences: list[Occurrence], stats: dict) -> None:
|
||||
masked = _mask_twig_comments(text)
|
||||
for match in _TWIG_TRANS.finditer(masked):
|
||||
if match.group("sbody") is not None:
|
||||
msgid = _decode_single(match.group("sbody"))
|
||||
else:
|
||||
body = match.group("dbody")
|
||||
if "#{" in body:
|
||||
stats["skipped_dynamic"] += 1
|
||||
stats["skipped"].append(f"{rel}:{_line_no(masked, match.start())} interpolated Twig string")
|
||||
continue
|
||||
msgid = _decode_php_double(body)
|
||||
occurrences.append(Occurrence(msgid, None, f"{rel}:{_line_no(masked, match.start())}"))
|
||||
stats["twig"] += 1
|
||||
|
||||
|
||||
def _read_php_literal(masked: str, start: int) -> tuple[str, int, bool] | None:
|
||||
"""Read one quoted literal at offset.
|
||||
|
||||
Returns (value, end_offset, is_dynamic_double) or None. Double-quoted
|
||||
literals containing an interpolation are flagged dynamic (xgettext
|
||||
treats them as non-constant).
|
||||
"""
|
||||
if start >= len(masked) or masked[start] not in ("'", '"'):
|
||||
return None
|
||||
quote = masked[start]
|
||||
pattern = (
|
||||
r"'(?:\\.|[^'\\])*'" if quote == "'" else r'"(?:\\.|[^"\\])*"'
|
||||
)
|
||||
match = re.compile(pattern, re.DOTALL).match(masked, start)
|
||||
if not match:
|
||||
return None
|
||||
body = match.group(0)[1:-1]
|
||||
if quote == "'":
|
||||
return _decode_single(body), match.end(), False
|
||||
return _decode_php_double(body), match.end(), bool(_PHP_DYNAMIC_DQOUTE.search(body))
|
||||
|
||||
|
||||
def extract_php(rel: str, text: str, occurrences: list[Occurrence], stats: dict) -> None:
|
||||
masked, spans = _mask_php_comments(text)
|
||||
starts = [start for start, _ in spans]
|
||||
for match in _PHP_CALL.finditer(masked):
|
||||
if _in_spans(spans, starts, match.start()):
|
||||
continue
|
||||
keyword = match.group("kw")
|
||||
if re.search(r"\bfunction\s*$", masked[max(0, match.start() - 200) : match.start()]):
|
||||
stats["skipped_defs"] += 1
|
||||
continue
|
||||
if match.group("arg") is None:
|
||||
# Non-literal first argument (variable, constant, expression...).
|
||||
stats["skipped_dynamic"] += 1
|
||||
snippet = masked[match.start() : match.start() + 60].replace("\n", " ")
|
||||
stats["skipped"].append(f"{rel}:{_line_no(masked, match.start())} {snippet}")
|
||||
continue
|
||||
if match.group("sbody") is not None:
|
||||
first = _decode_single(match.group("sbody"))
|
||||
first_dynamic = False
|
||||
else:
|
||||
body = match.group("dbody")
|
||||
first_dynamic = bool(_PHP_DYNAMIC_DQOUTE.search(body))
|
||||
first = _decode_php_double(body)
|
||||
if first_dynamic:
|
||||
stats["skipped_dynamic"] += 1
|
||||
snippet = masked[match.start() : match.start() + 60].replace("\n", " ")
|
||||
stats["skipped"].append(f"{rel}:{_line_no(masked, match.start())} interpolated {snippet}")
|
||||
continue
|
||||
# Rescan the first literal to find where it ends (for the plural arg).
|
||||
arg_start = match.start("arg")
|
||||
parsed = _read_php_literal(masked, arg_start)
|
||||
end = parsed[1] if parsed else match.end()
|
||||
# A concatenated first argument ('a ' . $b) never matches at runtime
|
||||
# as a msgid, so skip it instead of recording the literal prefix.
|
||||
if re.match(r"\s*[),]", masked[end:]) is None:
|
||||
stats["skipped_dynamic"] += 1
|
||||
snippet = masked[match.start() : match.start() + 60].replace("\n", " ")
|
||||
stats["skipped"].append(f"{rel}:{_line_no(masked, match.start())} concat {snippet}")
|
||||
continue
|
||||
plural: str | None = None
|
||||
if keyword in PHP_KEYWORDS_PLURAL:
|
||||
rest = re.match(r"\s*,\s*", masked[end:])
|
||||
second = (
|
||||
_read_php_literal(masked, end + rest.end())
|
||||
if rest
|
||||
else None
|
||||
)
|
||||
# A plural entry is only valid with a complete static second
|
||||
# literal; anything else (missing, dynamic, concatenated)
|
||||
# would mistranslate, so skip the whole call.
|
||||
if (
|
||||
second is None
|
||||
or second[2]
|
||||
or not re.match(r"\s*(?:,|\))", masked[second[1]:])
|
||||
):
|
||||
stats["skipped_dynamic"] += 1
|
||||
snippet = masked[match.start() : match.start() + 60].replace("\n", " ")
|
||||
stats["skipped"].append(f"{rel}:{_line_no(masked, match.start())} {snippet}")
|
||||
continue
|
||||
plural = second[0]
|
||||
occurrences.append(Occurrence(first, plural, f"{rel}:{_line_no(masked, match.start())}"))
|
||||
stats["php"] += 1
|
||||
|
||||
|
||||
def iter_source_files(src_dir: str):
|
||||
for root, dirs, files in os.walk(src_dir):
|
||||
dirs[:] = sorted(d for d in dirs if d not in EXCLUDED_DIR_NAMES)
|
||||
for name in sorted(files):
|
||||
if name in EXCLUDED_FILES:
|
||||
continue
|
||||
if os.path.splitext(name)[1] not in SCANNED_EXTENSIONS:
|
||||
continue
|
||||
full = os.path.join(root, name)
|
||||
yield full, os.path.relpath(full, src_dir).replace(os.sep, "/")
|
||||
|
||||
|
||||
def extract_all(src_dir: str) -> tuple[list[Occurrence], dict]:
|
||||
occurrences: list[Occurrence] = []
|
||||
stats: dict = {"twig": 0, "php": 0, "skipped_dynamic": 0, "skipped_defs": 0, "skipped": [], "files": 0}
|
||||
for full, rel in iter_source_files(src_dir):
|
||||
with open(full, encoding="utf-8", errors="replace") as handle:
|
||||
text = handle.read()
|
||||
stats["files"] += 1
|
||||
if full.endswith(".twig"):
|
||||
extract_twig(rel, text, occurrences, stats)
|
||||
else:
|
||||
extract_php(rel, text, occurrences, stats)
|
||||
return occurrences, stats
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# .pot output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _ref_sort_key(ref: str) -> tuple[str, int]:
|
||||
path, _, line = ref.rpartition(":")
|
||||
try:
|
||||
return (path, int(line))
|
||||
except ValueError:
|
||||
return (ref, 0)
|
||||
|
||||
|
||||
def write_pot(occurrences: list[Occurrence], out_path: str) -> int:
|
||||
merged: dict[str, dict] = {}
|
||||
for occ in occurrences:
|
||||
entry = merged.setdefault(occ.msgid, {"refs": [], "plural": None, "plurals_seen": set()})
|
||||
entry["refs"].append(occ.ref)
|
||||
if occ.plural is not None:
|
||||
entry["plurals_seen"].add(occ.plural)
|
||||
if entry["plural"] is None:
|
||||
entry["plural"] = occ.plural
|
||||
elif entry["plural"] != occ.plural:
|
||||
print(
|
||||
f"warning: conflicting plurals for {occ.msgid!r}: "
|
||||
f"{entry['plural']!r} vs {occ.plural!r} (keeping first)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
date = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M+0000")
|
||||
with open(out_path, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(HEADER_TEMPLATE.format(date=date))
|
||||
for msgid in sorted(merged):
|
||||
entry = merged[msgid]
|
||||
refs = sorted(set(entry["refs"]), key=_ref_sort_key)
|
||||
handle.write(f"\n#: {' '.join(refs)}\n")
|
||||
handle.write(f'msgid "{po_escape(msgid)}"\n')
|
||||
if entry["plural"] is not None:
|
||||
handle.write(f'msgid_plural "{po_escape(entry["plural"])}"\n')
|
||||
handle.write('msgstr[0] ""\nmsgstr[1] ""\n')
|
||||
else:
|
||||
handle.write('msgstr ""\n')
|
||||
return len(merged)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# .pot parsing (for --diff) and comparison
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _po_unescape(quoted: str) -> str:
|
||||
body = quoted[1:-1]
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
simple = {"n": "\n", "r": "\r", "t": "\t", '"': '"', "\\": "\\"}
|
||||
while i < len(body):
|
||||
char = body[i]
|
||||
if char != "\\" or i + 1 >= len(body):
|
||||
out.append(char)
|
||||
i += 1
|
||||
continue
|
||||
nxt = body[i + 1]
|
||||
out.append(simple.get(nxt, nxt))
|
||||
i += 2
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def parse_pot_entries(path: str) -> dict[str, str | None]:
|
||||
"""Parse a .pot into {msgid: msgid_plural}; handles wrapped lines."""
|
||||
with open(path, encoding="utf-8", errors="replace") as handle:
|
||||
text = handle.read()
|
||||
entries: dict[str, str | None] = {}
|
||||
for block in re.split(r"\n\s*\n", text):
|
||||
msgid_parts: list[str] = []
|
||||
plural_parts: list[str] = []
|
||||
target: list[str] | None = None
|
||||
for line in block.splitlines():
|
||||
if line.startswith("msgid_plural"):
|
||||
target = plural_parts
|
||||
plural_parts.append(_po_unescape(line[len("msgid_plural") :].strip()))
|
||||
elif line.startswith("msgid"):
|
||||
target = msgid_parts
|
||||
msgid_parts.append(_po_unescape(line[len("msgid") :].strip()))
|
||||
elif line.startswith('"') and target is not None:
|
||||
target.append(_po_unescape(line.strip()))
|
||||
else:
|
||||
target = None
|
||||
msgid = "".join(msgid_parts)
|
||||
if msgid:
|
||||
plural = "".join(plural_parts) or None
|
||||
entries[msgid] = plural
|
||||
return entries
|
||||
|
||||
|
||||
def cmd_diff(old_path: str, new_path: str) -> int:
|
||||
old_entries = parse_pot_entries(old_path)
|
||||
new_entries = parse_pot_entries(new_path)
|
||||
old_ids = set(old_entries)
|
||||
new_ids = set(new_entries)
|
||||
added = sorted(new_ids - old_ids)
|
||||
removed = sorted(old_ids - new_ids)
|
||||
|
||||
removed_lookup: dict[str, list[str]] = {}
|
||||
for msgid in removed:
|
||||
removed_lookup.setdefault(msgid.casefold(), []).append(msgid)
|
||||
casing_changes: list[tuple[str, str]] = []
|
||||
real_added: list[str] = []
|
||||
for msgid in added:
|
||||
candidates = removed_lookup.get(msgid.casefold(), [])
|
||||
if candidates:
|
||||
casing_changes.append((candidates.pop(0), msgid))
|
||||
else:
|
||||
real_added.append(msgid)
|
||||
changed_set = {old for old, _ in casing_changes}
|
||||
real_removed = [msgid for msgid in removed if msgid not in changed_set]
|
||||
|
||||
print(f"Added: {len(added)} (new: {len(real_added)}, casing-only: {len(casing_changes)})")
|
||||
print(f"Removed: {len(removed)} (gone: {len(real_removed)}, casing-only: {len(casing_changes)})")
|
||||
if casing_changes:
|
||||
print("\nCasing-only changes (invalidate existing translations!):")
|
||||
for old, new in sorted(casing_changes):
|
||||
print(f" - {old!r} -> {new!r}")
|
||||
if real_added:
|
||||
print("\nNew msgids needing translation:")
|
||||
for msgid in real_added:
|
||||
print(f" + {msgid!r}")
|
||||
if real_removed:
|
||||
print("\nDropped msgids:")
|
||||
for msgid in real_removed:
|
||||
print(f" - {msgid!r}")
|
||||
plural_changed = sorted(
|
||||
msgid
|
||||
for msgid in old_ids & new_ids
|
||||
if old_entries[msgid] != new_entries[msgid]
|
||||
)
|
||||
plural_changed_pairs = sorted(
|
||||
(old, new)
|
||||
for old, new in casing_changes
|
||||
if old_entries[old] != new_entries[new]
|
||||
)
|
||||
if plural_changed or plural_changed_pairs:
|
||||
print("\nChanged plural forms:")
|
||||
for msgid in plural_changed:
|
||||
print(f" ~ {msgid!r}: {old_entries[msgid]!r} -> {new_entries[msgid]!r}")
|
||||
for old, new in plural_changed_pairs:
|
||||
print(f" ~ {old!r} -> {new!r}: {old_entries[old]!r} -> {new_entries[new]!r}")
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Extract gettext msgids into messages.pot")
|
||||
parser.add_argument("src_dir", nargs="?", help="FOSSBilling src/ directory")
|
||||
parser.add_argument("out_pot", nargs="?", help="Output .pot path")
|
||||
parser.add_argument("--diff", nargs=2, metavar=("OLD_POT", "NEW_POT"), help="Compare two .pot files")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.diff:
|
||||
return cmd_diff(args.diff[0], args.diff[1])
|
||||
if not args.src_dir or not args.out_pot:
|
||||
parser.error("src_dir and out_pot are required (or use --diff)")
|
||||
occurrences, stats = extract_all(args.src_dir)
|
||||
count = write_pot(occurrences, args.out_pot)
|
||||
print(f"files scanned: {stats['files']}", file=sys.stderr)
|
||||
print(f"twig hits: {stats['twig']}, php hits: {stats['php']}", file=sys.stderr)
|
||||
print(f"unique msgids written: {count}", file=sys.stderr)
|
||||
print(f"skipped non-literals: {stats['skipped_dynamic']}, skipped defs: {stats['skipped_defs']}", file=sys.stderr)
|
||||
for item in stats["skipped"]:
|
||||
print(f" skip: {item}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,81 @@
|
||||
name: Translation Source Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'src/**/*.php'
|
||||
- 'src/**/*.twig'
|
||||
- '.github/scripts/extract_pot.py'
|
||||
- '.github/workflows/i18n-pot-check.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
pot-diff:
|
||||
runs-on: ubuntu-slim
|
||||
steps:
|
||||
- name: 'Checkout PR head'
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: 'Extract msgids (head)'
|
||||
run: python3 .github/scripts/extract_pot.py src /tmp/head.pot
|
||||
|
||||
- name: 'Extract msgids (base)'
|
||||
run: |
|
||||
git worktree add /tmp/base "${{ github.event.pull_request.base.sha }}"
|
||||
python3 .github/scripts/extract_pot.py /tmp/base/src /tmp/base.pot
|
||||
git worktree remove --force /tmp/base
|
||||
|
||||
- name: 'Compare'
|
||||
id: diff
|
||||
run: |
|
||||
python3 .github/scripts/extract_pot.py --diff /tmp/base.pot /tmp/head.pot > /tmp/pot-diff.txt 2>&1 || true
|
||||
{
|
||||
echo 'body<<EOF'
|
||||
cat /tmp/pot-diff.txt
|
||||
echo 'EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: 'Comment on PR'
|
||||
# Fork PRs get a read-only token, so commenting may fail there;
|
||||
# the extraction result above is still visible in the job log.
|
||||
continue-on-error: true
|
||||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
|
||||
env:
|
||||
POT_DIFF: ${{ steps.diff.outputs.body }}
|
||||
with:
|
||||
script: |
|
||||
const marker = '<!-- fossbilling-i18n-pot-diff -->';
|
||||
const diff = (process.env.POT_DIFF || '').trim() || '(no output)';
|
||||
const body = `${marker}\n### Translation Source Check\n\n` +
|
||||
`How this PR changes the gettext msgid set ` +
|
||||
`(casing-only changes silently invalidate existing translations):\n\n` +
|
||||
'```\n' + diff + '\n```\n\n' +
|
||||
`_Informational only. The .pot is regenerated by the scheduled job in FOSSBilling/locale._`;
|
||||
const comments = await github.paginate(github.rest.issues.listComments, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
per_page: 100,
|
||||
});
|
||||
const existing = comments.find(c => c.body && c.body.includes(marker));
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
@@ -35,3 +35,6 @@ tests/E2E/Playwright/artifacts
|
||||
### Demo seed (generated per-run artifacts) ###
|
||||
tools/demo-seed/manifest*.json
|
||||
.ddev/*.sql.gz
|
||||
|
||||
### Python (extractor script bytecode cache) ###
|
||||
.github/scripts/__pycache__/
|
||||
Reference in new issue
Block a user