ServerKit/backend/app/utils/remote_index.py

"""CachedRemoteIndex — the one remote-catalog engine.

The fetch → TTL cache → last-good fallback → bundled-file fallback pipeline
was re-implemented per catalog (extension registry, theme registry, …) and
has drifted in production before (bundled-index and proxy-rewrite
incidents). This class owns the hardened behavior once; services stay thin
normalizers and future catalogs get it free.

Semantics (the superset that theme_registry had already hardened to):

- Success: entries cached for ``ttl`` seconds, ``source='remote'``.
- Env override: unset ⇒ ``default_url``; set-but-EMPTY ⇒ explicitly disabled
  (bundled copy only — also how the test suite stays offline).
- Fetch failure with a last-good copy: serve it and stamp the clock so the
  TTL applies (every call retrying a dead upstream would stall each request
  behind the fetch timeout).
- Fetch failure with nothing cached: serve the bundled file, but hold it
  only ``error_ttl`` seconds before retrying upstream — the panel recovers
  quickly when the registry comes back, without hammering it.
- ``normalize_fn(payload, base_url)`` turns the raw JSON payload into the
  entry list; relative asset paths resolve against the URL actually fetched.
"""
import json
import logging
import os
import threading
import time

import requests

logger = logging.getLogger(__name__)


class CachedRemoteIndex:
    def __init__(self, name, env_var, default_url, normalize_fn,
                 bundled_path=None, bundled_base_url=None,
                 ttl=3600, ttl_env_var=None, error_ttl=60,
                 timeout=15, user_agent=None):
        self.name = name
        self.env_var = env_var
        self.default_url = default_url
        self.normalize_fn = normalize_fn
        self.bundled_path = bundled_path
        self.bundled_base_url = bundled_base_url
        self.error_ttl = error_ttl
        self.timeout = timeout
        self.user_agent = user_agent or f'ServerKit-{name}/1.0'
        self.ttl = ttl
        if ttl_env_var:
            try:
                self.ttl = int(os.environ.get(ttl_env_var, str(ttl)))
            except ValueError:
                self.ttl = ttl
        # Kept as a plain dict (not attributes) so a service module can alias
        # it (`_cache = _index._cache`) for tests that inspect/twiddle it.
        self._cache = {'ts': 0.0, 'entries': None, 'source': None}
        self._lock = threading.Lock()

    # ------------------------------------------------------------------ url
    def url(self):
        """Live index URL, resolved per call so env changes apply without a
        restart. Unset ⇒ default; set-but-empty ⇒ disabled (empty string)."""
        value = os.environ.get(self.env_var)
        if value is None:
            return self.default_url
        return value.strip()

    # ------------------------------------------------------------------ get
    def get(self, force=False):
        """Return the entries, refreshing when stale. Never raises."""
        now = time.time()
        with self._lock:
            if (not force and self._cache['entries'] is not None
                    and (now - self._cache['ts']) < self.ttl):
                return self._cache['entries']

        entries = None
        source = None
        failed = False
        try:
            entries = self._fetch_remote()
            if entries is not None:
                source = 'remote'
        except Exception as exc:
            failed = True
            logger.warning('%s index fetch failed (%s): %s',
                           self.name, self.url(), exc)

        with self._lock:
            if entries is None:
                if self._cache['entries'] is not None:
                    # Serve last-good; stamp the clock so the TTL applies.
                    self._cache['ts'] = now
                    return self._cache['entries']
                entries = self._load_bundled()
                source = 'bundled'
            ts = now - (self.ttl - self.error_ttl) if failed else now
            self._cache.update({'entries': entries, 'ts': ts, 'source': source})
            return entries

    def invalidate(self):
        with self._lock:
            self._cache.update({'ts': 0.0, 'entries': None, 'source': None})

    def source_label(self):
        return self._cache.get('source')

    # -------------------------------------------------------------- internals
    def _fetch_remote(self):
        url = self.url()
        if not url:
            return None
        resp = requests.get(url, timeout=self.timeout, headers={
            'Accept': 'application/json',
            'User-Agent': self.user_agent,
        })
        resp.raise_for_status()
        return self.normalize_fn(resp.json(), url)

    def _load_bundled(self):
        if not self.bundled_path:
            return []
        try:
            with open(self.bundled_path, 'r', encoding='utf-8') as f:
                return self.normalize_fn(json.load(f), self.bundled_base_url)
        except Exception as exc:
            logger.warning('Could not read bundled %s index: %s', self.name, exc)
            return []