feat(wordpress): shared-state contract and login rate limiting at the edge

Pin the Compose template's full-tree wordpress_html named volume as the
supported same-host shared-state contract (uploads, plugins, themes, disk
page cache), and make WordPress vhost publishing install shared
wp-login.php/xmlrpc.php limit_req zones keyed by server name + client IP.
Generic Docker vhosts are unaffected; drift repair regenerates existing
WordPress vhosts with the policy. Spec and roadmap rows move from Partial
to Shipped with their real-host boundaries recorded.
This commit is contained in:
Juan Denis committed 2026-08-27 22:35:54 -04:00
1 parent bb675f0d9a
commit bba169f04a
7 files changed
+425 -100

No files matched your search

+107 -2
View File
@@ -170,6 +170,47 @@ class NginxService:
proxy_send_timeout 60;
}}
}}
'''
# WordPress authentication endpoints receive a dedicated request limit in
# published reverse-proxy vhosts. The zones are shared across vhosts, while
# the key includes server_name so traffic to one site cannot consume
# another site's allowance.
WORDPRESS_RATE_LIMIT_CONF_NAME = 'serverkit-wordpress-rate-limit.conf'
WORDPRESS_RATE_LIMIT_ZONE_SNIPPET = '''# ServerKit WordPress request-limit zones (auto-generated; do not edit).
limit_req_zone $server_name$binary_remote_addr zone=serverkit_wp_login:10m rate=10r/m;
limit_req_zone $server_name$binary_remote_addr zone=serverkit_wp_xmlrpc:10m rate=30r/m;
'''
WORDPRESS_PROTECTION_BLOCK = ''' # ServerKit WordPress brute-force protection
location = /wp-login.php {{
limit_req zone=serverkit_wp_login burst=10 nodelay;
limit_req_status 429;
proxy_pass http://127.0.0.1:{port};
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 86400;
proxy_connect_timeout 60;
proxy_send_timeout 60;
}}
location = /xmlrpc.php {{
limit_req zone=serverkit_wp_xmlrpc burst=20 nodelay;
limit_req_status 429;
proxy_pass http://127.0.0.1:{port};
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_read_timeout 86400;
proxy_connect_timeout 60;
proxy_send_timeout 60;
}}
'''
# Reverse proxy to a service reached over a WireGuard tunnel (roadmap
@@ -523,7 +564,8 @@ location /p/ {{
php_version: str = '8.2',
ssl_cert: str = None, ssl_key: str = None,
upstream: str = None,
micro_cache: bool = False) -> Dict:
micro_cache: bool = False,
wordpress_protection: bool = False) -> Dict:
"""Render the vhost config for a site to a string — no side effects.
This is the exact template pipeline :meth:`create_site` writes to disk;
@@ -595,11 +637,61 @@ location /p/ {{
# also carries a server_name line) never receives cache directives.
config = cls._with_micro_cache(config, app_type)
if wordpress_protection:
if app_type != 'docker' or not port:
return {
'success': False,
'error': 'WordPress protection requires a Docker proxy port',
}
config = cls._with_wordpress_protection(config, port)
if ssl_cert and ssl_key:
config = cls._with_ssl(config, domains_str, ssl_cert, ssl_key)
return {'success': True, 'config': config}
@classmethod
def _with_wordpress_protection(cls, config: str, port: int) -> str:
"""Add exact-match throttling locations before the catch-all proxy."""
anchor = ' location / {\n'
if anchor not in config:
return config
block = cls.WORDPRESS_PROTECTION_BLOCK.format(port=port)
return config.replace(anchor, block + anchor, 1)
@classmethod
def ensure_wordpress_rate_limit_zones(cls) -> Dict:
"""Write the shared WordPress rate-limit zones to nginx conf.d."""
conf_path = os.path.join(
cls.NGINX_CONF_DIR,
'conf.d',
cls.WORDPRESS_RATE_LIMIT_CONF_NAME,
)
try:
if os.path.isfile(conf_path):
with open(conf_path, 'r') as f:
if f.read() == cls.WORDPRESS_RATE_LIMIT_ZONE_SNIPPET:
return {
'success': True,
'changed': False,
'path': conf_path,
'message': 'WordPress rate-limit zones already configured',
}
written = write_privileged_file(
conf_path,
cls.WORDPRESS_RATE_LIMIT_ZONE_SNIPPET,
)
if not written['success']:
return {'success': False, 'error': written['error']}
return {
'success': True,
'changed': True,
'path': conf_path,
'message': 'WordPress rate-limit zones configured',
}
except Exception as e:
return {'success': False, 'error': str(e)}
@classmethod
def _with_micro_cache(cls, config: str, app_type: str) -> str:
"""Inject the shared-zone micro-cache directives into a rendered vhost.
@@ -694,7 +786,8 @@ location /p/ {{
def create_site(cls, name: str, app_type: str, domains: List[str],
root_path: str = None, port: int = None, php_version: str = '8.2',
ssl_cert: str = None, ssl_key: str = None,
upstream: str = None, micro_cache: bool = False) -> Dict:
upstream: str = None, micro_cache: bool = False,
wordpress_protection: bool = False) -> Dict:
"""Create a new site configuration.
When ``ssl_cert``/``ssl_key`` are given the generated vhost serves HTTPS
@@ -707,6 +800,7 @@ location /p/ {{
name, app_type, domains, root_path=root_path, port=port,
php_version=php_version, ssl_cert=ssl_cert, ssl_key=ssl_key,
upstream=upstream, micro_cache=micro_cache,
wordpress_protection=wordpress_protection,
)
if not rendered.get('success'):
return rendered
@@ -720,6 +814,17 @@ location /p/ {{
return {'success': False,
'error': f"micro-cache zone setup failed: {zone.get('error')}"}
if wordpress_protection:
zones = cls.ensure_wordpress_rate_limit_zones()
if not zones.get('success'):
return {
'success': False,
'error': (
'WordPress rate-limit zone setup failed: '
f"{zones.get('error')}"
),
}
try:
result = cls.write_vhost(name, config)
if result.get('success'):
+8 -1
View File
@@ -497,7 +497,14 @@ class SiteDomainService:
if t in ('docker', 'wordpress'):
if not app.port:
return None, f'{t} app has no published port to route to.'
return dict(base, app_type='docker', port=app.port), None
docker_image = (getattr(app, 'docker_image', None) or '').strip().lower()
wordpress_protection = t == 'wordpress' or docker_image == 'wordpress'
return dict(
base,
app_type='docker',
port=app.port,
wordpress_protection=wordpress_protection,
), None
if t in ('flask', 'django', 'python'):
if not app.port:
return None, f'{t} app has no published port to route to.'
@@ -71,7 +71,10 @@ def _nginx_t(tmp_path, vhost_config):
(tmp_path / 'fastcgi_params').write_text(
open(system_params).read() if os.path.exists(system_params) else '')
zones = tmp_path / 'serverkit-microcache.conf'
zones.write_text(_localize(NginxService.MICROCACHE_ZONE_SNIPPET, tmp_path))
zones.write_text(
_localize(NginxService.MICROCACHE_ZONE_SNIPPET, tmp_path)
+ NginxService.WORDPRESS_RATE_LIMIT_ZONE_SNIPPET
)
vhost = tmp_path / 'vhost.conf'
vhost.write_text(_localize(vhost_config, tmp_path))
wrapper = tmp_path / 'nginx.conf'
@@ -109,6 +112,9 @@ def _render(**kwargs):
('docker-micro-cache', dict(name='fast', app_type='docker',
domains=['fast.example.com'], port=8004,
micro_cache=True)),
('wordpress-proxy', dict(name='wp', app_type='docker',
domains=['wp.example.com'], port=8005,
wordpress_protection=True)),
])
def test_generated_vhost_parses_with_real_nginx(tmp_path, flavor, kwargs):
proc = _nginx_t(tmp_path, _render(**kwargs))
+3 -1
View File
@@ -21,7 +21,8 @@ def _mk_user(db, username='owner'):
def _mk_app(db, user_id, name='blog', port=8300):
from app.models import Application
a = Application(name=name, app_type='docker', user_id=user_id,
root_path=f'/srv/{name}', port=port)
root_path=f'/srv/{name}', port=port,
docker_image='WordPress')
db.session.add(a)
db.session.commit()
return a
@@ -87,6 +88,7 @@ def test_provision_routing_creates_primary_domain_and_docker_vhost(app, monkeypa
# Reverse-proxy vhost, not php-fpm: docker app_type + the container's port.
assert captured['app_type'] == 'docker'
assert captured['port'] == 8300
assert captured['wordpress_protection'] is True
assert 'blog.lvh.me' in captured['domains']
@@ -0,0 +1,198 @@
"""WordPress shared-state and edge-protection contracts."""
from pathlib import Path
from types import SimpleNamespace
import yaml
from app.services.nginx_service import NginxService
from app.services.site_domain_service import SiteDomainService
from subprocess_stub import FakeProc
def _wordpress_template():
repo_root = Path(__file__).resolve().parents[2]
return yaml.safe_load((repo_root / 'backend' / 'templates' / 'wordpress.yaml').read_text())
def _tee_to_disk(fake):
def write(argv, kwargs):
path = Path(argv[2] if argv[1] == '-a' else argv[1])
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(kwargs.get('input', ''))
return FakeProc()
fake.when(['tee'], write)
def test_wordpress_template_shares_the_full_site_tree_between_replicas():
compose = _wordpress_template()['compose']
wordpress = compose['services']['wordpress']
assert 'wordpress_html:/var/www/html' in wordpress['volumes']
assert 'wordpress_html' in compose['volumes']
def test_wordpress_protection_adds_exact_login_and_xmlrpc_limits():
result = NginxService.render_site_config(
'blog',
'docker',
['blog.example.com'],
port=8300,
wordpress_protection=True,
)
assert result['success'] is True
config = result['config']
assert 'location = /wp-login.php {' in config
assert 'limit_req zone=serverkit_wp_login burst=10 nodelay;' in config
assert 'location = /xmlrpc.php {' in config
assert 'limit_req zone=serverkit_wp_xmlrpc burst=20 nodelay;' in config
assert config.count('proxy_pass http://127.0.0.1:8300;') == 3
assert config.index('location = /wp-login.php') < config.index('location / {')
def test_generic_docker_vhost_has_no_wordpress_limits():
result = NginxService.render_site_config(
'api',
'docker',
['api.example.com'],
port=8400,
)
assert result['success'] is True
assert 'serverkit_wp_login' not in result['config']
assert 'serverkit_wp_xmlrpc' not in result['config']
def test_wordpress_rate_limit_zones_are_partitioned_by_site_and_client():
snippet = NginxService.WORDPRESS_RATE_LIMIT_ZONE_SNIPPET
assert snippet.count('$server_name$binary_remote_addr') == 2
assert 'zone=serverkit_wp_login:10m rate=10r/m;' in snippet
assert 'zone=serverkit_wp_xmlrpc:10m rate=30r/m;' in snippet
def test_wordpress_rate_limit_zone_write_is_idempotent(
tmp_path, fake_subprocess, monkeypatch):
conf_dir = tmp_path / 'nginx'
monkeypatch.setattr(NginxService, 'NGINX_CONF_DIR', str(conf_dir))
_tee_to_disk(fake_subprocess)
first = NginxService.ensure_wordpress_rate_limit_zones()
second = NginxService.ensure_wordpress_rate_limit_zones()
assert first['success'] is True and first['changed'] is True
assert second['success'] is True and second['changed'] is False
conf_path = conf_dir / 'conf.d' / NginxService.WORDPRESS_RATE_LIMIT_CONF_NAME
assert conf_path.read_text() == NginxService.WORDPRESS_RATE_LIMIT_ZONE_SNIPPET
assert len([c for c in fake_subprocess.commands() if c[0] == 'tee']) == 1
def test_create_wordpress_vhost_installs_zones_before_write(monkeypatch):
events = []
monkeypatch.setattr(
NginxService,
'ensure_wordpress_rate_limit_zones',
classmethod(lambda cls: events.append('zones') or {'success': True}),
)
monkeypatch.setattr(
NginxService,
'write_vhost',
classmethod(lambda cls, name, content: events.append('vhost') or {
'success': True,
'path': f'/etc/nginx/sites-available/{name}',
}),
)
result = NginxService.create_site(
'blog',
'docker',
['blog.example.com'],
port=8300,
wordpress_protection=True,
)
assert result['success'] is True
assert events == ['zones', 'vhost']
def test_wordpress_template_application_enables_protection():
app = SimpleNamespace(
name='blog',
app_type='docker',
port=8300,
docker_image='WordPress',
micro_cache_enabled=False,
)
kwargs, warning = SiteDomainService._vhost_create_kwargs(
app,
['blog.example.com'],
None,
None,
)
assert warning is None
assert kwargs['app_type'] == 'docker'
assert kwargs['wordpress_protection'] is True
def test_generic_docker_application_does_not_enable_wordpress_protection():
app = SimpleNamespace(
name='api',
app_type='docker',
port=8400,
docker_image='nginx',
micro_cache_enabled=False,
)
kwargs, warning = SiteDomainService._vhost_create_kwargs(
app,
['api.example.com'],
None,
None,
)
assert warning is None
assert kwargs['wordpress_protection'] is False
def test_existing_wordpress_site_drift_target_includes_protection(app):
from app import db
from app.models import Application, User
from app.models.domain import Domain
from app.services.drift_service import _nginx_render_expected
from werkzeug.security import generate_password_hash
owner = User(
email='wp-drift@example.com',
username='wp-drift-owner',
password_hash=generate_password_hash('x'),
role=User.ROLE_ADMIN,
is_active=True,
)
db.session.add(owner)
db.session.commit()
wordpress = Application(
name='existing-wp',
app_type='docker',
status='running',
root_path='/srv/existing-wp',
docker_image='WordPress',
user_id=owner.id,
port=8500,
)
db.session.add(wordpress)
db.session.commit()
db.session.add(Domain(
name='existing-wp.example.com',
is_primary=True,
application_id=wordpress.id,
))
db.session.commit()
expected = _nginx_render_expected(wordpress.id)
[config] = expected.values()
assert 'limit_req zone=serverkit_wp_login burst=10 nodelay;' in config
assert 'limit_req zone=serverkit_wp_xmlrpc burst=20 nodelay;' in config
+96 -89
View File
@@ -1,29 +1,27 @@
# ServerKit — Per-Site Edge & Horizontal Scaling Spec
*Splitting the #34 epic — and specifying the **keystone** it shares with #21, #15-TLS, #30-jail, and #8/#25.*
*Implementation record and remaining design for WordPress horizontal scaling.*
> **Status:** design only. No code shipped. This doc exists so #34 (and its
> prerequisites) can be executed in focused passes **on a real Linux/Docker
> host**, where the infra can actually be verified — which is exactly why it
> wasn't built blind from the Windows dev environment.
> **Status (2026-08-27):** ServerKit ships the per-site edge and generic local
> Docker Compose replica controls. WordPress high availability remains
> incomplete. The supported single-host stack now has shared WordPress files,
> but WordPress replica orchestration, scaler-to-nginx upstream updates,
> health-based draining, and live failover proof have not shipped.
---
## 1. Why this doc
`docs/WORDPRESS_ROADMAP.md` defers **#34 (horizontal scaling)** as *"XL + blocked on
the same per-site reverse-proxy/upstream layer as #21."* That deferral is correct,
but it undersells how much already exists. A capability sweep of the nginx/SSL/storage
services shows the **load-balancing, rate-limiting, caching, per-vhost-log, wildcard-TLS,
and S3-upload primitives are already built** — they're just bound to the generic
`Application`/domain model, never to a `WordPressSite` running as a `localhost:PORT`
container.
`docs/WORDPRESS_ROADMAP.md` originally deferred **#34 (horizontal scaling)** on
the missing per-site reverse-proxy layer. ServerKit has since shipped that layer.
`SiteDomainService` publishes local managed sites through host nginx, the base-domain
registry manages DNS and wildcard HTTPS state, and `ContainerScaleService` changes a
local Compose service's replica count manually or from CPU thresholds.
So the real work is three layers, in order:
1. **The keystone** — a per-site *edge* (public domain → reverse-proxy vhost → the WP
container) with auto-TLS. This is the missing infrastructure that blocks **#21, #15-TLS,
#30-jail, #8-backend, and the original #25-nginx design** — and it is #34's prerequisite.
1. **The keystone**: a per-site edge from a public domain through an nginx vhost to
the WordPress container. ServerKit now ships this layer for local managed sites.
2. **Stateless WP** — shared media + shared object cache so replicas are interchangeable.
3. **Replicas + upstream LB** — multiple WP containers behind the keystone vhost.
@@ -35,20 +33,19 @@ real-host verification checklist.
## 2. Current reality (verified against the code)
| Fact | Evidence |
|---|---|
| A managed WP site is a single `wordpress:${VERSION}-apache` container publishing `${HTTP_PORT}:80` straight to the host; per-site MySQL + per-site Redis (#23). **No domain, no per-site vhost at create.** | `backend/templates/wordpress.yaml` (`ports: "${HTTP_PORT}:80"`) |
| `site.url` falls back to the host port unless a domain is attached. | `wordpress_service.py` `_enrich_site_data` (#3) |
| **A container-proxying vhost primitive already exists**, incl. HTTPS via cert + HTTP→HTTPS redirect. | `nginx_service.py:391` `create_site(..., ssl_cert, ssl_key)`, `:450` `_with_ssl`, `:535` `add_ssl_to_site` |
| **Upstream load-balancing, `limit_req`, `proxy_cache`, and per-vhost log reads already exist** — bound to a generic `domain`/`data` dict, not to a WP site. | `nginx_advanced_service.py:31` `create_reverse_proxy` (`upstreams`→`upstream{}` at `:47`, `proxy_pass http://{upstream}` at `:95/:113`, `limit_req_zone` at `:61`, `proxy_cache_path` at `:68`), `:173` `get_vhost_logs`, `:196` `get_load_balancing_methods` |
| Wildcard cert issuance + auto-renew already exist. | `advanced_ssl_service.py:39` `issue_wildcard_cert(domain, dns_provider, credentials, email)`, `ssl_service.py:256` `setup_auto_renewal` |
| Subdomain generation for environments already exists. | `environment_domain_service.py:61` `generate_domain(production_domain, env_type, …)` |
| S3/B2/MinIO directory upload already exists (for media offload). | `storage_provider_service.py:260` `upload_directory(local_dir, remote_prefix)` |
| Area | State | Evidence and boundary |
|---|---|---|
| Base domains, DNS mode, wildcard HTTPS | **Shipped** | `SiteBaseDomain`, `SiteBaseDomainService`, `SitesHttpsService`, and the Settings UI manage multiple bases and provider bindings. Certificate issuance still needs a real DNS provider and Linux host. |
| WordPress site edge | **Shipped for local sites** | WordPress provisioning calls its routing path, which records a primary `Domain`; `SiteDomainService.write_app_vhost` renders and enables a Docker proxy vhost. The site URL falls back to `localhost:<port>` when no base domain exists. |
| Per-site logs and protection | **Shipped for local WordPress sites** | Nginx templates write per-site access/error logs. WordPress publishing adds exact-match `wp-login.php` and `xmlrpc.php` limits backed by shared zones partitioned by site and client IP. Bandwidth and Fail2ban services consume the same access log. |
| Generic Compose scaling | **Shipped for local apps** | `ContainerScaleService` and the Container Ops UI support policy read/write, manual replica changes, one policy evaluation, and an admin sweep. Migration `093_container_scale_policies` versions the policy table. `current_replicas` tracks ServerKit commands rather than live Docker reconciliation. |
| Generic load-balancer rendering | **Shipped as a separate primitive** | `NginxAdvancedService.create_reverse_proxy` renders round-robin, least-connections, or IP-hash upstreams. The scaler does not update this config. |
| WordPress shared state | **Shipped for the supported single-host topology** | The WordPress template mounts the full `/var/www/html` tree from one Compose named volume, so replicas in that project share uploads, plugins, themes, and Cache Enabler files. Redis supplies the shared object cache when enabled. Multi-host replicas still need NFS/CIFS or object offload. |
| WordPress replica orchestration and failover | **Open** | The WordPress stack still publishes one container port. ServerKit does not create internal WordPress replicas, register them in the vhost, drain unhealthy replicas, or prove traffic survives a replica loss. |
**Conclusion:** the gap is *wiring + a public base-domain + statelessness*, not greenfield nginx
work. That makes both the keystone and #34 far smaller than "XL from zero" implies — but every
phase touches nginx apply/reload, cert issuance, or container orchestration, **none of which is
runtime-verifiable on Windows.**
Generic replica scaling improves capacity management. It does not provide WordPress high
availability by itself because the request path, persistent files, and health lifecycle remain
outside the scaling policy.
---
@@ -75,53 +72,53 @@ runtime-verifiable on Windows.**
└────────────────────────────────────────────────────────────────────────────────────────┘
```
**Do K0→K2 first** — it alone closes #15-TLS, #8-backend, and #21 (with K3), independent of #34.
**#34 cannot start before H0**, and H0/H1 require the keystone's edge to exist (replicas can't each
publish a host port).
K0 through K2 and K4 now exist for local managed sites. H1 and H2 remain the critical path
for WordPress: internal replicas first, then an upstream that tracks those replicas.
---
## 4. Part 1 — The keystone: per-site edge
### K0 — Panel-wide public base domain + wildcard DNS/TLS `[M]`
### K0: Panel-wide public base domain + wildcard DNS/TLS `[SHIPPED]`
- **Goal:** a configurable base domain (e.g. `apps.example.com`) with `*.apps.example.com` DNS and a wildcard cert, so *any* managed site can get a real public URL. This is the single setting whose absence makes every public-URL feature impossible today.
- **Do:** add a `base_domain` + DNS-provider binding to settings; on save, issue/refresh the wildcard cert and schedule renewal. Validate the wildcard record resolves before declaring it ready.
- **Reuse:** `advanced_ssl_service.issue_wildcard_cert` (`:39`), `ssl_service.setup_auto_renewal` (`:256`), the existing DNS-provider credentials (Connections / `provider_secret` encryption).
- **Shipped:** `SiteBaseDomain` stores multiple base domains, default selection, DNS mode, HTTPS state, and a DNS-provider binding. `SitesHttpsService` provisions wildcard DNS and certificates. Setup Health reports missing DNS and HTTPS prerequisites.
- **Reuse:** `advanced_ssl_service`, `ssl_service`, and encrypted DNS-provider connections.
- **Acceptance:** with a base domain configured, `dig *.base` resolves and a valid `*.base` cert exists.
- **Verifiable here?** Config/model/validation logic yes; DNS + cert issuance **real host only**.
- **Risk:** registrar/DNS-provider variance; wildcard issuance needs DNS-01 (already supported). If no public base domain is set, the whole edge degrades gracefully to today's `localhost:PORT` behavior — keep that fallback.
- **Verification:** pytest covers registry behavior, publishing gaps, setup health, and setup reconciliation. DNS propagation and certificate issuance still require a real provider and host.
- **Risk:** registrar and DNS-provider behavior varies. A site falls back to `localhost:<port>` when the operator has not configured a base domain.
### K1 — Per-site reverse-proxy vhost at create/attach `[L]`
### K1: Per-site reverse-proxy vhost at create/attach `[SHIPPED: local sites]`
- **Goal:** put an nginx vhost in front of each managed WP container so it answers on a real hostname instead of `localhost:PORT`.
- **Do:** on create (and on domain-attach), call `NginxService.create_site(name, app_type='docker', domains=[hostname], port=<container published port>)` + `enable_site` + `reload`. Persist the hostname on `WordPressSite`. Make WP proxy-aware: set `WP_HOME`/`WP_SITEURL` to the hostname, trust `X-Forwarded-Proto`/`-For` in `wp-config`, and run the serialized-safe `search-replace` (#5/#9 primitive) from `localhost:PORT` → the hostname.
- **Reuse:** `nginx_service.create_site/enable_site/reload` (`:391/:483/:288`), `db_sync_service` search-replace, the `domains.py` attach flow (#8 already wires domain-attach for generic docker apps).
- **Shipped:** local WordPress provisioning records a primary `Domain` and calls the shared site-vhost writer with the application's published port. Domain attachment uses the same `SiteDomainService` rendering path.
- **Reuse:** `NginxService.create_site`, `SiteDomainService.write_app_vhost`, and the WordPress URL-swap path.
- **Acceptance:** a freshly created site is reachable at `http://<hostname>` and `wp-admin` loads with correct URLs (no redirect loop, no mixed `localhost` links).
- **Verifiable here?** Template rendering + the `wp-config`/search-replace logic yes; nginx apply + live HTTP **real host only**.
- **Risk:** the classic WP-behind-proxy redirect loop (needs `X-Forwarded-Proto` handling); a port is still published in K1 (internal-only comes in H1).
- **Verification:** `test_site_routing.py`, `test_dns_give_subdomain.py`, and the nginx rendering suites cover the database and config path. Live HTTP and redirect-loop checks still require a Linux host.
- **Risk:** the container still publishes one host port. H1 must change that topology without cutting off the existing vhost.
### K2 — Auto-TLS per site `[M]`
### K2: Auto-TLS per site `[SHIPPED: wildcard base domains]`
- **Goal:** every site is HTTPS by default.
- **Do:** for a `*.base` subdomain, serve the K0 wildcard cert via `create_site(ssl_cert=…, ssl_key=…)` (the `_with_ssl` path already does the 80→443 redirect). For an attached custom domain, issue a per-host LE cert. Add `ssl_status` to `WordPressSite` and surface grade/expiry (the #8 `SiteSSLPanel` already renders this once a domain is routed).
- **Reuse:** `nginx_service.create_site(ssl_cert, ssl_key)` (`:391`) / `add_ssl_to_site` (`:535`), `ssl_service`/`advanced_ssl_service`, the #8 frontend.
- **Shipped:** a base-domain row marked HTTPS-ready supplies the wildcard certificate to generated vhosts. HTTP redirects to HTTPS through the nginx SSL rendering path. Custom-domain certificate workflows remain separate from base-domain wildcard setup.
- **Reuse:** `NginxService.create_site`, `NginxService.add_ssl_to_site`, `SitesHttpsService`, and the SSL services.
- **Acceptance:** the site loads over HTTPS with a valid cert; HTTP redirects to HTTPS.
- **Verifiable here?** `ssl_status` model + wiring yes; cert + TLS handshake **real host only**.
- **Verification:** unit tests cover certificate-path selection and vhost rendering. A real host must prove issuance, renewal, and the TLS handshake.
- **Risk:** custom-domain certs need DNS pointed first — keep a "pending DNS" state.
### K3 — Preview URL before DNS (#21) `[S–M, on top of K0–K2]`
### K3: Preview URL before DNS (#21) `[OPEN: exact design]`
- **Goal:** a working HTTPS preview link the instant a site exists, pre-DNS.
- **Do:** mint `<site|hash>.preview.<base>` via the K0 wildcard (covered by the wildcard cert, so no per-host issuance). Optionally a basic-auth gate (the env-pipeline already has basic-auth, P4). This **is** #21 once K0–K2 exist.
- **Current state:** a configured base domain gives new local sites an immediate managed hostname. ServerKit also ships PR preview environments for generic applications. It does not mint the separate `<site>.preview.<base>` WordPress URL described here.
- **Do:** add the separate namespace if product requirements still need it. The current managed hostname may satisfy the pre-custom-DNS workflow.
- **Reuse:** `environment_domain_service.generate_domain` (`:61`), K0 wildcard, `create_site`.
- **Acceptance:** every new site shows an immediately-working `https://….preview.base` link.
- **Verifiable here?** Hostname generation yes; live preview **real host only**.
### K4 — Per-site access log + `limit_req` jail (restores #25-nginx, unblocks #30-jail) `[M]`
### K4: Per-site access log + `limit_req` jail `[SHIPPED: local WordPress sites]`
- **Goal:** a per-site nginx access log on the host + brute-force protection — the exact signals #25 and #30 had to defer because the container model exposes none.
- **Do:** emit a per-site `access_log` path in the K1 vhost. Add a `limit_req` zone for `wp-login.php`/`xmlrpc.php` via `create_reverse_proxy` (it already emits `limit_req_zone`/`limit_req`). Point #25's analytics at `get_vhost_logs` (a real per-vhost log) instead of/alongside the `docker logs` source it fell back to; this also restores `%D` response-time once we own the `LogFormat`.
- **Reuse:** `nginx_advanced_service.create_reverse_proxy` (`:31`, `limit_req` at `:61/:85`), `get_vhost_logs` (`:173`), `security_service` Fail2ban (optional, watching the new log).
- **Acceptance:** repeated `wp-login` hits get throttled; per-site visits/5xx/slow-pages come from the vhost log.
- **Verifiable here?** Log-parsing + zone-rendering yes; live throttling + log emission **real host only**.
- **Shipped:** generated vhosts write per-site access and error logs. WordPress vhosts add exact locations for `wp-login.php` and `xmlrpc.php`, return HTTP 429 after their burst allowance, and reference shared http-level zones keyed by `$server_name$binary_remote_addr`. The site key prevents one site's traffic from consuming another site's allowance. Bandwidth analytics and the WordPress Fail2ban jail use the canonical access-log path. Existing vhosts appear as drift until the operator runs the normal nginx repair, which regenerates them with the policy.
- **Reuse:** `NginxService.create_site`, `NginxService.site_access_log_path`, and `Fail2banJailService`.
- **Acceptance:** the generated WordPress vhost contains both endpoint limits; repeated login hits get throttled on a real host; per-site visits, status codes, and bandwidth come from the vhost log.
- **Verification:** `test_wordpress_ha_foundations.py` covers zone rendering, WordPress-only vhost injection, idempotent zone installation, publishing-path detection, and the generic-Docker boundary. The opt-in real-nginx suite parses the generated vhost. Live throttling and log emission still require a Linux host.
**After Part 1:** #15-TLS ✅, #21 ✅, #8-backend ✅, #30-jail ✅, #25 (response-time/slow-pages) ✅ — independent of #34.
**After Part 1:** #15-TLS ✅, #21 ✅, #8-backend ✅, and #30-jail ✅. The vhost log supplies #25 traffic/status/bandwidth data; response-time and slow-page metrics still need a timed log format.
---
@@ -131,28 +128,24 @@ publish a host port).
> host port; the **edge vhost (K1)** becomes the only public entry and load-balances across
> internal replicas.
### H0 — Make WP stateless `[L]`
- **Goal:** replicas must be interchangeable — no replica-local state.
- **What's already fine:** the **object cache is already shareable** — the #23 Redis is one service in the site's compose stack, so N WP replicas pointing at the same `redis` service share it automatically. No change needed there.
- **The real statefulness is two things:**
1. **Uploads / `wp-content/uploads`** (local disk per replica). **Decision (open):**
- *Option A — S3/object offload* (a media-offload plugin writing to the site's configured `storage_provider`). Pro: truly stateless, scales infinitely. Con: a plugin dependency (mild tension with the WP-CLI-over-plugin thesis), egress cost, and rewriting existing media URLs. Reuse `storage_provider_service.upload_directory` (`:260`) + creds.
- *Option B — shared volume* (NFS/CIFS or a Docker named volume shared across replicas on one host). Pro: no plugin, transparent. Con: needs shared-FS infra; single-host only unless NFS; locking edge-cases.
- **Recommendation:** Option A for multi-host scale; Option B as a single-host MVP. Spec both; pick at execution.
2. **On-disk page cache** — if #22 landed a *disk-writing* page-cache plugin, that's replica-local and must be moved to Redis or disabled under LB. Verify #22's cache backend before H1.
### H0: Share WordPress application state `[SHIPPED: single-host topology]`
- **Goal:** replicas must be interchangeable within the supported local Compose project.
- **Shipped:** the template mounts one `wordpress_html` named volume at `/var/www/html`. Every replica of the Compose `wordpress` service mounts that same volume, so uploads, plugins, themes, and Cache Enabler's disk cache are shared instead of replica-local. The database service is shared, and the optional Redis object cache already points every replica at the same `redis` service.
- **Boundary:** Docker named volumes are host-local. A future multi-host topology must replace this volume with NFS/CIFS or an object-storage media offload. That is not required for ServerKit's current local-only scaler.
- **Acceptance:** uploading media on replica A is immediately served by replica B.
- **Verifiable here?** Offload-config + URL-rewrite logic yes; cross-replica consistency **real host only**.
- **Risk:** media migration for *existing* sites; plugin licensing for offload.
- **Verification:** `test_wordpress_ha_foundations.py` pins the template's full-tree shared-volume mount. Cross-replica file visibility still needs the real-host H1 test because the current WordPress service cannot start multiple replicas yet.
- **Risk:** concurrent plugin, theme, and core updates must remain serialized once H1 ships. Multi-host scale needs a different storage backend.
### H1 — Replica orchestration `[L]`
### H1: Replica orchestration `[OPEN for WordPress; generic scaler shipped]`
- **Goal:** run N WP containers for one site.
- **Current state:** Container Ops can run `docker compose up --scale <service>=N` for a local, scale-capable Compose application. Policies support minimum and maximum replicas, CPU thresholds, cooldown, one evaluation, and a cron-driven sweep. The service must avoid fixed host ports and `container_name`. The WordPress template does not meet that topology yet.
- **Do:** change the stack so the WP service is replica-able (`deploy.replicas` under swarm, or N enumerated services / `docker compose up --scale` on a compose host) on an **internal** network — **drop the `${HTTP_PORT}:80` host publish**; only the edge nginx reaches them. This is a **breaking change to the single-container model**, so it needs a per-site opt-in + a migration path (existing sites stay single-container until "Enable scaling").
- **Reuse:** the `wordpress.yaml` template + the `_ensure_*_in_stack` additive-compose pattern (#23 redis injection is the proven precedent).
- **Acceptance:** `docker ps` shows N WP replicas for the site, none publishing a host port.
- **Verifiable here?** Template rendering yes; actual replicas **real host only**.
- **Risk:** the publish→internal flip must be coordinated with K1 (edge must exist first or the site goes dark); MySQL stays single (not replicated here — that's a separate epic).
### H2 — Upstream load-balancing wiring `[M]`
### H2: Upstream load-balancing wiring `[OPEN integration]`
- **Goal:** the edge vhost balances across the H1 replicas.
- **Do:** upgrade the K1 vhost to an upstream-backed one: feed the replica endpoints to `create_reverse_proxy(upstreams=[…], method=…)`. Anonymous WP is stateless → round-robin; logged-in/cart needs affinity → `ip_hash` or cookie-sticky (expose via `get_load_balancing_methods`).
- **Reuse:** `nginx_advanced_service.create_reverse_proxy` (`upstream{}` at `:47`, `proxy_pass` at `:95`), `get_load_balancing_methods` (`:196`).
@@ -160,54 +153,68 @@ publish a host port).
- **Verifiable here?** Upstream-config rendering yes; live balancing/failover **real host only**.
- **Risk:** session affinity for WooCommerce carts; cache coherence across replicas (mitigated by H0 shared Redis).
### H3 — Health checks + rolling deploys `[M]`
### H3: Health checks + rolling deploys `[OPEN integration]`
- **Goal:** safe scale up/down and zero-downtime deploys.
- **Do:** per-replica health probe; drain a replica out of the upstream before stop; a scale API on the `WordPressSite`. Tie deploys (#13/#29) into a rolling pattern: update one replica, health-check, proceed.
- **Reuse:** `environment_health_service` (#26 poller), `nginx_advanced_service` (rewrite upstream on scale), #29 safe-update health gate.
- **Acceptance:** scaling and deploys cause no failed requests.
- **Verifiable here?** Orchestration logic yes; zero-downtime claim **real host only**.
### H4 — Autoscale signal (optional) `[M]`
### H4: Autoscale signal (optional) `[PARTIAL: generic CPU policy shipped]`
- **Goal:** scale on load.
- **Do:** drive H3's scale API from the #25 per-site traffic metrics (now real, via K4). Thresholds + cooldown.
- **Shipped:** generic local Compose applications can scale from average service CPU with minimum, maximum, and cooldown bounds. Raising the minimum now forces the next evaluation to restore that floor without waiting for high CPU.
- **Open:** connect an internal scheduler, health gates, and the WordPress upstream lifecycle. Per-site traffic metrics do not drive the policy.
- **Reuse:** #25 analytics, H3 scale API.
- **Acceptance:** sustained load adds a replica; quiet removes one.
---
## 6. Verifiable on Windows vs. needs a real Docker host
## 6. Verification record
| Verifiable here (pytest / unit) | Real Linux+Docker host only |
Automated coverage now checks these paths:
| Layer | Coverage |
|---|---|
| `WordPressSite` schema additions (hostname, ssl_status, scaling flags, replica count) | nginx config **apply + reload** |
| nginx **template/string rendering** (vhost, upstream, limit_req, ssl block) | Let's Encrypt / wildcard **cert issuance** |
| `wp-config` mutation + serialized search-replace logic | live HTTP / HTTPS, redirect-loop checks |
| service unit logic with mocked shell/Docker | **container replicas**, `--scale`, internal networking |
| API routes + RBAC + workspace scoping | **load-balancing / failover** behavior |
| media-offload config + URL-rewrite logic | cross-replica media + cache consistency |
| `limit_req` zone generation | actual brute-force **throttling** |
| Frontend policy form | `autoScalePolicy.test.mjs` covers defaults, request normalization, the one-replica floor, and use of the API's applied replica count. |
| Frontend API contract | `containerOps.test.mjs` pins the four scaling paths, methods, and request bodies used by the UI. |
| Backend decision logic | `test_container_scale.py` covers threshold scale-up/down, bounds, cooldown, missing metrics, invalid policy input, and minimum-floor restoration. |
| API workflow | `test_policy_to_evaluation_to_manual_scale_workflow` drives GET policy, PUT policy, POST evaluation, POST manual scale, and a final persisted GET through Flask and SQLite. |
| Authorization and schema | The route authorization suites cover the mutating endpoints. The migration drift test includes migration `093_container_scale_policies`. |
| WordPress shared state | `test_wordpress_ha_foundations.py` pins the shared `/var/www/html` named-volume contract used by uploads and disk caches. |
| WordPress edge protection | `test_wordpress_ha_foundations.py` covers endpoint-specific limits, shared-zone isolation, idempotent installation, publishing-path detection, and the generic-Docker boundary. |
→ Each real-host item gets a **manual verification checklist** in the executing PR; nothing infra
ships claimed-working until exercised on a host (per the project's verify-before-claiming bar).
A Linux host with Docker and nginx must still prove the runtime behavior. Use a disposable
Compose application whose target service has no `container_name` or fixed host port:
1. Open the application's Settings tab, save a policy with minimum 2, maximum 3, and the target service name.
2. Call `POST /api/v1/apps/<id>/scale/evaluate`, then confirm `docker compose ps <service>` reports two running replicas.
3. Send load until CPU exceeds the high threshold, run another evaluation after cooldown, and confirm a third replica starts.
4. Send traffic through the public hostname and confirm each replica receives requests. This step will fail until an upstream-aware vhost is configured; the generic scaler does not create one.
5. Stop one replica while requests continue. Record any failed requests. ServerKit cannot claim failover until this test passes through a ServerKit-managed upstream.
6. For WordPress, upload media through one replica and request it through another. The template provides the shared volume, but ServerKit cannot claim WordPress HA until H1 makes this live test possible and it passes.
Steps 4 through 6 define the remaining HA acceptance gate. Passing the automated suite proves
control-plane behavior, not traffic continuity.
---
## 7. Sequencing recommendation
## 7. Remaining sequence
1. **K0 → K1 → K2** — the highest-value slice. Closes **#15-TLS, #8-backend**, and (with **K3**) **#21**. Ship and verify on a host *before* touching #34.
2. **K4** — closes **#30-jail** and restores **#25** response-time/slow-pages.
3. **H0 → H1 → H2** — the #34 MVP (stateless + replicas + LB). Gate H1 behind a per-site "Enable scaling" opt-in so the single-container model stays the default.
4. **H3 → H4** — production hardening + autoscale.
1. Add a WordPress-specific opt-in that changes the single published container into internal replicas.
2. Make the site vhost derive its upstream list from those replicas and update it during scale operations.
3. Add per-replica health checks and draining, then run the real-host continuity test in section 6.
4. Connect the existing CPU policy to the internal scheduler after health-based scaling works.
Each arrow is a separately shippable, separately verifiable PR. **#34's "Done when" (N replicas behind
a balancer with consistent media + cache) is reached at the end of H2.**
#34 reaches its acceptance condition after one site runs multiple replicas behind a managed
balancer, serves consistent media and cache state, and survives a replica loss without failed
requests.
---
## 8. Open decisions (resolve at execution)
- **Media:** S3 offload (Option A) vs shared volume (Option B) — pick per deployment topology (multi-host ⇒ A).
- **Port model:** the K1-publishes-a-port → H1-internal-only flip is a breaking change; needs a migration for existing sites and a clear opt-in.
- **Multi-host media:** keep the shipped named volume for local Compose; choose NFS/CIFS or object offload before adding a multi-host topology.
- **Port model:** the K1-publishes-a-port to H1-internal-only change needs a migration for existing sites and a clear opt-in.
- **MySQL:** stays single-instance in this spec. DB replication/clustering is a distinct epic, explicitly out of scope for #34.
- **Panel vs sites:** the *panel's* single-gevent-worker constraint (agent gateway, see `CLAUDE.md`/`ARCHITECTURE.md`) is unrelated — this LB is for the **managed sites**, not the panel. Don't conflate.
- **Plugin thesis:** media-offload (Option A) and any disk→Redis page-cache move may need plugins; weigh against the WP-CLI-over-plugin preference.
+6 -6
View File
@@ -278,8 +278,8 @@ Highest-effort, broadest surface — but by now the `WordPressSite` is a first-c
- **Do:** Per-site scan-target resolution; file integrity via `wp core/plugin/theme verify-checksums`; a Fail2ban jail/filter (or nginx `limit_req` zone) targeting each site's access log for `wp-login.php`/`xmlrpc.php`; per-site `WP_DEBUG`/`SCRIPT_DEBUG` toggle and WP-Cron list/run/disable.
- **Reuse:** `security_service` (Fail2ban jails), `wp_cli`, `cron_service`, `nginx_service` WP template.
- **Done when:** A site can be scanned, checksum-verified, brute-force-protected, and debug-toggled.
- **Landed:** New `WpSecurityService` + a per-site **Security tab** (Docker-correct, all via the WP-CLI bridge): **(1) File integrity** — a background `wp core verify-checksums` + `wp plugin verify-checksums --all` (in-container, so it reaches the volume; runs off the request thread and the UI polls), flagging tampered/unexpected files. **(2) Debug toggle** — sets `WP_DEBUG`/`SCRIPT_DEBUG` with `WP_DEBUG_DISPLAY=false` and, crucially, **`WP_DEBUG_LOG=/tmp/wp-debug.log`** — *outside* the web root, so the log is never publicly fetchable (the default `wp-content/debug.log` is apache-served — a real info leak this avoids); the write is gated so a read-only/stopped site reports an honest failure. **(3) WP-Cron** — `DISABLE_WP_CRON` status + due-event list, run-due-now, and enable/disable (with a clear "needs a real system cron" warning). Routes mirror the established per-site pattern (owner-or-admin GET, `@admin_required` mutations). *(Deferred — the brute-force `wp-login`/`xmlrpc` **rate-limit jail**: Fail2ban/nginx `limit_req` both need a host-side per-site access log to watch, which the `localhost:PORT` apache-container model doesn't expose — the same per-site reverse-proxy/log gap as #21/#25/#8-backend. A WP-level login-limiter plugin is the Docker-correct alternative, deferred to honour the WP-CLI-over-plugin thesis.)* Verified: backend `py_compile`, frontend lint-clean + build, focused security review (debug-log web-exposure caught + fixed). Not runtime-exercised on a real host.
- **Update — brute-force jail shipped:** the deferral above is now redeemed. Every managed site already has a per-site nginx reverse-proxy vhost whose access log (`NginxService.site_access_log_path`) records `wp-login.php`/`xmlrpc.php` hits with the **real client IP** (nginx is the edge listener), so there *is* a host-side per-site log for a jail to watch. New `Fail2banJailService` (the write half of the previously read-only Fail2ban integration) generates a ServerKit-owned filter + per-site jail (`jail.d/serverkit-<app>.conf`, ownership-guarded, Linux-only, best-effort). Protection is **on by default**: the jail is created when a WP site is created (after its vhost, so the log exists) and removed on delete; a per-site **Brute-force protection** card in the Security tab toggles it, shows live bans, and unbans (5 login POSTs / 10 min → 1 h ban). Degrades gracefully where Fail2ban is absent (e.g. Windows dev → "fail2ban not available"). Verified: 17 new unit tests (rendered filter/jail, ownership/traversal guards, graceful skip) + full backend suite green + frontend build. Not runtime-exercised on a real host.
- **Landed:** New `WpSecurityService` + a per-site **Security tab** (Docker-correct, all via the WP-CLI bridge): **(1) File integrity** — a background `wp core verify-checksums` + `wp plugin verify-checksums --all` (in-container, so it reaches the volume; runs off the request thread and the UI polls), flagging tampered/unexpected files. **(2) Debug toggle** — sets `WP_DEBUG`/`SCRIPT_DEBUG` with `WP_DEBUG_DISPLAY=false` and, crucially, **`WP_DEBUG_LOG=/tmp/wp-debug.log`** — *outside* the web root, so the log is never publicly fetchable (the default `wp-content/debug.log` is apache-served — a real info leak this avoids); the write is gated so a read-only/stopped site reports an honest failure. **(3) WP-Cron** — `DISABLE_WP_CRON` status + due-event list, run-due-now, and enable/disable (with a clear "needs a real system cron" warning). Routes mirror the established per-site pattern (owner-or-admin GET, `@admin_required` mutations). Verified: backend `py_compile`, frontend lint-clean + build, focused security review (debug-log web-exposure caught + fixed). Not runtime-exercised on a real host.
- **Update — layered brute-force protection shipped:** every managed site has a per-site nginx reverse-proxy vhost whose access log (`NginxService.site_access_log_path`) records `wp-login.php`/`xmlrpc.php` hits with the real client IP. `Fail2banJailService` generates a ServerKit-owned filter and per-site jail (`jail.d/serverkit-<app>.conf`, ownership-guarded, Linux-only, best-effort); the Security tab controls it, shows live bans, and unbans. WordPress publishing also installs shared nginx request-limit zones and exact endpoint locations automatically: `wp-login.php` allows 10 requests/minute with a burst of 10, while `xmlrpc.php` allows 30 requests/minute with a burst of 20. The zone key combines server name and client IP, which isolates each site's allowance. Focused tests cover rendering, idempotent zone writes, WordPress-only publishing, and the generic-Docker boundary. Live enforcement still needs a Linux nginx host.
### #31 — Selective push UI `[S]` 🟡 — ✅ Done (specific-tables selectors added; files-only/db-only/specific-folders already existed)
- **Today:** The dump layer already supports table filters / files-only / db-only; `PromoteModal` exposes only coarse code/db/full.
@@ -310,13 +310,13 @@ Highest-effort, broadest surface — but by now the `WordPressSite` is a first-c
- **Landed (per-site ACL — backend):** per-resource **sharing** — an owner/admin can grant a specific user access to a specific application (transitively its WordPress site / DB / domains) without that user owning it (access-level; per-grant view/edit roles reserved for later via a `role` column). New `ResourceGrant` model (Alembic **016**, unique per user×resource) + `ResourceGrantService` (grant/revoke/list/`user_has_grant`/`granted_ids`). **Enforcement at the cleanest seams:** `WorkspaceService.scope_query` gained `grant_resource_type` so granted apps appear in the apps list (own **OR** granted, then workspace-narrowed); `get_app` and — crucially — the WordPress `_owner_or_admin_app` helper became grant-aware, so **one change opens every WP per-site route** to a grantee. Grant-management API on `apps.py` (`GET/POST/DELETE /apps/<id>/grants`, owner-or-admin; can't grant to the owner; revoke is resource-scoped so you can't revoke another app's grant by id). Also uses `user.id` (int) not the stringified `get_jwt_identity()`. Verified: **4 new ACL tests** (visibility via list, get_app, WP route access, management perms + idempotency + revoke), full suite **127 passed**, end-to-end `alembic upgrade` to `016`. **Sharing UI (shipped):** a per-app **Share** action in the Workspaces "Resources" modal opens an inline sub-view (grant a user → access list with revoke; the owner and existing grantees are filtered out of the add-list); client methods `getAppGrants`/`grantAppAccess`/`revokeAppAccess`. Verified: frontend lint-clean + build. **Viewer/editor roles + operate access (shipped):** grants now carry a real **role** — `editor` (view + operate) or `viewer` (read-only) — and the generic app **operate** endpoints became grant-aware. Two helpers split it: `_can_access_app` (owner/admin/**any** grant) gates reads (`get_app`, `/linked`, `/logs`, `/container-logs`, `/containers`, `/status`), `_can_edit_app` (owner/admin/**editor** grant) gates mutations (`/link`, `/unlink`, `/environment`, `PUT`, `/start`, `/stop`, `/restart`); `ResourceGrantService.grant_role` backs the split. **Delete stays owner-or-admin** (an editor can't delete — tested). The grant POST validates `role ∈ {viewer, editor}` (default editor); the Sharing UI gained a role selector. The same sweep **fixed the latent int-vs-str ownership bug** across ~14 app endpoints (they compared `app.user_id` to the stringified `get_jwt_identity()` — now `user.id`). Verified: **3 more ACL tests** (viewer read-only + can't-operate, editor operates + can't-delete, role validation/default), full suite **130 passed**, frontend lint-clean + build. **Cross-blueprint sweep (shipped):** the per-resource grant now extends to every app-related blueprint, not just apps.py + WordPress. The two access helpers were promoted to a shared seam (`ResourceGrantService.can_access_app` / `can_edit_app`; `apps.py` delegates), and **33 ownership checks across 6 blueprints** were made grant-aware, categorized read-vs-mutate (via a parallel mapping workflow): `builds.py` (12), `domains.py` (7, incl. SSL enable/disable/renew + domain CRUD = editor), `deploy.py` (6), `private_urls.py` (5), `templates.py` (2), `databases.py` (1). Reads → any grant; mutates → editor grant. The sweep also **fixed the same latent int-vs-str ownership bug** in all 33 (now `user.id`). Verified: a cross-blueprint integration test (grantee reaches `builds` deployments; stranger 403), full suite **131 passed**. **`python.py` (shipped):** re-reading showed every mutating endpoint there is independently `@admin_required` (runs before the shared `get_app_or_404` gate), so the gate only governs the 4 read GETs — pointing it at `can_access_app` safely opens those to grantees while the admin-only mutations are untouched. No per-method split needed. With this, **every app-related blueprint is grant-aware.** Verified: a python read-gate test (stranger 403, grantee passes), full suite **132 passed**.
- **Landed (domains/databases scoping + role reconciliation):** the two named remaining scoping slices, plus the reconciliation primitive that makes the per-workspace role *mean* something. **(1) Domains** — `get_domains` now scopes through the parent `Application` (domains carry no `workspace_id` of their own): it resolves the active workspace and lists the domains of the apps `scope_query` returns (own + granted, workspace-narrowed). With no workspace context this preserves prior behavior (admin → all domains, owner → own apps' domains) *and* now also surfaces **granted** apps' domains — closing a latent gap where the per-domain routes authorized via `can_access_app` but the list ignored grants. Since `Domain.application_id` is non-nullable, the admin-no-context path is byte-identical to the old `Domain.query.all()`. **(2) Databases** — only the **Docker-databases** enumeration (`/docker/databases`) is app-linked and therefore workspace-scopable; it now drives its app iteration through the same `scope_query` (replacing a manual `can_access_app` loop with the equivalent query-level filter + workspace narrowing). The **host-level MySQL/PostgreSQL/SQLite** endpoints are server-global system resources with no app/workspace linkage (and already admin-gated for mutation), so they're **intentionally left unscoped** — documented inline. **(3) Role reconciliation** — new `WorkspaceService.effective_role(user, workspace_id)` + `can_write_in_workspace`: reconcile the user's **global** `User.role` with their **per-workspace** `WorkspaceMember.role`, **narrow-only** (mirrors `scope_query`) — a workspace role can only *reduce* capability, never elevate. A system admin is never capped (no workspace lock-out); a workspace **viewer** caps a global developer to read-only *in that workspace*; any other membership leaves the global role intact. Enforced at the resource-stamping write seams (apps + servers `create` and `…/workspace` reassign): a workspace viewer gets a **403** creating/moving resources into that workspace — checked **only** when a workspace context is active, so no-context behavior is unchanged. The reconciled role is surfaced on `GET /workspaces` (`my_role` / `my_effective_role`) so the frontend can show read-only state. **No schema/migration** (pure query + capability logic on existing columns) and **no frontend change** (the WP hub/apps/domains/databases already fetch through the `X-Workspace-Id`-injecting client; the 403s surface via the existing toast). Verified: backend `py_compile`, **6 new pytest tests** (effective-role reconciliation incl. the never-elevate + admin-never-capped cases, viewer-blocked create + reassign, domains scoping incl. grant-visibility + admin-sees-all, docker-databases scoping, list-workspaces surfacing), full suite **193 passed / 12 skipped**. *(Still remaining for the dedicated milestone: server-side workspace-scoped **logo** swap, and the broad per-route write-enforcement of the viewer-cap beyond the create/reassign seams — an enforcement sweep across the mutating routes, the same XL surface as the rest of #33.)* Not runtime-exercised against the live frontend (backend + API-level tests only).
### #34 — Horizontal scaling `[XL]` ❌ — ⏸️ Deferred, now **split into a spec** ([docs/HORIZONTAL_SCALING_SPEC.md](HORIZONTAL_SCALING_SPEC.md)) — XL + blocked on the same per-site reverse-proxy/upstream layer as #21
- **Today:** `nginx_advanced_service` has load-balancing primitives (round-robin/least-conn/IP-hash) but WP containers are stateful (local uploads, no shared object cache).
### #34: Horizontal scaling `[XL]` 🟡 | ⏸️ WordPress HA deferred; edge and generic scaling shipped ([docs/HORIZONTAL_SCALING_SPEC.md](HORIZONTAL_SCALING_SPEC.md))
- **Today:** local managed WordPress sites have a base-domain, DNS/TLS, primary `Domain`, and host-nginx reverse-proxy path. Generic local Compose apps also have manual and CPU-driven replica controls in Container Ops. WordPress still runs one published container; uploads remain local, and the scaler does not update the site's nginx upstream.
- **Do:** Make WP stateless — shared media (uploads on a shared volume or S3 offload), shared Redis object cache (#23), then nginx `upstream` load balancing across replicas with sticky handling where needed.
- **Reuse:** `nginx_advanced_service`, #23, `storage_provider_service`.
- **Done when:** A site can run N replicas behind a balancer with consistent media + cache.
- **Assessment / why deferred:** **`XL`** *and* infra-blocked. Managed sites are single `wordpress:*-apache` containers published directly on `localhost:PORT` with **no per-site nginx in front** — the very reverse-proxy layer #34's `upstream` load-balancing needs, and the same missing infra that blocks **#21** (preview URLs) and that #8's backend/#25's "nginx" premise also ran into. Beyond the balancer, making WP **stateless** is itself a large build: shared media (an S3/offload plugin or a shared uploads volume across replicas, via `storage_provider_service`), a shared Redis object cache (the **#23** redis sidecar exists but is per-site, not shared across replicas), and replica orchestration in the compose stack. So #34 needs (a) the per-site reverse-proxy/routing layer to exist first (a prerequisite shared with #21), then (b) a stateless-WP refactor, then (c) the `upstream` wiring — a milestone-sized effort gated on infrastructure not yet built. Deferred. No code shipped.
- **Spec (landed):** the epic is now split into [docs/HORIZONTAL_SCALING_SPEC.md](HORIZONTAL_SCALING_SPEC.md). A capability sweep found the hard parts **already exist** — `NginxAdvancedService.create_reverse_proxy` already emits `upstream{}` load-balancing + `limit_req` + `proxy_cache` + per-vhost log reads (`nginx_advanced_service.py:31,47,61,68,173`), `NginxService.create_site` already renders a container-proxying vhost with cert + 80→443 redirect (`nginx_service.py:391,450`), and wildcard TLS / S3 directory upload are built (`advanced_ssl_service.py:39`, `storage_provider_service.py:260`). So #34 is **wiring + a public base-domain + stateless WP**, not greenfield nginx. The spec defines a shared **keystone** (K0–K4: public base-domain → per-site reverse-proxy vhost → auto-TLS → preview URL → per-site access-log/`limit_req`) that *also* closes **#21, #15-TLS, #8-backend, #30-jail, and #25's response-time/slow-pages**, then **#34 proper** (H0–H4: stateless WP via shared media + already-shared #23 Redis → internal-network replicas → upstream LB → rolling/health → optional autoscale). Each phase is a separately shippable + separately verifiable PR, with an explicit "verifiable on Windows vs needs a real Docker host" split so the infra is exercised before it's claimed working. Recommended order: K0→K2 (closes #15/#8/#21) before any #34 work; #34 MVP done at H2.
- **Assessment / why deferred:** the old reverse-proxy blocker is gone. The remaining work is WordPress-specific: shared uploads, an internal replica topology, scaler-to-upstream registration, health-based draining, and live failover verification. The site's Redis service can serve all replicas in one Compose project, but it does not solve uploads or disk caches. Generic `docker compose --scale` support is a control-plane building block, not proof of HA.
- **Verification:** frontend tests pin policy normalization and API calls. Backend tests drive the policy-to-evaluation-to-manual-scale workflow through the API and database. A real Linux/Docker host must still prove request distribution, replica-loss continuity, and cross-replica media consistency before this item can move to done.
### #35 — WordPress workflow nodes, events & CLI `[L]` 🟡 — ✅ Done (WP lifecycle events emitted + subscribable; workflow action-nodes + CLI deferred)
- **Today:** The event/webhook + workflow engine can't see a `WordPressSite` — `EVENT_CATALOG` has no WP lifecycle events, WP operations emit nothing, the workflow builder has no WP node types, and there's no `serverkit wp` CLI.