mirror of
https://github.com/webstudio-is/webstudio.git
synced 2026-10-29 08:07:48 +09:00
Oleg Isonen
12ad74584a
## Summary This improves how coding agents discover, author, verify, and audit Webstudio projects through the CLI and MCP. The work focuses on making agents more reliable with less context, while keeping edits semantic and Builder-editable. ## What changes for users - Agents can verify dynamic bindings after authoring and receive actionable diagnostics when expressions, resources, scopes, or component props do not line up. - Agents can create complete resource-backed runtime UI through a semantic operation instead of coordinating fragile low-level edits. - MCP discovery and command output are compact by default, with pagination and focused verbose detail when an agent needs more context. - Asset, style, token, font, page, and component inspection provides stronger usage context so agents can preserve the existing design system. - Rendered audits provide better evidence for responsive behavior, geometry, contrast, generated-build size, and large-project runs. - Cross-project MCP workflows can run from a manifest with explicit mutation approval, isolated failures, resumable progress, and persistent checkpoints. - Project-specific agent onboarding is clearer and does not add telemetry. - Packaged CLI releases gain stronger smoke verification for install, MCP startup, authoring, preview, screenshots, and audits. ## Reliability and architecture - Runtime operation output schemas are generated from the runtime registry and remain client-compatible. - Project-build exports are consolidated around architectural boundaries instead of feature-by-feature convenience entry points. - HTTP, preview, screenshot, and project-session paths are hardened for longer autonomous workflows. - Craft compatibility work is read-only and provenance-aware; it does not silently opt ordinary projects into Craft behavior. - A minimal-context evaluation harness exercises key workflows with a weaker model and stores bounded, privacy-safe evidence. ## Validation - Rebased on the latest `origin/main`. - `pnpm lint` - Repository-wide `oxfmt --check` - CLI typecheck and full test suite: 773 tests passed. - Project-build typecheck and full test suite: 1,866 tests passed. - Protocol typecheck and full test suite: 83 tests passed. - HTTP client tests: 23 passed. - CSS data tests: 3,620 passed, 1 skipped. - Focused Builder tests: 67 passed. - Package-boundary, generated API, and generated documentation checks passed. - Minimal-context agent evaluations passed for authenticated resource-backed UI and design-system-aware responsive authoring.
2136 lines
65 KiB
TypeScript
2136 lines
65 KiB
TypeScript
import { execFile, spawn, type ChildProcess } from "node:child_process";
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
import {
|
|
cp,
|
|
mkdir,
|
|
mkdtemp,
|
|
readFile,
|
|
readdir,
|
|
rm,
|
|
stat,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { dirname, join } from "node:path";
|
|
import { promisify } from "node:util";
|
|
import { bundleVersion } from "@webstudio-is/protocol";
|
|
import type { Instance } from "@webstudio-is/sdk";
|
|
import type { BuilderState } from "@webstudio-is/project-build/state";
|
|
import type { BuilderPatchTransaction } from "@webstudio-is/project-build/contracts";
|
|
import type { BuilderRuntimeMutation } from "@webstudio-is/project-build/runtime";
|
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
import React from "react";
|
|
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
import {
|
|
compactSmokeOperations,
|
|
countCompactTokens,
|
|
createReleaseSmokeReporter,
|
|
defaultCandidateTokenBudget,
|
|
getModelVerificationItemCount,
|
|
modelEvaluationReportSchema,
|
|
modelEvaluationSubmissionSchema,
|
|
modelEvaluationTaskId,
|
|
publishedCliManifestContractSchema,
|
|
writeModelEvaluationReport,
|
|
type CompactOperationMeasurement,
|
|
type CompactSmokeOperation,
|
|
} from "./release-smoke-report";
|
|
import {
|
|
createRuntimeFixtureBuildSnapshot,
|
|
createRuntimeFixtureSerializedBuild,
|
|
runtimeFixturePermissions,
|
|
startRuntimeFixtureApi,
|
|
} from "./runtime-fixture-api";
|
|
import { runAgentCommand } from "./run-agent-command";
|
|
import { isPlainRecord } from "../src/type-utils";
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
const projectId = "release-smoke-project";
|
|
const buildId = "release-smoke-build";
|
|
const projectVersion = 1;
|
|
const sessionVersion = "cli-project-session-v1";
|
|
const releaseVersionPlaceholder = "0.0.0-webstudio-version";
|
|
const privacyCanary = "releaseSmokePrivateCanaryAuthTokenDoNotReport";
|
|
const duplicatedPage = { name: "About", path: "/about" } as const;
|
|
const createdPageInputs = [
|
|
{ name: "Services", path: "/services" },
|
|
{ name: "Work", path: "/work" },
|
|
{ name: "Blog", path: "/blog" },
|
|
{ name: "Contact", path: "/contact" },
|
|
] as const;
|
|
const additionalPageCount = 1 + createdPageInputs.length;
|
|
const agentSmokeClients = [
|
|
{ connect: "claude", name: "claude-code" },
|
|
{ connect: "codex", name: "codex" },
|
|
{ connect: "cursor", name: "cursor" },
|
|
{ connect: "vscode", name: "vscode" },
|
|
] as const;
|
|
|
|
const configuredTokenBudget = Number(
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_TOKEN_BUDGET ??
|
|
defaultCandidateTokenBudget
|
|
);
|
|
const reporter = createReleaseSmokeReporter({
|
|
privacyCanary,
|
|
candidateTokenBudget: configuredTokenBudget,
|
|
});
|
|
const reportPhase = reporter.complete;
|
|
|
|
type ReleaseTarget = {
|
|
source: "local" | "registry";
|
|
version: string;
|
|
installSpec: string;
|
|
integrity?: string;
|
|
};
|
|
|
|
const readNpmJson = async (args: string[]) =>
|
|
JSON.parse((await execFileAsync("npm", args)).stdout) as unknown;
|
|
|
|
const resolveRegistryTarget = async (spec: string): Promise<ReleaseTarget> => {
|
|
const version = await readNpmJson(["view", spec, "version", "--json"]);
|
|
const integrity = await readNpmJson([
|
|
"view",
|
|
spec,
|
|
"dist.integrity",
|
|
"--json",
|
|
]);
|
|
if (typeof version !== "string" || version.length === 0) {
|
|
throw new Error(`Could not resolve registry package ${spec}.`);
|
|
}
|
|
if (typeof integrity !== "string" || integrity.length === 0) {
|
|
throw new Error(`Registry package webstudio@${version} has no integrity.`);
|
|
}
|
|
return {
|
|
source: "registry",
|
|
version,
|
|
installSpec: `webstudio@${version}`,
|
|
integrity,
|
|
};
|
|
};
|
|
|
|
const installReleaseTarget = async ({
|
|
directory,
|
|
target,
|
|
enforceArtifactContract = false,
|
|
}: {
|
|
directory: string;
|
|
target: ReleaseTarget;
|
|
enforceArtifactContract?: boolean;
|
|
}) => {
|
|
await writeFile(
|
|
join(directory, "package.json"),
|
|
JSON.stringify({ private: true }),
|
|
"utf8"
|
|
);
|
|
const install = await execFileAsync(
|
|
"npm",
|
|
[
|
|
"install",
|
|
target.installSpec,
|
|
"--no-audit",
|
|
"--no-fund",
|
|
"--loglevel=warn",
|
|
],
|
|
{ cwd: directory, maxBuffer: 10 * 1024 * 1024 }
|
|
);
|
|
if (/ERESOLVE|overriding peer dependency/i.test(install.stderr)) {
|
|
throw new Error(
|
|
`npm install emitted peer dependency warnings for ${target.source} ${target.version}.`
|
|
);
|
|
}
|
|
const installedManifest = JSON.parse(
|
|
await readFile(
|
|
join(directory, "node_modules", "webstudio", "package.json"),
|
|
"utf8"
|
|
)
|
|
) as Record<string, unknown>;
|
|
if (installedManifest.version !== target.version) {
|
|
throw new Error(
|
|
`Clean install resolved webstudio ${String(installedManifest.version)} instead of ${target.version}.`
|
|
);
|
|
}
|
|
if (enforceArtifactContract) {
|
|
publishedCliManifestContractSchema.parse({
|
|
name: installedManifest.name,
|
|
version: installedManifest.version,
|
|
type: installedManifest.type,
|
|
bin: installedManifest.bin,
|
|
files: installedManifest.files,
|
|
engines: installedManifest.engines,
|
|
});
|
|
const packageDirectory = join(directory, "node_modules", "webstudio");
|
|
const packageEntries = new Set(await readdir(packageDirectory));
|
|
for (const requiredEntry of [
|
|
"bin.js",
|
|
"lib",
|
|
"package.json",
|
|
"templates",
|
|
]) {
|
|
if (packageEntries.has(requiredEntry) === false) {
|
|
throw new Error(
|
|
`Installed webstudio artifact is missing required entry ${requiredEntry}.`
|
|
);
|
|
}
|
|
}
|
|
const allowedTopLevelEntries = new Set([
|
|
"LICENSE",
|
|
"README.md",
|
|
"bin.js",
|
|
"lib",
|
|
"package.json",
|
|
"templates",
|
|
]);
|
|
const unexpectedEntry = Array.from(packageEntries).find(
|
|
(entry) => allowedTopLevelEntries.has(entry) === false
|
|
);
|
|
if (unexpectedEntry !== undefined) {
|
|
throw new Error(
|
|
`Installed webstudio artifact contains unexpected top-level entry ${unexpectedEntry}.`
|
|
);
|
|
}
|
|
const scanTree = async (path: string, relativePath = "") => {
|
|
for (const entry of await readdir(path, { withFileTypes: true })) {
|
|
const entryPath = join(path, entry.name);
|
|
const relativeEntryPath = join(relativePath, entry.name);
|
|
if (entry.isSymbolicLink()) {
|
|
throw new Error(
|
|
`Installed webstudio artifact contains symlink ${relativeEntryPath}.`
|
|
);
|
|
}
|
|
if (
|
|
/(?:^|\/)(?:\.env(?:\.|$)|\.npmrc$)|\.(?:map|test\.[^.]+|stories\.[^.]+)$/i.test(
|
|
relativeEntryPath
|
|
)
|
|
) {
|
|
throw new Error(
|
|
`Installed webstudio artifact contains forbidden file ${relativeEntryPath}.`
|
|
);
|
|
}
|
|
if (entry.isDirectory()) {
|
|
await scanTree(entryPath, relativeEntryPath);
|
|
}
|
|
}
|
|
};
|
|
await scanTree(packageDirectory);
|
|
const executable = await stat(join(packageDirectory, "bin.js"));
|
|
if (executable.isFile() === false || (executable.mode & 0o111) === 0) {
|
|
throw new Error("Installed webstudio bin.js is not executable.");
|
|
}
|
|
}
|
|
if (target.integrity !== undefined) {
|
|
const lock = JSON.parse(
|
|
await readFile(join(directory, "package-lock.json"), "utf8")
|
|
) as { packages?: Record<string, { integrity?: unknown }> };
|
|
const installedIntegrity =
|
|
lock.packages?.["node_modules/webstudio"]?.integrity;
|
|
if (installedIntegrity !== target.integrity) {
|
|
throw new Error(
|
|
`Installed webstudio integrity ${String(installedIntegrity)} does not match registry metadata.`
|
|
);
|
|
}
|
|
}
|
|
};
|
|
|
|
const runAgentWorkflowStep = async <Result>({
|
|
step,
|
|
recovery,
|
|
run,
|
|
}: {
|
|
step: string;
|
|
recovery: string;
|
|
run: () => Promise<Result>;
|
|
}) => {
|
|
try {
|
|
return await run();
|
|
} catch (error) {
|
|
throw new Error(
|
|
`Agent connection workflow stopped at "${step}": ${error instanceof Error ? error.message : String(error)} Recovery: ${recovery}`,
|
|
{ cause: error }
|
|
);
|
|
}
|
|
};
|
|
|
|
const replaceReleaseVersion = async (
|
|
directory: string,
|
|
releaseVersion: string,
|
|
workspaceVersions: ReadonlyMap<string, string>
|
|
) => {
|
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
await Promise.all(
|
|
entries.map(async (entry) => {
|
|
const entryPath = join(directory, entry.name);
|
|
if (entry.isDirectory()) {
|
|
await replaceReleaseVersion(
|
|
entryPath,
|
|
releaseVersion,
|
|
workspaceVersions
|
|
);
|
|
return;
|
|
}
|
|
if (/\.(?:js|mjs|json)$/.test(entry.name) === false) {
|
|
return;
|
|
}
|
|
const source = await readFile(entryPath, "utf8");
|
|
let releaseSource = source.replaceAll(
|
|
releaseVersionPlaceholder,
|
|
releaseVersion
|
|
);
|
|
if (entry.name === "package.json") {
|
|
const manifest = JSON.parse(releaseSource) as Record<string, unknown>;
|
|
for (const field of [
|
|
"dependencies",
|
|
"devDependencies",
|
|
"optionalDependencies",
|
|
"peerDependencies",
|
|
]) {
|
|
const dependencies = manifest[field];
|
|
if (
|
|
typeof dependencies !== "object" ||
|
|
dependencies === null ||
|
|
Array.isArray(dependencies)
|
|
) {
|
|
continue;
|
|
}
|
|
const dependencyVersions = dependencies as Record<string, unknown>;
|
|
for (const [name, version] of Object.entries(dependencyVersions)) {
|
|
if (
|
|
typeof version === "string" &&
|
|
version.startsWith("workspace:")
|
|
) {
|
|
const workspaceVersion = workspaceVersions.get(name);
|
|
if (workspaceVersion === undefined) {
|
|
throw new Error(
|
|
`Cannot resolve workspace dependency ${name} in ${entryPath}.`
|
|
);
|
|
}
|
|
dependencyVersions[name] = workspaceVersion;
|
|
}
|
|
}
|
|
}
|
|
releaseSource = JSON.stringify(manifest, undefined, 2);
|
|
}
|
|
if (releaseSource !== source) {
|
|
await writeFile(entryPath, releaseSource, "utf8");
|
|
}
|
|
})
|
|
);
|
|
};
|
|
|
|
const packReleaseShapedCli = async ({
|
|
stagingDirectory,
|
|
packDirectory,
|
|
releaseVersion,
|
|
}: {
|
|
stagingDirectory: string;
|
|
packDirectory: string;
|
|
releaseVersion: string;
|
|
}) => {
|
|
const cliDirectory = join(import.meta.dirname, "..");
|
|
await Promise.all(
|
|
["bin.js", "package.json", "lib", "templates"].map((entry) =>
|
|
cp(join(cliDirectory, entry), join(stagingDirectory, entry), {
|
|
recursive: true,
|
|
})
|
|
)
|
|
);
|
|
if (typeof releaseVersion !== "string" || releaseVersion.length === 0) {
|
|
throw new Error("Could not resolve a published Webstudio release version.");
|
|
}
|
|
const packagesDirectory = join(import.meta.dirname, "../..");
|
|
const workspaceVersions = new Map<string, string>();
|
|
for (const entry of await readdir(packagesDirectory, {
|
|
withFileTypes: true,
|
|
})) {
|
|
if (entry.isDirectory() === false) {
|
|
continue;
|
|
}
|
|
try {
|
|
const manifest = JSON.parse(
|
|
await readFile(
|
|
join(packagesDirectory, entry.name, "package.json"),
|
|
"utf8"
|
|
)
|
|
) as { name?: string; version?: string };
|
|
if (manifest.name !== undefined && manifest.version !== undefined) {
|
|
workspaceVersions.set(
|
|
manifest.name,
|
|
manifest.version.replace(releaseVersionPlaceholder, releaseVersion)
|
|
);
|
|
}
|
|
} catch (error) {
|
|
if (
|
|
error instanceof Error &&
|
|
"code" in error &&
|
|
error.code === "ENOENT"
|
|
) {
|
|
continue;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
await replaceReleaseVersion(
|
|
stagingDirectory,
|
|
releaseVersion,
|
|
workspaceVersions
|
|
);
|
|
await execFileAsync("npm", ["pack", "--pack-destination", packDirectory], {
|
|
cwd: stagingDirectory,
|
|
});
|
|
return releaseVersion;
|
|
};
|
|
|
|
const createFixture = () => {
|
|
const assets = [
|
|
{
|
|
id: "asset-hero",
|
|
projectId,
|
|
name: "release-hero.png",
|
|
filename: "release-hero.png",
|
|
type: "image" as const,
|
|
size: 128,
|
|
format: "png",
|
|
createdAt: "2026-01-01T00:00:00.000Z",
|
|
description: "Release smoke hero",
|
|
meta: { width: 1200, height: 800 },
|
|
},
|
|
];
|
|
const basePages = [
|
|
{
|
|
id: "home",
|
|
name: "Home",
|
|
title: privacyCanary,
|
|
path: "",
|
|
rootInstanceId: "home-root",
|
|
meta: {},
|
|
},
|
|
{
|
|
id: "pricing",
|
|
name: "Pricing",
|
|
title: "Pricing",
|
|
path: "/pricing",
|
|
rootInstanceId: "pricing-root",
|
|
meta: {},
|
|
},
|
|
];
|
|
const extraPageCount =
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_LARGE_AUDIT === "1" ? 114 : 0;
|
|
const pages = [
|
|
...basePages,
|
|
...Array.from({ length: extraPageCount }, (_, index) => {
|
|
const number = index + 3;
|
|
return {
|
|
id: `audit-${number}`,
|
|
name: `Audit ${number}`,
|
|
title: `Audit ${number}`,
|
|
path: `/audit-${number}`,
|
|
rootInstanceId: `audit-${number}-root`,
|
|
meta: {},
|
|
};
|
|
}),
|
|
];
|
|
const persistedPages = {
|
|
meta: { siteName: "Release Smoke", contactEmail: "" },
|
|
compiler: { atomicStyles: true },
|
|
redirects: [],
|
|
homePageId: "home",
|
|
rootFolderId: "root-folder",
|
|
pages,
|
|
folders: [
|
|
{
|
|
id: "root-folder",
|
|
name: "Root",
|
|
slug: "",
|
|
children: pages.map((page) => page.id),
|
|
},
|
|
],
|
|
};
|
|
const instances: Array<[string, Instance]> = pages.map((page) => [
|
|
page.rootInstanceId,
|
|
{
|
|
type: "instance",
|
|
id: page.rootInstanceId,
|
|
component: "Box",
|
|
children:
|
|
page.id === "home"
|
|
? [
|
|
{ type: "text", value: `Release smoke ${page.name}` },
|
|
{ type: "id", value: "home-image" },
|
|
]
|
|
: [{ type: "text", value: `Release smoke ${page.name}` }],
|
|
},
|
|
]);
|
|
instances.push([
|
|
"home-image",
|
|
{
|
|
type: "instance",
|
|
id: "home-image",
|
|
component: "Image",
|
|
children: [],
|
|
},
|
|
]);
|
|
const props = [
|
|
{
|
|
id: "home-image-src",
|
|
instanceId: "home-image",
|
|
name: "src",
|
|
type: "asset" as const,
|
|
value: "asset-hero",
|
|
},
|
|
];
|
|
const build = {
|
|
id: buildId,
|
|
projectId,
|
|
version: projectVersion,
|
|
createdAt: "2026-01-01T00:00:00.000Z",
|
|
updatedAt: "2026-01-01T00:00:00.000Z",
|
|
pages: persistedPages,
|
|
props: props.map((prop) => [prop.id, prop]) as never,
|
|
instances: instances as never,
|
|
dataSources: [],
|
|
resources: [],
|
|
styleSources: [],
|
|
styleSourceSelections: [],
|
|
styles: [],
|
|
breakpoints: [],
|
|
projectSettings: { meta: {}, compiler: {} },
|
|
};
|
|
return {
|
|
data: {
|
|
bundleVersion,
|
|
origin: "https://assets.example",
|
|
projectDomain: "example.com",
|
|
projectTitle: "Release Smoke",
|
|
page: pages[0],
|
|
pages,
|
|
assets,
|
|
build,
|
|
},
|
|
build,
|
|
persistedPages,
|
|
instances,
|
|
assets,
|
|
props,
|
|
};
|
|
};
|
|
|
|
const waitForUrl = async (url: string, child: ChildProcess) => {
|
|
const deadline = Date.now() + 60_000;
|
|
while (Date.now() < deadline) {
|
|
if (child.exitCode !== null) {
|
|
throw new Error(`Preview exited with code ${child.exitCode}.`);
|
|
}
|
|
try {
|
|
const response = await fetch(url);
|
|
if (response.ok) {
|
|
return;
|
|
}
|
|
} catch {}
|
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
}
|
|
throw new Error(`Preview did not become ready at ${url}.`);
|
|
};
|
|
|
|
const stopProcessTree = async (child: ChildProcess | undefined) => {
|
|
if (child?.pid === undefined) {
|
|
return;
|
|
}
|
|
const pid = child.pid;
|
|
const usesProcessGroup = process.platform !== "win32";
|
|
const signal = (value: NodeJS.Signals) => {
|
|
try {
|
|
if (usesProcessGroup) {
|
|
process.kill(-pid, value);
|
|
} else {
|
|
child.kill(value);
|
|
}
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== "ESRCH") {
|
|
throw error;
|
|
}
|
|
}
|
|
};
|
|
const isRunning = () => {
|
|
try {
|
|
process.kill(usesProcessGroup ? -pid : pid, 0);
|
|
return true;
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code === "ESRCH") {
|
|
return false;
|
|
}
|
|
throw error;
|
|
}
|
|
};
|
|
const waitUntilStopped = async (timeoutMs: number) => {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (isRunning() && Date.now() < deadline) {
|
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
}
|
|
return isRunning() === false;
|
|
};
|
|
|
|
signal("SIGTERM");
|
|
if (await waitUntilStopped(5_000)) {
|
|
return;
|
|
}
|
|
signal("SIGKILL");
|
|
if ((await waitUntilStopped(5_000)) === false) {
|
|
throw new Error(`Preview process group ${pid} did not stop.`);
|
|
}
|
|
};
|
|
|
|
const startFixtureApi = async (
|
|
fixture: ReturnType<typeof createFixture>,
|
|
{
|
|
initialState,
|
|
executeRuntime,
|
|
applyTransactions,
|
|
}: {
|
|
initialState: BuilderState;
|
|
executeRuntime: (input: {
|
|
id: string;
|
|
state: BuilderState;
|
|
input: unknown;
|
|
context: {
|
|
createId: () => string;
|
|
projectId: string;
|
|
projectVersion: number;
|
|
};
|
|
}) => unknown | Promise<unknown>;
|
|
applyTransactions: (
|
|
state: BuilderState,
|
|
transactions: readonly BuilderPatchTransaction[]
|
|
) => { state: BuilderState };
|
|
}
|
|
) => {
|
|
let version = projectVersion;
|
|
let state = initialState;
|
|
let generatedId = 0;
|
|
const getCurrentBuildSnapshot = () =>
|
|
createRuntimeFixtureBuildSnapshot({
|
|
state,
|
|
projectId,
|
|
buildId,
|
|
version,
|
|
});
|
|
const getCurrentSerializedBuild = () =>
|
|
createRuntimeFixtureSerializedBuild({
|
|
state,
|
|
baseBuild: fixture.build,
|
|
buildId,
|
|
version,
|
|
});
|
|
return startRuntimeFixtureApi(
|
|
async ({ pathname, response, operationPath, readInput }) => {
|
|
if (pathname.endsWith("/release-hero.png")) {
|
|
response.writeHead(200, { "content-type": "image/png" });
|
|
response.end(
|
|
Buffer.from(
|
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
|
|
"base64"
|
|
)
|
|
);
|
|
return undefined;
|
|
}
|
|
let data: unknown;
|
|
if (operationPath === "build.loadProjectBundleByProjectId") {
|
|
data = { ...fixture.data, build: getCurrentSerializedBuild() };
|
|
} else if (operationPath === "projects.get") {
|
|
data = {
|
|
id: projectId,
|
|
name: "Release smoke project",
|
|
createdAt: "2026-01-01T00:00:00.000Z",
|
|
updatedAt: "2026-01-01T00:00:00.000Z",
|
|
buildId,
|
|
version,
|
|
homePageId: "home",
|
|
features: {},
|
|
};
|
|
} else if (operationPath === "publish.create") {
|
|
data = { jobId: "release-smoke-staging-job" };
|
|
} else if (operationPath === "build.patch") {
|
|
const input = (await readInput()) as {
|
|
transactions?: BuilderPatchTransaction[];
|
|
};
|
|
state = applyTransactions(state, input.transactions ?? []).state;
|
|
version += 1;
|
|
data = { version };
|
|
} else if (operationPath === "build.get") {
|
|
data = getCurrentBuildSnapshot();
|
|
} else if (
|
|
operationPath !== "projects.permissions" &&
|
|
operationPath !== ""
|
|
) {
|
|
const input = await readInput();
|
|
const result = await executeRuntime({
|
|
id: operationPath,
|
|
state,
|
|
input,
|
|
context: {
|
|
createId: () => `fixture-${generatedId++}`,
|
|
projectId,
|
|
projectVersion: version,
|
|
},
|
|
});
|
|
if (
|
|
typeof result === "object" &&
|
|
result !== null &&
|
|
"kind" in result &&
|
|
(result as BuilderRuntimeMutation).kind === "mutation"
|
|
) {
|
|
const mutation = result as BuilderRuntimeMutation;
|
|
state = applyTransactions(state, [
|
|
{
|
|
id: `fixture-transaction-${generatedId++}`,
|
|
payload: mutation.payload,
|
|
},
|
|
]).state;
|
|
version += 1;
|
|
data = mutation.result;
|
|
} else {
|
|
data = result;
|
|
}
|
|
} else {
|
|
data = runtimeFixturePermissions;
|
|
}
|
|
return data;
|
|
}
|
|
);
|
|
};
|
|
|
|
const assertJsonStdout = async (stdout: string, label: string) => {
|
|
try {
|
|
JSON.parse(stdout);
|
|
} catch (error) {
|
|
throw new Error(`${label} did not print valid JSON.`, { cause: error });
|
|
}
|
|
};
|
|
|
|
const getRecord = (value: unknown, label: string) => {
|
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
throw new Error(`${label} is not an object.`);
|
|
}
|
|
return value as Record<string, unknown>;
|
|
};
|
|
|
|
const createMcpSmokeClient = ({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
name,
|
|
}: {
|
|
bin: string;
|
|
cwd: string;
|
|
env: NodeJS.ProcessEnv;
|
|
name: string;
|
|
}) => {
|
|
const transport = new StdioClientTransport({
|
|
command: bin,
|
|
args: ["mcp"],
|
|
cwd,
|
|
env: Object.fromEntries(
|
|
Object.entries(env).filter(
|
|
(entry): entry is [string, string] => typeof entry[1] === "string"
|
|
)
|
|
),
|
|
stderr: "pipe",
|
|
});
|
|
const client = new Client({ name, version: "1.0.0" });
|
|
return { client, transport };
|
|
};
|
|
|
|
const emptyFallbackCounts = () => ({
|
|
verbose: 0,
|
|
fullResource: 0,
|
|
source: 0,
|
|
});
|
|
|
|
const runCompactMeasurementSuite = async ({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
expectedDetail,
|
|
}: {
|
|
bin: string;
|
|
cwd: string;
|
|
env: NodeJS.ProcessEnv;
|
|
expectedDetail: "compact" | "verbose";
|
|
}) => {
|
|
const { client, transport } = createMcpSmokeClient({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
name: "webstudio-release-smoke-compact-measurement",
|
|
});
|
|
const measurements = new Map<
|
|
CompactSmokeOperation,
|
|
CompactOperationMeasurement
|
|
>();
|
|
const contractFailures: CompactSmokeOperation[] = [];
|
|
let canarySeen = false;
|
|
await client.connect(transport);
|
|
try {
|
|
for (const operation of compactSmokeOperations) {
|
|
let attempts = 0;
|
|
let failures = 0;
|
|
let bytes = 0;
|
|
let tokens = 0;
|
|
const fallbacks = emptyFallbackCounts();
|
|
let passed = false;
|
|
for (let attempt = 0; attempt < 2 && passed === false; attempt += 1) {
|
|
attempts += 1;
|
|
try {
|
|
const result = await client.callTool({
|
|
name: operation,
|
|
arguments: {
|
|
limit: 25,
|
|
...(expectedDetail === "verbose" ? { verbose: true } : {}),
|
|
},
|
|
});
|
|
const source = JSON.stringify(result.structuredContent);
|
|
canarySeen ||= source.includes(privacyCanary);
|
|
const structured = getRecord(
|
|
result.structuredContent,
|
|
`${operation} compact structured content`
|
|
);
|
|
const data = getRecord(
|
|
structured.data,
|
|
`${operation} compact response data`
|
|
);
|
|
if (result.isError === true || structured.ok !== true) {
|
|
throw new Error(`${expectedDetail} operation failed`);
|
|
}
|
|
bytes = Buffer.byteLength(source, "utf8");
|
|
tokens = countCompactTokens(source);
|
|
if (data.detail !== expectedDetail) {
|
|
contractFailures.push(operation);
|
|
if (data.detail === "verbose") {
|
|
fallbacks.verbose += 1;
|
|
}
|
|
}
|
|
passed = true;
|
|
} catch {
|
|
failures += 1;
|
|
}
|
|
}
|
|
measurements.set(operation, {
|
|
bytes,
|
|
tokens,
|
|
attempts,
|
|
retries: Math.max(0, attempts - 1),
|
|
failures,
|
|
fallbacks,
|
|
});
|
|
if (passed === false) {
|
|
contractFailures.push(operation);
|
|
}
|
|
}
|
|
} finally {
|
|
await client.close();
|
|
}
|
|
return { measurements, contractFailures, canarySeen };
|
|
};
|
|
|
|
const verifyModelEvaluationOutcome = async ({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
}: {
|
|
bin: string;
|
|
cwd: string;
|
|
env: NodeJS.ProcessEnv;
|
|
}) => {
|
|
const { client, transport } = createMcpSmokeClient({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
name: "webstudio-release-smoke-model-verifier",
|
|
});
|
|
await client.connect(transport);
|
|
try {
|
|
const countItems = async (operation: "list-pages" | "list-assets") => {
|
|
let result = await client.callTool({
|
|
name: operation,
|
|
arguments: { limit: 25 },
|
|
});
|
|
if (result.isError === true) {
|
|
result = await client.callTool({ name: operation, arguments: {} });
|
|
}
|
|
const structured = getRecord(
|
|
result.structuredContent,
|
|
`${operation} model evaluation verification response`
|
|
);
|
|
if (result.isError === true || structured.ok === false) {
|
|
throw new Error(`${operation} model evaluation verification failed.`);
|
|
}
|
|
return getModelVerificationItemCount(operation, structured);
|
|
};
|
|
return {
|
|
pageCount: await countItems("list-pages"),
|
|
assetCount: await countItems("list-assets"),
|
|
};
|
|
} finally {
|
|
await client.close();
|
|
}
|
|
};
|
|
|
|
const runSingleModelEvaluation = async ({
|
|
targetName,
|
|
command,
|
|
subject,
|
|
bin,
|
|
projectDirectory,
|
|
configDirectory,
|
|
temp,
|
|
}: {
|
|
targetName: "candidate" | "baseline";
|
|
command: string;
|
|
subject: ReleaseTarget;
|
|
bin: string;
|
|
projectDirectory: string;
|
|
configDirectory: string;
|
|
temp: string;
|
|
}) => {
|
|
const evaluationDirectory = join(
|
|
temp,
|
|
`model-evaluation-${targetName}-project`
|
|
);
|
|
const evaluationConfigDirectory = join(
|
|
temp,
|
|
`model-evaluation-${targetName}-config`
|
|
);
|
|
const taskPath = join(temp, `model-evaluation-${targetName}-task.json`);
|
|
const submissionPath = join(
|
|
temp,
|
|
`model-evaluation-${targetName}-submission.json`
|
|
);
|
|
await Promise.all([
|
|
cp(projectDirectory, evaluationDirectory, { recursive: true }),
|
|
cp(configDirectory, evaluationConfigDirectory, { recursive: true }),
|
|
]);
|
|
const task = {
|
|
schemaVersion: 2,
|
|
taskId: modelEvaluationTaskId,
|
|
objective:
|
|
"Using only the packaged Webstudio MCP server, determine the exact number of pages and assets in the linked project. Choose the minimal tool sequence yourself.",
|
|
mcp: { command: bin, args: ["mcp"], cwd: evaluationDirectory },
|
|
resultPath: submissionPath,
|
|
resultContract: {
|
|
schemaVersion: 2,
|
|
taskId: modelEvaluationTaskId,
|
|
provider: "agent provider identifier",
|
|
model: "agent model identifier",
|
|
outcome: "completed or failed",
|
|
toolCallCount: "non-negative integer",
|
|
observations: {
|
|
pageCount: "non-negative integer",
|
|
assetCount: "non-negative integer",
|
|
},
|
|
},
|
|
};
|
|
await writeFile(taskPath, JSON.stringify(task, undefined, 2), "utf8");
|
|
const evaluationEnv = {
|
|
...process.env,
|
|
WEBSTUDIO_CONFIG_DIR: evaluationConfigDirectory,
|
|
WEBSTUDIO_RELEASE_SMOKE_AGENT_TASK: taskPath,
|
|
WEBSTUDIO_RELEASE_SMOKE_AGENT_RESULT: submissionPath,
|
|
WEBSTUDIO_RELEASE_SMOKE_MCP_COMMAND: bin,
|
|
WEBSTUDIO_RELEASE_SMOKE_MCP_CWD: evaluationDirectory,
|
|
WEBSTUDIO_RELEASE_SMOKE_MODEL_TARGET: targetName,
|
|
};
|
|
const execution = await runAgentCommand({
|
|
command,
|
|
cwd: evaluationDirectory,
|
|
env: evaluationEnv,
|
|
timeoutMs: 10 * 60_000,
|
|
});
|
|
const submission = modelEvaluationSubmissionSchema.parse(
|
|
JSON.parse(await readFile(submissionPath, "utf8"))
|
|
);
|
|
const verifiedObservations = await verifyModelEvaluationOutcome({
|
|
bin,
|
|
cwd: evaluationDirectory,
|
|
env: evaluationEnv,
|
|
});
|
|
const projectOutcome =
|
|
submission.observations.pageCount === verifiedObservations.pageCount &&
|
|
submission.observations.assetCount === verifiedObservations.assetCount;
|
|
const passed =
|
|
execution.exitCode === 0 &&
|
|
submission.outcome === "completed" &&
|
|
projectOutcome;
|
|
return {
|
|
passed,
|
|
run: {
|
|
subject: { source: subject.source, version: subject.version },
|
|
evidence: {
|
|
source: "configured-agent-command" as const,
|
|
provider: submission.provider,
|
|
model: submission.model,
|
|
commandSha256: createHash("sha256").update(command).digest("hex"),
|
|
durationMs: execution.durationMs,
|
|
exitCode: execution.exitCode,
|
|
toolCallCount: submission.toolCallCount,
|
|
},
|
|
observations: submission.observations,
|
|
checks: {
|
|
submissionContract: "passed" as const,
|
|
projectOutcome: projectOutcome
|
|
? ("passed" as const)
|
|
: ("failed" as const),
|
|
},
|
|
},
|
|
};
|
|
};
|
|
|
|
const runModelEvaluation = async ({
|
|
command,
|
|
reportPath,
|
|
candidate,
|
|
baseline,
|
|
candidateBin,
|
|
baselineBin,
|
|
candidateProjectDirectory,
|
|
baselineProjectDirectory,
|
|
candidateConfigDirectory,
|
|
baselineConfigDirectory,
|
|
temp,
|
|
}: {
|
|
command: string;
|
|
reportPath: string;
|
|
candidate: ReleaseTarget;
|
|
baseline: ReleaseTarget;
|
|
candidateBin: string;
|
|
baselineBin: string;
|
|
candidateProjectDirectory: string;
|
|
baselineProjectDirectory: string;
|
|
candidateConfigDirectory: string;
|
|
baselineConfigDirectory: string;
|
|
temp: string;
|
|
}) => {
|
|
const results = await Promise.allSettled([
|
|
runSingleModelEvaluation({
|
|
targetName: "candidate",
|
|
command,
|
|
subject: candidate,
|
|
bin: candidateBin,
|
|
projectDirectory: candidateProjectDirectory,
|
|
configDirectory: candidateConfigDirectory,
|
|
temp,
|
|
}),
|
|
runSingleModelEvaluation({
|
|
targetName: "baseline",
|
|
command,
|
|
subject: baseline,
|
|
bin: baselineBin,
|
|
projectDirectory: baselineProjectDirectory,
|
|
configDirectory: baselineConfigDirectory,
|
|
temp,
|
|
}),
|
|
]);
|
|
const rejected = results.find(
|
|
(result): result is PromiseRejectedResult => result.status === "rejected"
|
|
);
|
|
if (rejected !== undefined) {
|
|
throw rejected.reason;
|
|
}
|
|
const [candidateResult, baselineResult] = results.map(
|
|
(result) =>
|
|
(
|
|
result as PromiseFulfilledResult<
|
|
Awaited<ReturnType<typeof runSingleModelEvaluation>>
|
|
>
|
|
).value
|
|
);
|
|
if (candidateResult === undefined || baselineResult === undefined) {
|
|
throw new Error(
|
|
"Both packaged CLI model evaluations must produce evidence."
|
|
);
|
|
}
|
|
const passed = candidateResult.passed && baselineResult.passed;
|
|
const report = modelEvaluationReportSchema.parse({
|
|
schemaVersion: 2,
|
|
kind: "model-driven-packaged-cli-evaluation",
|
|
outcome: passed ? "passed" : "failed",
|
|
subjects: {
|
|
candidate: { source: candidate.source, version: candidate.version },
|
|
baseline: { source: baseline.source, version: baseline.version },
|
|
},
|
|
taskId: modelEvaluationTaskId,
|
|
runs: {
|
|
candidate: candidateResult.run,
|
|
baseline: baselineResult.run,
|
|
},
|
|
checks: {
|
|
bothOutcomes: passed ? "passed" : "failed",
|
|
canaryRedaction: "passed",
|
|
},
|
|
});
|
|
await writeModelEvaluationReport(reportPath, report, [privacyCanary]);
|
|
if (passed === false) {
|
|
throw new Error(
|
|
"Configured model-driven evaluation did not pass against both packaged CLIs."
|
|
);
|
|
}
|
|
};
|
|
|
|
const getRenderedNetworkTransferBytes = (manifest: Record<string, unknown>) =>
|
|
(Array.isArray(manifest.checks) ? manifest.checks : []).reduce(
|
|
(total, check) => {
|
|
if (
|
|
isPlainRecord(check) === false ||
|
|
isPlainRecord(check.layout) === false
|
|
) {
|
|
return total;
|
|
}
|
|
return (
|
|
total +
|
|
(Array.isArray(check.layout.resources)
|
|
? check.layout.resources
|
|
: []
|
|
).reduce(
|
|
(resourceTotal, resource) =>
|
|
resourceTotal +
|
|
(isPlainRecord(resource) &&
|
|
typeof resource.transferSize === "number"
|
|
? resource.transferSize
|
|
: 0),
|
|
0
|
|
)
|
|
);
|
|
},
|
|
0
|
|
);
|
|
|
|
const getRetainedArtifactBytes = async ({
|
|
manifest,
|
|
manifestPath,
|
|
projectDirectory,
|
|
}: {
|
|
manifest: Record<string, unknown>;
|
|
manifestPath: string;
|
|
projectDirectory: string;
|
|
}) => {
|
|
const artifactPaths = new Set([manifestPath]);
|
|
for (const collection of [manifest.checks, manifest.failures]) {
|
|
if (Array.isArray(collection) === false) {
|
|
continue;
|
|
}
|
|
for (const item of collection) {
|
|
if (isPlainRecord(item) && typeof item.screenshotPath === "string") {
|
|
artifactPaths.add(
|
|
item.screenshotPath.startsWith("/")
|
|
? item.screenshotPath
|
|
: join(projectDirectory, item.screenshotPath)
|
|
);
|
|
}
|
|
}
|
|
}
|
|
const sizes = await Promise.all(
|
|
Array.from(artifactPaths, async (path) =>
|
|
stat(path)
|
|
.then((file) => file.size)
|
|
.catch(() => 0)
|
|
)
|
|
);
|
|
return sizes.reduce((total, size) => total + size, 0);
|
|
};
|
|
|
|
const assertScreenshotEvidence = async ({
|
|
payload,
|
|
label,
|
|
path,
|
|
status,
|
|
viewport,
|
|
projectDirectory,
|
|
generatedSite = true,
|
|
origin = "http://127.0.0.1:5190",
|
|
}: {
|
|
payload: Record<string, unknown>;
|
|
label: string;
|
|
path: string;
|
|
status: number;
|
|
viewport: { width: number; height: number };
|
|
projectDirectory: string;
|
|
generatedSite?: boolean;
|
|
origin?: string;
|
|
}) => {
|
|
const data = getRecord(payload.data, `${label} data`);
|
|
const navigation = getRecord(data.navigation, `${label} navigation`);
|
|
const finalUrl = new URL(String(navigation.finalUrl));
|
|
if (
|
|
finalUrl.origin !== origin ||
|
|
finalUrl.pathname !== path ||
|
|
navigation.status !== status ||
|
|
(generatedSite && navigation.generatedSiteRootPresent !== true)
|
|
) {
|
|
throw new Error(
|
|
`${label} returned unexpected navigation evidence: ${JSON.stringify(navigation)}`
|
|
);
|
|
}
|
|
const actualViewport = getRecord(data.viewport, `${label} viewport`);
|
|
if (
|
|
actualViewport.width !== viewport.width ||
|
|
actualViewport.height !== viewport.height
|
|
) {
|
|
throw new Error(
|
|
`${label} returned unexpected viewport: ${JSON.stringify(actualViewport)}`
|
|
);
|
|
}
|
|
const output = String(data.output);
|
|
const screenshot = await readFile(
|
|
output.startsWith("/") ? output : join(projectDirectory, output)
|
|
);
|
|
const isPng = screenshot
|
|
.subarray(0, 8)
|
|
.equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]));
|
|
const isWebp =
|
|
screenshot.subarray(0, 4).toString("ascii") === "RIFF" &&
|
|
screenshot.subarray(8, 12).toString("ascii") === "WEBP";
|
|
if (screenshot.length === 0 || (isPng === false && isWebp === false)) {
|
|
throw new Error(`${label} did not create a valid PNG or WebP artifact.`);
|
|
}
|
|
};
|
|
|
|
const run = async () => {
|
|
Object.assign(globalThis, {
|
|
React,
|
|
});
|
|
const [pagesModule, state, namespaces, projectSession, runtimeRegistry] =
|
|
await Promise.all([
|
|
import("@webstudio-is/project-migrations/pages"),
|
|
import("@webstudio-is/project-build/state"),
|
|
import("@webstudio-is/project-build/contracts"),
|
|
import("@webstudio-is/project-build/project-session"),
|
|
import("@webstudio-is/project-build/runtime"),
|
|
]);
|
|
const temp = await mkdtemp(join(tmpdir(), "webstudio-release-smoke-"));
|
|
let preview: ChildProcess | undefined;
|
|
let closeApiServer: (() => Promise<void>) | undefined;
|
|
try {
|
|
const startedAt = Date.now();
|
|
const fixture = createFixture();
|
|
const initialState = state.createBuilderStateFromBuildData({
|
|
...fixture.build,
|
|
pages: pagesModule.migratePages(fixture.persistedPages),
|
|
assets: fixture.assets,
|
|
instances: fixture.instances.map(([, instance]) => instance),
|
|
props: fixture.props,
|
|
});
|
|
const fixtureApi = await startFixtureApi(fixture, {
|
|
initialState,
|
|
executeRuntime: runtimeRegistry.executeBuilderRuntimeOperation,
|
|
applyTransactions: state.applyBuilderPatchTransactions,
|
|
});
|
|
fixture.data.origin = fixtureApi.origin;
|
|
reportPhase("fixture-api", startedAt);
|
|
closeApiServer = fixtureApi.close;
|
|
const packDirectory = join(temp, "pack");
|
|
const stagingDirectory = join(temp, "staging");
|
|
const projectDirectory = join(temp, "candidate-project");
|
|
const baselineProjectDirectory = join(temp, "baseline-project");
|
|
const configDirectory = join(temp, "candidate-config");
|
|
const baselineConfigDirectory = join(temp, "baseline-config");
|
|
await Promise.all([
|
|
mkdir(packDirectory, { recursive: true }),
|
|
mkdir(stagingDirectory, { recursive: true }),
|
|
mkdir(join(projectDirectory, ".webstudio"), { recursive: true }),
|
|
mkdir(join(baselineProjectDirectory, ".webstudio"), { recursive: true }),
|
|
mkdir(join(configDirectory, "webstudio-nodejs"), { recursive: true }),
|
|
mkdir(join(baselineConfigDirectory, "webstudio-nodejs"), {
|
|
recursive: true,
|
|
}),
|
|
]);
|
|
const requestedCandidate =
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_CANDIDATE ?? "local";
|
|
let candidate: ReleaseTarget;
|
|
if (requestedCandidate === "local") {
|
|
const releaseVersion =
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_VERSION ??
|
|
(await resolveRegistryTarget("webstudio@latest")).version;
|
|
const packStartedAt = Date.now();
|
|
await packReleaseShapedCli({
|
|
stagingDirectory,
|
|
packDirectory,
|
|
releaseVersion,
|
|
});
|
|
reportPhase("package", packStartedAt);
|
|
const tarballName = (await readdir(packDirectory)).find((name) =>
|
|
name.endsWith(".tgz")
|
|
);
|
|
if (tarballName === undefined) {
|
|
throw new Error("npm pack did not produce a tarball.");
|
|
}
|
|
candidate = {
|
|
source: "local",
|
|
version: releaseVersion,
|
|
installSpec: join(packDirectory, tarballName),
|
|
};
|
|
} else {
|
|
candidate = await resolveRegistryTarget(requestedCandidate);
|
|
}
|
|
const baseline = await resolveRegistryTarget(
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_BASELINE ?? "webstudio@latest"
|
|
);
|
|
reporter.setSubjects({
|
|
candidate: { source: candidate.source, version: candidate.version },
|
|
baseline: { source: baseline.source, version: baseline.version },
|
|
});
|
|
const expectedRegistryVersion =
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_EXPECT_REGISTRY_VERSION;
|
|
if (expectedRegistryVersion !== undefined) {
|
|
if (
|
|
candidate.source !== "registry" ||
|
|
candidate.version !== expectedRegistryVersion ||
|
|
candidate.integrity === undefined
|
|
) {
|
|
reporter.setRegistryArtifact("failed");
|
|
throw new Error(
|
|
`Expected exact registry artifact webstudio@${expectedRegistryVersion}, resolved ${candidate.source} ${candidate.version}.`
|
|
);
|
|
}
|
|
}
|
|
|
|
const candidateInstallStartedAt = Date.now();
|
|
await installReleaseTarget({
|
|
directory: projectDirectory,
|
|
target: candidate,
|
|
enforceArtifactContract: true,
|
|
});
|
|
if (expectedRegistryVersion !== undefined) {
|
|
reporter.setRegistryArtifact("passed");
|
|
}
|
|
reportPhase("candidate-install", candidateInstallStartedAt);
|
|
const baselineInstallStartedAt = Date.now();
|
|
await installReleaseTarget({
|
|
directory: baselineProjectDirectory,
|
|
target: baseline,
|
|
});
|
|
reportPhase("baseline-install", baselineInstallStartedAt);
|
|
|
|
const bin = join(projectDirectory, "node_modules", ".bin", "webstudio");
|
|
const baselineBin = join(
|
|
baselineProjectDirectory,
|
|
"node_modules",
|
|
".bin",
|
|
"webstudio"
|
|
);
|
|
const env = {
|
|
...process.env,
|
|
WEBSTUDIO_CONFIG_DIR: configDirectory,
|
|
};
|
|
const baselineEnv = {
|
|
...process.env,
|
|
WEBSTUDIO_CONFIG_DIR: baselineConfigDirectory,
|
|
};
|
|
const globalConfigPath = join(configDirectory, "webstudio-config.json");
|
|
const baselineGlobalConfigPath = join(
|
|
baselineConfigDirectory,
|
|
"webstudio-config.json"
|
|
);
|
|
await Promise.all([
|
|
mkdir(dirname(globalConfigPath), { recursive: true }),
|
|
mkdir(dirname(baselineGlobalConfigPath), { recursive: true }),
|
|
writeFile(globalConfigPath, "{}", "utf8"),
|
|
writeFile(baselineGlobalConfigPath, "{}", "utf8"),
|
|
]);
|
|
const workflowStartedAt = Date.now();
|
|
const runCliStep = (step: string, recovery: string, args: string[]) =>
|
|
runAgentWorkflowStep({
|
|
step,
|
|
recovery,
|
|
run: () =>
|
|
execFileAsync(bin, args, {
|
|
cwd: projectDirectory,
|
|
env,
|
|
maxBuffer: 20 * 1024 * 1024,
|
|
}),
|
|
});
|
|
|
|
const shareLink = `${fixtureApi.origin}/builder/${projectId}?authToken=fixture`;
|
|
const initialized = await runCliStep(
|
|
"link",
|
|
"Create a fresh editable API share link and rerun webstudio init --link <share-link> --json.",
|
|
["init", "--link", shareLink, "--json"]
|
|
);
|
|
await assertJsonStdout(initialized.stdout, "agent workflow link");
|
|
await runCliStep(
|
|
"sync",
|
|
"Verify the share link has API access, then run webstudio sync again.",
|
|
["sync"]
|
|
);
|
|
for (const { connect: client } of agentSmokeClients) {
|
|
await runCliStep(
|
|
`connect ${client}`,
|
|
`Run webstudio connect ${client} again and reload ${client}.`,
|
|
["connect", client]
|
|
);
|
|
}
|
|
|
|
const builderState = state.createBuilderStateFromBuildData({
|
|
...fixture.build,
|
|
pages: pagesModule.migratePages(fixture.persistedPages),
|
|
assets: fixture.assets,
|
|
instances: fixture.instances.map(([, instance]) => instance),
|
|
props: fixture.props,
|
|
});
|
|
const loadedAt = "2026-01-01T00:00:00.000Z";
|
|
const freshness = Object.fromEntries(
|
|
namespaces.builderNamespaces.map((namespace) => [
|
|
namespace,
|
|
{
|
|
status: "fresh",
|
|
version: projectVersion,
|
|
source: "remote",
|
|
loadedAt,
|
|
},
|
|
])
|
|
);
|
|
const candidateSessionSnapshot = {
|
|
projectId,
|
|
buildId,
|
|
version: projectVersion,
|
|
state: state.createSerializedBuilderStateSnapshotFromState(builderState),
|
|
freshness,
|
|
compatibilityVersion: sessionVersion,
|
|
compatibility: {
|
|
...projectSession.createDefaultProjectSessionCompatibility(
|
|
sessionVersion
|
|
),
|
|
apiCompatibilityVersion: candidate.version,
|
|
},
|
|
revision: randomUUID(),
|
|
};
|
|
await writeFile(
|
|
join(projectDirectory, ".webstudio", "project-session.json"),
|
|
JSON.stringify(candidateSessionSnapshot),
|
|
"utf8"
|
|
);
|
|
await cp(
|
|
join(projectDirectory, ".webstudio"),
|
|
join(baselineProjectDirectory, ".webstudio"),
|
|
{ recursive: true, force: true }
|
|
);
|
|
await Promise.all([
|
|
writeFile(
|
|
join(baselineProjectDirectory, ".webstudio", "project-session.json"),
|
|
JSON.stringify({
|
|
...candidateSessionSnapshot,
|
|
compatibility: {
|
|
...candidateSessionSnapshot.compatibility,
|
|
apiCompatibilityVersion: baseline.version,
|
|
},
|
|
revision: randomUUID(),
|
|
}),
|
|
"utf8"
|
|
),
|
|
cp(globalConfigPath, baselineGlobalConfigPath, { force: true }),
|
|
]);
|
|
const agentCommand = process.env.WEBSTUDIO_RELEASE_SMOKE_AGENT_COMMAND;
|
|
if (
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_MODEL_EVAL_REQUIRED === "1" &&
|
|
(agentCommand === undefined || agentCommand === "")
|
|
) {
|
|
throw new Error(
|
|
"WEBSTUDIO_RELEASE_SMOKE_AGENT_COMMAND is required for model evaluation; no model evidence was created."
|
|
);
|
|
}
|
|
if (agentCommand !== undefined && agentCommand !== "") {
|
|
const agentReportPath = process.env.WEBSTUDIO_RELEASE_SMOKE_AGENT_REPORT;
|
|
if (agentReportPath === undefined || agentReportPath === "") {
|
|
throw new Error(
|
|
"WEBSTUDIO_RELEASE_SMOKE_AGENT_REPORT is required when a model evaluation command is configured."
|
|
);
|
|
}
|
|
const modelEvaluationStartedAt = Date.now();
|
|
await runModelEvaluation({
|
|
command: agentCommand,
|
|
reportPath: agentReportPath,
|
|
candidate,
|
|
baseline,
|
|
candidateBin: bin,
|
|
baselineBin,
|
|
candidateProjectDirectory: projectDirectory,
|
|
baselineProjectDirectory,
|
|
candidateConfigDirectory: configDirectory,
|
|
baselineConfigDirectory,
|
|
temp,
|
|
});
|
|
reportPhase("model-evaluation", modelEvaluationStartedAt);
|
|
}
|
|
const compactComparisonStartedAt = Date.now();
|
|
const candidateCompact = await runCompactMeasurementSuite({
|
|
bin,
|
|
cwd: projectDirectory,
|
|
env,
|
|
expectedDetail: "compact",
|
|
});
|
|
const candidateVerbose = await runCompactMeasurementSuite({
|
|
bin,
|
|
cwd: projectDirectory,
|
|
env,
|
|
expectedDetail: "verbose",
|
|
});
|
|
const baselineCompact = await runCompactMeasurementSuite({
|
|
bin: baselineBin,
|
|
cwd: baselineProjectDirectory,
|
|
env: baselineEnv,
|
|
expectedDetail: "compact",
|
|
});
|
|
for (const operation of compactSmokeOperations) {
|
|
const candidateMeasurement = candidateCompact.measurements.get(operation);
|
|
const verboseMeasurement = candidateVerbose.measurements.get(operation);
|
|
const baselineMeasurement = baselineCompact.measurements.get(operation);
|
|
if (
|
|
candidateMeasurement === undefined ||
|
|
verboseMeasurement === undefined ||
|
|
baselineMeasurement === undefined
|
|
) {
|
|
throw new Error(`Missing comparison measurement for ${operation}.`);
|
|
}
|
|
reporter.recordOperation(
|
|
operation,
|
|
candidateMeasurement,
|
|
verboseMeasurement,
|
|
baselineMeasurement
|
|
);
|
|
}
|
|
const candidateContractPassed =
|
|
candidateCompact.contractFailures.length === 0;
|
|
const candidateVerboseContractPassed =
|
|
candidateVerbose.contractFailures.length === 0;
|
|
const baselineContractPassed =
|
|
baselineCompact.contractFailures.length === 0;
|
|
reporter.setContract(
|
|
"candidateCompact",
|
|
candidateContractPassed ? "passed" : "failed"
|
|
);
|
|
reporter.setContract(
|
|
"candidateVerbose",
|
|
candidateVerboseContractPassed ? "passed" : "failed"
|
|
);
|
|
reporter.setContract(
|
|
"historicalBaselineCompact",
|
|
baselineContractPassed ? "passed" : "failed"
|
|
);
|
|
reportPhase("compact-comparison", compactComparisonStartedAt);
|
|
if (candidateContractPassed === false) {
|
|
throw new Error(
|
|
`Compact output contract failed for candidate [${candidateCompact.contractFailures.join(", ")}].`
|
|
);
|
|
}
|
|
if (candidateVerboseContractPassed === false) {
|
|
throw new Error(
|
|
`Verbose semantic baseline failed for candidate [${candidateVerbose.contractFailures.join(", ")}].`
|
|
);
|
|
}
|
|
if (reporter.candidatePassesCompactRollout() === false) {
|
|
const report = reporter.createReport("failed");
|
|
const totals = report.sameCandidate.totals;
|
|
throw new Error(
|
|
`Candidate compact rollout gate failed: compact=${totals.compact.tokens} tokens, verbose=${totals.verbose.tokens} tokens, budget=${report.candidateTokenBudget}, retryDelta=${totals.retryDelta}, failureDelta=${totals.failureDelta}. Compact must be strictly smaller with no retry or failure regression.`
|
|
);
|
|
}
|
|
if (candidateCompact.canarySeen === false) {
|
|
throw new Error(
|
|
"Privacy canary was not exercised by the candidate compact list-pages response."
|
|
);
|
|
}
|
|
reporter.markCanaryExercised();
|
|
const invoke = async (label: string, args: string[]) => {
|
|
const result = await execFileAsync(bin, args, {
|
|
cwd: projectDirectory,
|
|
env,
|
|
maxBuffer: 20 * 1024 * 1024,
|
|
});
|
|
await assertJsonStdout(result.stdout, label);
|
|
return JSON.parse(result.stdout) as Record<string, unknown>;
|
|
};
|
|
|
|
const stagingPublish = await invoke("agent workflow staging publish", [
|
|
"publish",
|
|
"deploy",
|
|
"--target",
|
|
"staging",
|
|
"--json",
|
|
]);
|
|
if (
|
|
stagingPublish.ok !== true ||
|
|
getRecord(stagingPublish.data, "staging publish data").jobId !==
|
|
"release-smoke-staging-job"
|
|
) {
|
|
throw new Error(
|
|
`Agent connection workflow stopped at "staging publish": ${JSON.stringify(stagingPublish)} Recovery: Verify edit permission and retry webstudio publish deploy --target staging --json.`
|
|
);
|
|
}
|
|
|
|
const docs = await readFile(
|
|
join(import.meta.dirname, "../src/docs/api-use-cases.md"),
|
|
"utf8"
|
|
);
|
|
const documentedAuditCommands = docs
|
|
.split("\n")
|
|
.filter((line) => line.startsWith("- webstudio audit "))
|
|
.map((line) =>
|
|
line
|
|
.slice("- webstudio ".length)
|
|
.trim()
|
|
.split(/\s+/)
|
|
.filter((arg) => arg !== "--json")
|
|
);
|
|
if (documentedAuditCommands.length === 0) {
|
|
throw new Error("No documented audit CLI examples were found.");
|
|
}
|
|
const helpStartedAt = Date.now();
|
|
for (const args of documentedAuditCommands) {
|
|
const result = await execFileAsync(bin, [...args, "--help"], {
|
|
cwd: projectDirectory,
|
|
env,
|
|
});
|
|
if (!result.stdout.includes("webstudio audit")) {
|
|
throw new Error(
|
|
`Packed audit help is not audit-specific: ${result.stdout}`
|
|
);
|
|
}
|
|
}
|
|
reportPhase("audit-help", helpStartedAt);
|
|
|
|
const previewStartedAt = Date.now();
|
|
preview = spawn(bin, ["preview", "--port", "5190"], {
|
|
cwd: projectDirectory,
|
|
env,
|
|
detached: process.platform !== "win32",
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
await waitForUrl("http://127.0.0.1:5190/", preview);
|
|
const previewManifest = JSON.parse(
|
|
await readFile(
|
|
join(projectDirectory, ".webstudio/preview/package.json"),
|
|
"utf8"
|
|
)
|
|
) as { dependencies?: Record<string, string> };
|
|
if (
|
|
previewManifest.dependencies?.ipx !== undefined ||
|
|
previewManifest.dependencies?.h3 !== undefined
|
|
) {
|
|
throw new Error(
|
|
"Generic React Router preview unexpectedly includes Docker-only image proxy dependencies."
|
|
);
|
|
}
|
|
reportPhase("preview-start", previewStartedAt);
|
|
const commandsStartedAt = Date.now();
|
|
const assertLocalDryRun = async (
|
|
label: string,
|
|
command: string,
|
|
input: Record<string, unknown>
|
|
) => {
|
|
const payload = await invoke(label, [
|
|
command,
|
|
JSON.stringify(input),
|
|
"--dry-run",
|
|
]);
|
|
const meta = getRecord(payload.meta, `${label} meta`);
|
|
const session = getRecord(meta.session, `${label} session`);
|
|
if (
|
|
payload.ok !== true ||
|
|
session.source !== "dry-run" ||
|
|
session.committed !== false ||
|
|
typeof session.transaction !== "object"
|
|
) {
|
|
throw new Error(
|
|
`${label} did not produce a local dry-run transaction: ${JSON.stringify(payload)}`
|
|
);
|
|
}
|
|
};
|
|
await invoke("packed list pages", ["list-pages"]);
|
|
await assertLocalDryRun("plain page metadata update", "update-page", {
|
|
pageId: "home",
|
|
values: {
|
|
title: "My title: seven pages & more",
|
|
meta: { description: "Ordinary prose, punctuation included." },
|
|
},
|
|
});
|
|
await assertLocalDryRun("cold page duplicate", "duplicate-page", {
|
|
pageId: "home",
|
|
name: "Home copy",
|
|
path: "/home-copy",
|
|
});
|
|
await assertLocalDryRun("local fragment insertion", "insert-fragment", {
|
|
parentInstanceId: "home-root",
|
|
fragment: '<ws.element ws:tag="p">Release smoke</ws.element>',
|
|
});
|
|
await assertLocalDryRun("local component insertion", "insert-component", {
|
|
parentInstanceId: "home-root",
|
|
component: "ws:element",
|
|
tag: "p",
|
|
});
|
|
|
|
await stopProcessTree(preview);
|
|
preview = undefined;
|
|
await rm(join(projectDirectory, ".webstudio", "project-session.json"), {
|
|
force: true,
|
|
});
|
|
const stdioStartedAt = Date.now();
|
|
for (const { name: clientName } of agentSmokeClients) {
|
|
await runAgentWorkflowStep({
|
|
step: `${clientName} connection smoke`,
|
|
recovery: `Reload ${clientName}, then verify its Webstudio MCP command starts in the linked project folder.`,
|
|
run: async () => {
|
|
const { client, transport } = createMcpSmokeClient({
|
|
bin,
|
|
cwd: projectDirectory,
|
|
env,
|
|
name: clientName,
|
|
});
|
|
transport.stderr?.on("data", () => {});
|
|
await client.connect(transport);
|
|
try {
|
|
const tools = await client.listTools();
|
|
if (
|
|
tools.tools.some(({ name }) => name === "meta.index") === false
|
|
) {
|
|
throw new Error("Webstudio connected but meta.index is missing.");
|
|
}
|
|
} finally {
|
|
await client.close();
|
|
}
|
|
},
|
|
});
|
|
}
|
|
const { client: stdioClient, transport: stdioTransport } =
|
|
createMcpSmokeClient({
|
|
bin,
|
|
cwd: projectDirectory,
|
|
env,
|
|
name: "webstudio-release-smoke",
|
|
});
|
|
let stdioLogs = "";
|
|
stdioTransport.stderr?.on("data", (chunk) => {
|
|
stdioLogs += String(chunk);
|
|
});
|
|
const callTool = async (
|
|
name: string,
|
|
args: Record<string, unknown> = {}
|
|
) => {
|
|
const result = await stdioClient.callTool({ name, arguments: args });
|
|
const structured = getRecord(
|
|
result.structuredContent,
|
|
`${name} structured content`
|
|
);
|
|
if (result.isError === true || structured.ok !== true) {
|
|
throw new Error(
|
|
`${name} failed over packed stdio MCP: ${JSON.stringify(structured)}`
|
|
);
|
|
}
|
|
return getRecord(structured.data, `${name} data`);
|
|
};
|
|
const listAllPages = async () => {
|
|
const pages: unknown[] = [];
|
|
let cursor: string | undefined;
|
|
do {
|
|
const result = await callTool("list-pages", { cursor, limit: 200 });
|
|
if (Array.isArray(result.pages) === false) {
|
|
throw new Error(
|
|
`list-pages returned invalid data: ${JSON.stringify(result)}`
|
|
);
|
|
}
|
|
pages.push(...result.pages);
|
|
cursor =
|
|
typeof result.nextCursor === "string" ? result.nextCursor : undefined;
|
|
} while (cursor !== undefined);
|
|
return pages;
|
|
};
|
|
await stdioClient.connect(stdioTransport);
|
|
try {
|
|
const inspection = await callTool("inspect");
|
|
if (inspection.id !== projectId) {
|
|
throw new Error(
|
|
`Inspect returned the wrong project: ${JSON.stringify(inspection)}`
|
|
);
|
|
}
|
|
const initialPages = await listAllPages();
|
|
if (initialPages.length !== fixture.data.pages.length) {
|
|
throw new Error(
|
|
`Cold stdio discovery returned unexpected pages: ${JSON.stringify(initialPages)}`
|
|
);
|
|
}
|
|
await callTool("refresh", {
|
|
namespaces: ["pages", "instances", "props", "assets"],
|
|
});
|
|
const assets = await callTool("list-assets");
|
|
if (Array.isArray(assets.items) === false || assets.items.length !== 1) {
|
|
throw new Error(
|
|
`Cold stdio asset discovery failed: ${JSON.stringify(assets)}`
|
|
);
|
|
}
|
|
|
|
const createdPages: Array<{
|
|
pageId: string;
|
|
name: string;
|
|
path: string;
|
|
}> = [];
|
|
const duplicate = await callTool("duplicate-page", {
|
|
pageId: "home",
|
|
...duplicatedPage,
|
|
});
|
|
createdPages.push({
|
|
pageId: String(duplicate.pageId),
|
|
...duplicatedPage,
|
|
});
|
|
for (const page of createdPageInputs) {
|
|
const created = await callTool("create-page", page);
|
|
createdPages.push({ pageId: String(created.pageId), ...page });
|
|
}
|
|
|
|
for (const page of createdPages) {
|
|
await callTool("update-page", {
|
|
pageId: page.pageId,
|
|
values: {
|
|
title: `${page.name}: release smoke & verification`,
|
|
meta: { description: `A plain-text description for ${page.name}.` },
|
|
},
|
|
});
|
|
const details = await callTool("get-page", { pageId: page.pageId });
|
|
const inserted = await callTool("insert-component", {
|
|
parentInstanceId: String(details.rootInstanceId),
|
|
component: "ws:element",
|
|
tag: "section",
|
|
});
|
|
const sectionId = String(
|
|
(inserted.rootInstanceIds as unknown[] | undefined)?.[0]
|
|
);
|
|
await callTool("update-props", {
|
|
updates: [
|
|
{
|
|
instanceId: sectionId,
|
|
name: "data-imported-page",
|
|
type: "string",
|
|
value: page.name,
|
|
},
|
|
],
|
|
});
|
|
}
|
|
|
|
await callTool("update-text", {
|
|
instanceId: "pricing-root",
|
|
childIndex: 0,
|
|
text: "Pricing imported content",
|
|
mode: "text",
|
|
});
|
|
|
|
for (let index = 0; index < 10; index += 1) {
|
|
await callTool("list-pages");
|
|
await callTool("update-page", {
|
|
pageId: "pricing",
|
|
values: { title: `Pricing session check ${index}` },
|
|
});
|
|
}
|
|
const committedPricing = await callTool("get-page", {
|
|
pageId: "pricing",
|
|
});
|
|
await callTool("refresh", { namespaces: ["pages"] });
|
|
const refreshedPricing = await callTool("get-page", {
|
|
pageId: "pricing",
|
|
});
|
|
if (refreshedPricing.title !== committedPricing.title) {
|
|
throw new Error(
|
|
`Committed page title did not survive refresh: before=${JSON.stringify(committedPricing)} after=${JSON.stringify(refreshedPricing)}`
|
|
);
|
|
}
|
|
|
|
const usage = await callTool("find-asset-usage", {
|
|
assetId: "asset-hero",
|
|
});
|
|
if (Array.isArray(usage.usages) === false || usage.usages.length < 2) {
|
|
throw new Error(
|
|
`Duplicated page lost its image asset reference: ${JSON.stringify(usage)}`
|
|
);
|
|
}
|
|
|
|
await callTool("preview.start", {
|
|
host: "127.0.0.1",
|
|
port: 5191,
|
|
source: "session",
|
|
});
|
|
const workflowScreenshot = await callTool("screenshot", {
|
|
path: "/work",
|
|
output: ".webstudio/workflow-work.png",
|
|
viewport: { width: 1280, height: 800 },
|
|
waitUntil: "load",
|
|
});
|
|
await assertScreenshotEvidence({
|
|
payload: { data: workflowScreenshot },
|
|
label: "seven-page stdio workflow screenshot",
|
|
path: "/work",
|
|
status: 200,
|
|
viewport: { width: 1280, height: 800 },
|
|
projectDirectory,
|
|
origin: "http://127.0.0.1:5191",
|
|
});
|
|
await callTool("preview.stop");
|
|
|
|
const workflowDuration = Date.now() - workflowStartedAt;
|
|
if (workflowDuration >= 10 * 60_000) {
|
|
throw new Error(
|
|
`Agent connection workflow exceeded ten minutes (${workflowDuration}ms). Recovery: Inspect the reported release-smoke phase timings and optimize the slowest blocked step.`
|
|
);
|
|
}
|
|
reportPhase("agent-workflow", workflowStartedAt);
|
|
|
|
const finalPages = await listAllPages();
|
|
const finalPageNames = finalPages.map(
|
|
(page) => getRecord(page, "final page").name
|
|
);
|
|
if (
|
|
finalPageNames.length !==
|
|
fixture.data.pages.length + createdPages.length ||
|
|
createdPages.some(
|
|
(page) => finalPageNames.includes(page.name) === false
|
|
)
|
|
) {
|
|
throw new Error(
|
|
`Seven-page workflow verification failed: ${JSON.stringify(finalPages)}`
|
|
);
|
|
}
|
|
} finally {
|
|
await stdioClient.close();
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
if (
|
|
stdioLogs.includes("API contract negotiated") === false ||
|
|
stdioLogs.includes('"event":"stdio_connection_closed"') === false ||
|
|
stdioLogs.includes('"event":"stdio_transport_error"')
|
|
) {
|
|
throw new Error(`Unexpected packed stdio lifecycle logs:\n${stdioLogs}`);
|
|
}
|
|
reportPhase("stdio-workflow", stdioStartedAt);
|
|
|
|
preview = spawn(bin, ["preview", "--port", "5190"], {
|
|
cwd: projectDirectory,
|
|
env,
|
|
detached: process.platform !== "win32",
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
await waitForUrl("http://127.0.0.1:5190/", preview);
|
|
|
|
const pricingScreenshot = await invoke("standalone screenshot", [
|
|
"screenshot",
|
|
"http://127.0.0.1:5190/pricing",
|
|
"--output",
|
|
".webstudio/pricing.png",
|
|
"--width",
|
|
"1440",
|
|
"--height",
|
|
"900",
|
|
"--json",
|
|
]);
|
|
await assertScreenshotEvidence({
|
|
payload: pricingScreenshot,
|
|
label: "standalone pricing screenshot",
|
|
path: "/pricing",
|
|
status: 200,
|
|
viewport: { width: 1440, height: 900 },
|
|
projectDirectory,
|
|
});
|
|
const homeScreenshot = await invoke("MCP screenshot", [
|
|
"mcp",
|
|
"single-op-call",
|
|
"screenshot",
|
|
JSON.stringify({
|
|
baseUrl: "http://127.0.0.1:5190",
|
|
path: "/",
|
|
viewport: { width: 375, height: 812 },
|
|
}),
|
|
]);
|
|
await assertScreenshotEvidence({
|
|
payload: homeScreenshot,
|
|
label: "MCP home screenshot",
|
|
path: "/",
|
|
status: 200,
|
|
viewport: { width: 375, height: 812 },
|
|
projectDirectory,
|
|
});
|
|
await invoke("MCP run", [
|
|
"mcp",
|
|
"run",
|
|
JSON.stringify([
|
|
{
|
|
tool: "screenshot",
|
|
input: {
|
|
baseUrl: "http://127.0.0.1:5190",
|
|
path: "/pricing",
|
|
viewport: { width: 1440, height: 900 },
|
|
},
|
|
},
|
|
]),
|
|
]);
|
|
const missingScreenshot = await invoke("missing route screenshot", [
|
|
"screenshot",
|
|
"http://127.0.0.1:5190/missing",
|
|
"--output",
|
|
".webstudio/missing.png",
|
|
"--width",
|
|
"375",
|
|
"--height",
|
|
"812",
|
|
"--json",
|
|
]);
|
|
await assertScreenshotEvidence({
|
|
payload: missingScreenshot,
|
|
label: "missing route screenshot",
|
|
path: "/missing",
|
|
status: 404,
|
|
viewport: { width: 375, height: 812 },
|
|
projectDirectory,
|
|
generatedSite: false,
|
|
});
|
|
const renderedAudit = await invoke("rendered audit", [
|
|
"audit",
|
|
"--rendered",
|
|
"--page-path",
|
|
"/",
|
|
"--json",
|
|
]);
|
|
const renderedAuditData = getRecord(
|
|
renderedAudit.data,
|
|
"rendered audit data"
|
|
);
|
|
const renderedAuditSummary = getRecord(
|
|
renderedAuditData.renderedCaptureSummary,
|
|
"rendered audit capture summary"
|
|
);
|
|
if (
|
|
renderedAudit.ok !== true ||
|
|
renderedAuditData.renderedCheckCount !== 1 ||
|
|
renderedAuditSummary.planned !== 1 ||
|
|
renderedAuditSummary.failed !== 0 ||
|
|
renderedAuditSummary.skipped !== 0
|
|
) {
|
|
throw new Error(
|
|
`Rendered audit was incomplete: ${JSON.stringify(renderedAuditData)}`
|
|
);
|
|
}
|
|
if (process.env.WEBSTUDIO_RELEASE_SMOKE_LARGE_AUDIT === "1") {
|
|
const expectedCaptureCount =
|
|
fixture.data.pages.length + additionalPageCount;
|
|
const planPayload = await invoke("large rendered audit plan", [
|
|
"audit",
|
|
"--rendered",
|
|
"--scopes",
|
|
"performance",
|
|
"--json",
|
|
]);
|
|
const planData = getRecord(planPayload.data, "large audit plan data");
|
|
const plan = getRecord(planData.renderedPlan, "large rendered plan");
|
|
if (
|
|
plan.captureCount !== expectedCaptureCount ||
|
|
typeof plan.confirmationToken !== "string"
|
|
) {
|
|
throw new Error(
|
|
`Large rendered audit did not return the expected ${expectedCaptureCount}-capture confirmation plan: ${JSON.stringify(plan)}`
|
|
);
|
|
}
|
|
const confirmedAuditStartedAt = Date.now();
|
|
const confirmedPayload = await invoke("confirmed large rendered audit", [
|
|
"audit",
|
|
"--rendered",
|
|
"--scopes",
|
|
"performance",
|
|
"--confirm-large-run",
|
|
"--confirmation-token",
|
|
plan.confirmationToken,
|
|
"--json",
|
|
]);
|
|
const confirmedData = getRecord(
|
|
confirmedPayload.data,
|
|
"confirmed large audit data"
|
|
);
|
|
const summary = getRecord(
|
|
confirmedData.renderedCaptureSummary,
|
|
"confirmed large audit capture summary"
|
|
);
|
|
const manifestSummary = getRecord(
|
|
confirmedData.renderedArtifactManifest,
|
|
"confirmed large audit artifact manifest"
|
|
);
|
|
const manifestPath = String(manifestSummary.path);
|
|
const absoluteManifestPath = manifestPath.startsWith("/")
|
|
? manifestPath
|
|
: join(projectDirectory, manifestPath);
|
|
const manifest = JSON.parse(
|
|
await readFile(absoluteManifestPath, "utf8")
|
|
) as Record<string, unknown>;
|
|
if (
|
|
confirmedPayload.ok !== true ||
|
|
confirmedData.renderedCheckCount !== expectedCaptureCount ||
|
|
summary.planned !== expectedCaptureCount ||
|
|
summary.failed !== 0 ||
|
|
summary.skipped !== 0
|
|
) {
|
|
throw new Error(
|
|
`Confirmed large rendered audit was incomplete: ${JSON.stringify({ summary, failures: manifest.failures })}`
|
|
);
|
|
}
|
|
const memory = process.memoryUsage();
|
|
const buildMetrics = isPlainRecord(manifest.generatedBuildMetrics)
|
|
? manifest.generatedBuildMetrics
|
|
: undefined;
|
|
reportPhase("rendered-audit", confirmedAuditStartedAt, {
|
|
rssBytes: memory.rss,
|
|
heapUsedBytes: memory.heapUsed,
|
|
networkTransferBytes: getRenderedNetworkTransferBytes(manifest),
|
|
retainedArtifactBytes: await getRetainedArtifactBytes({
|
|
manifest,
|
|
manifestPath: absoluteManifestPath,
|
|
projectDirectory,
|
|
}),
|
|
...(typeof buildMetrics?.bytes === "number"
|
|
? { generatedBuildBytes: buildMetrics.bytes }
|
|
: {}),
|
|
...(typeof buildMetrics?.gzipBytes === "number"
|
|
? { generatedBuildGzipBytes: buildMetrics.gzipBytes }
|
|
: {}),
|
|
});
|
|
const confirmedAuditDuration = Date.now() - confirmedAuditStartedAt;
|
|
if (confirmedAuditDuration > 120_000) {
|
|
throw new Error(
|
|
`Confirmed ${expectedCaptureCount}-capture packaged audit took ${confirmedAuditDuration}ms; expected at most 120000ms.`
|
|
);
|
|
}
|
|
const cachedAuditStartedAt = Date.now();
|
|
await invoke("cached rendered audit", [
|
|
"audit",
|
|
"--rendered",
|
|
"--page-path",
|
|
"/",
|
|
"--scopes",
|
|
"performance",
|
|
"--json",
|
|
]);
|
|
reportPhase("cached-audit", cachedAuditStartedAt);
|
|
const cachedAuditDuration = Date.now() - cachedAuditStartedAt;
|
|
if (cachedAuditDuration > 20_000) {
|
|
throw new Error(
|
|
`Cached rendered audit took ${cachedAuditDuration}ms; expected at most 20000ms.`
|
|
);
|
|
}
|
|
}
|
|
reportPhase("structured-commands", commandsStartedAt);
|
|
} finally {
|
|
await stopProcessTree(preview);
|
|
await closeApiServer?.();
|
|
await rm(temp, { recursive: true, force: true });
|
|
reportPhase("cleanup");
|
|
}
|
|
};
|
|
|
|
let outcome: "passed" | "failed" = "failed";
|
|
try {
|
|
await run();
|
|
outcome = "passed";
|
|
} finally {
|
|
const reportPath = process.env.WEBSTUDIO_RELEASE_SMOKE_REPORT;
|
|
if (reportPath !== undefined && reportPath !== "") {
|
|
await reporter.write(reportPath, outcome);
|
|
}
|
|
}
|