mirror of
https://github.com/webstudio-is/webstudio.git
synced 2026-11-10 06:24:35 +09:00
Oleg Isonen
58eaf8c7c9
## Summary - finish the `publish` job after npm publication and exact registry-artifact verification - run packaged CLI verification in a separate `release-smoke` job - keep `publish-docs` dependent on successful release creation and package publication, but independent of post-publish smoke results - verify the Codex registration command through `webstudio connect codex --print` in CI instead of requiring an installed Codex client ## Root cause The release workflow ran post-publish CLI smoke verification inside the `publish` job. GitHub therefore marked package publication as failed whenever the smoke failed, even though npm publication had already completed. Because `publish-docs` declared `needs: [release, publish]`, GitHub skipped documentation publication. The recurring failure was the smoke invoking `webstudio connect codex` on a runner without the `codex` executable. ## Technical choices and tradeoffs Package publication remains strict: npm publication and exact registry visibility must both succeed before downstream jobs run. Post-publish smoke verification remains a red, visible release job and still uploads its diagnostic artifact, but it no longer changes the truthful outcome of publication or blocks docs. The Codex package smoke now validates the generated registration command without mutating a real Codex installation. The command subprocess boundary remains covered by the existing focused connect-command tests. No migrations or compatibility changes are involved. ## Todo - [x] Separate post-publish CLI smoke from package publication. - [x] Keep docs gated on successful release and registry publication. - [x] Make Codex connection verification deterministic in CI. - [x] Manually advance the `docs` branch to release 0.290.0. - [x] Review documentation impact: no product documentation update is required because this changes release automation only. - [x] Verify formatting, workflow syntax and topology, CLI tests, lint, and typechecking. - [x] Run the non-model release smoke and record the remaining unrelated failure. ## Verification - `pnpm exec oxfmt --check .github/workflows/release.yml packages/cli/scripts/release-smoke.ts` - parsed `.github/workflows/release.yml` with Ruby Psych - asserted the parsed workflow job graph with the repository's `yaml` dependency - `pnpm --filter webstudio exec vitest run scripts/release-smoke-registry.test.ts src/commands/connect.test.ts` — 22 tests passed - `pnpm --filter webstudio typecheck` - `pnpm exec oxlint packages/cli/scripts/release-smoke.ts` - `pnpm --filter webstudio test:release-smoke` — the corrected Codex path passed; the smoke later reached `preview.start` and failed on an unrelated current-main fixture error: `Project session is missing preview data: assetFolders` Fixture review: this change does not modify fixture inputs or generated fixture outputs. Closes #6119
2123 lines
65 KiB
TypeScript
2123 lines
65 KiB
TypeScript
import { execFile, spawn, type ChildProcess } from "node:child_process";
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
import {
|
|
cp,
|
|
mkdir,
|
|
mkdtemp,
|
|
readFile,
|
|
readdir,
|
|
rm,
|
|
stat,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { dirname, join } from "node:path";
|
|
import { promisify } from "node:util";
|
|
import { bundleVersion } from "@webstudio-is/protocol";
|
|
import type { Instance } from "@webstudio-is/sdk";
|
|
import type { BuilderState } from "@webstudio-is/project-build/state";
|
|
import type { BuilderPatchTransaction } from "@webstudio-is/project-build/contracts";
|
|
import type { BuilderRuntimeMutation } from "@webstudio-is/project-build/runtime";
|
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
import React from "react";
|
|
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
import {
|
|
compactSmokeOperations,
|
|
countCompactTokens,
|
|
createReleaseSmokeReporter,
|
|
defaultCandidateTokenBudget,
|
|
getModelVerificationItemCount,
|
|
modelEvaluationReportSchema,
|
|
modelEvaluationSubmissionSchema,
|
|
modelEvaluationTaskId,
|
|
publishedCliManifestContractSchema,
|
|
writeModelEvaluationReport,
|
|
type CompactOperationMeasurement,
|
|
type CompactSmokeOperation,
|
|
} from "./release-smoke-report";
|
|
import {
|
|
createRuntimeFixtureBuildSnapshot,
|
|
createRuntimeFixtureSerializedBuild,
|
|
runtimeFixturePermissions,
|
|
startRuntimeFixtureApi,
|
|
} from "./runtime-fixture-api";
|
|
import { runAgentCommand } from "./run-agent-command";
|
|
import {
|
|
resolveRegistryTarget,
|
|
type ReleaseTarget,
|
|
} from "./release-smoke-registry";
|
|
import { isPlainRecord } from "../src/type-utils";
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
const projectId = "release-smoke-project";
|
|
const buildId = "release-smoke-build";
|
|
const projectVersion = 1;
|
|
const sessionVersion = "cli-project-session-v1";
|
|
const releaseVersionPlaceholder = "0.0.0-webstudio-version";
|
|
const privacyCanary = "releaseSmokePrivateCanaryAuthTokenDoNotReport";
|
|
const duplicatedPage = { name: "About", path: "/about" } as const;
|
|
const createdPageInputs = [
|
|
{ name: "Services", path: "/services" },
|
|
{ name: "Work", path: "/work" },
|
|
{ name: "Blog", path: "/blog" },
|
|
{ name: "Contact", path: "/contact" },
|
|
] as const;
|
|
const additionalPageCount = 1 + createdPageInputs.length;
|
|
const agentSmokeClients = [
|
|
{ connect: "claude", name: "claude-code", print: false },
|
|
{ connect: "codex", name: "codex", print: true },
|
|
{ connect: "cursor", name: "cursor", print: false },
|
|
{ connect: "vscode", name: "vscode", print: false },
|
|
] as const;
|
|
|
|
const configuredTokenBudget = Number(
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_TOKEN_BUDGET ??
|
|
defaultCandidateTokenBudget
|
|
);
|
|
const reporter = createReleaseSmokeReporter({
|
|
privacyCanary,
|
|
candidateTokenBudget: configuredTokenBudget,
|
|
});
|
|
const reportPhase = reporter.complete;
|
|
|
|
const readNpmJson = async (args: string[]) =>
|
|
JSON.parse((await execFileAsync("npm", args)).stdout) as unknown;
|
|
|
|
const installReleaseTarget = async ({
|
|
directory,
|
|
target,
|
|
enforceArtifactContract = false,
|
|
}: {
|
|
directory: string;
|
|
target: ReleaseTarget;
|
|
enforceArtifactContract?: boolean;
|
|
}) => {
|
|
await writeFile(
|
|
join(directory, "package.json"),
|
|
JSON.stringify({ private: true }),
|
|
"utf8"
|
|
);
|
|
const install = await execFileAsync(
|
|
"npm",
|
|
[
|
|
"install",
|
|
target.installSpec,
|
|
"--no-audit",
|
|
"--no-fund",
|
|
"--loglevel=warn",
|
|
],
|
|
{ cwd: directory, maxBuffer: 10 * 1024 * 1024 }
|
|
);
|
|
if (/ERESOLVE|overriding peer dependency/i.test(install.stderr)) {
|
|
throw new Error(
|
|
`npm install emitted peer dependency warnings for ${target.source} ${target.version}.`
|
|
);
|
|
}
|
|
const installedManifest = JSON.parse(
|
|
await readFile(
|
|
join(directory, "node_modules", "webstudio", "package.json"),
|
|
"utf8"
|
|
)
|
|
) as Record<string, unknown>;
|
|
if (installedManifest.version !== target.version) {
|
|
throw new Error(
|
|
`Clean install resolved webstudio ${String(installedManifest.version)} instead of ${target.version}.`
|
|
);
|
|
}
|
|
if (enforceArtifactContract) {
|
|
publishedCliManifestContractSchema.parse({
|
|
name: installedManifest.name,
|
|
version: installedManifest.version,
|
|
type: installedManifest.type,
|
|
bin: installedManifest.bin,
|
|
files: installedManifest.files,
|
|
engines: installedManifest.engines,
|
|
});
|
|
const packageDirectory = join(directory, "node_modules", "webstudio");
|
|
const packageEntries = new Set(await readdir(packageDirectory));
|
|
for (const requiredEntry of [
|
|
"bin.js",
|
|
"lib",
|
|
"package.json",
|
|
"templates",
|
|
]) {
|
|
if (packageEntries.has(requiredEntry) === false) {
|
|
throw new Error(
|
|
`Installed webstudio artifact is missing required entry ${requiredEntry}.`
|
|
);
|
|
}
|
|
}
|
|
const allowedTopLevelEntries = new Set([
|
|
"LICENSE",
|
|
"README.md",
|
|
"bin.js",
|
|
"lib",
|
|
"package.json",
|
|
"templates",
|
|
]);
|
|
const unexpectedEntry = Array.from(packageEntries).find(
|
|
(entry) => allowedTopLevelEntries.has(entry) === false
|
|
);
|
|
if (unexpectedEntry !== undefined) {
|
|
throw new Error(
|
|
`Installed webstudio artifact contains unexpected top-level entry ${unexpectedEntry}.`
|
|
);
|
|
}
|
|
const scanTree = async (path: string, relativePath = "") => {
|
|
for (const entry of await readdir(path, { withFileTypes: true })) {
|
|
const entryPath = join(path, entry.name);
|
|
const relativeEntryPath = join(relativePath, entry.name);
|
|
if (entry.isSymbolicLink()) {
|
|
throw new Error(
|
|
`Installed webstudio artifact contains symlink ${relativeEntryPath}.`
|
|
);
|
|
}
|
|
if (
|
|
/(?:^|\/)(?:\.env(?:\.|$)|\.npmrc$)|\.(?:map|test\.[^.]+|stories\.[^.]+)$/i.test(
|
|
relativeEntryPath
|
|
)
|
|
) {
|
|
throw new Error(
|
|
`Installed webstudio artifact contains forbidden file ${relativeEntryPath}.`
|
|
);
|
|
}
|
|
if (entry.isDirectory()) {
|
|
await scanTree(entryPath, relativeEntryPath);
|
|
}
|
|
}
|
|
};
|
|
await scanTree(packageDirectory);
|
|
const executable = await stat(join(packageDirectory, "bin.js"));
|
|
if (executable.isFile() === false || (executable.mode & 0o111) === 0) {
|
|
throw new Error("Installed webstudio bin.js is not executable.");
|
|
}
|
|
}
|
|
if (target.integrity !== undefined) {
|
|
const lock = JSON.parse(
|
|
await readFile(join(directory, "package-lock.json"), "utf8")
|
|
) as { packages?: Record<string, { integrity?: unknown }> };
|
|
const installedIntegrity =
|
|
lock.packages?.["node_modules/webstudio"]?.integrity;
|
|
if (installedIntegrity !== target.integrity) {
|
|
throw new Error(
|
|
`Installed webstudio integrity ${String(installedIntegrity)} does not match registry metadata.`
|
|
);
|
|
}
|
|
}
|
|
};
|
|
|
|
const runAgentWorkflowStep = async <Result>({
|
|
step,
|
|
recovery,
|
|
run,
|
|
}: {
|
|
step: string;
|
|
recovery: string;
|
|
run: () => Promise<Result>;
|
|
}) => {
|
|
try {
|
|
return await run();
|
|
} catch (error) {
|
|
throw new Error(
|
|
`Agent connection workflow stopped at "${step}": ${error instanceof Error ? error.message : String(error)} Recovery: ${recovery}`,
|
|
{ cause: error }
|
|
);
|
|
}
|
|
};
|
|
|
|
const replaceReleaseVersion = async (
|
|
directory: string,
|
|
releaseVersion: string,
|
|
workspaceVersions: ReadonlyMap<string, string>
|
|
) => {
|
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
await Promise.all(
|
|
entries.map(async (entry) => {
|
|
const entryPath = join(directory, entry.name);
|
|
if (entry.isDirectory()) {
|
|
await replaceReleaseVersion(
|
|
entryPath,
|
|
releaseVersion,
|
|
workspaceVersions
|
|
);
|
|
return;
|
|
}
|
|
if (/\.(?:js|mjs|json)$/.test(entry.name) === false) {
|
|
return;
|
|
}
|
|
const source = await readFile(entryPath, "utf8");
|
|
let releaseSource = source.replaceAll(
|
|
releaseVersionPlaceholder,
|
|
releaseVersion
|
|
);
|
|
if (entry.name === "package.json") {
|
|
const manifest = JSON.parse(releaseSource) as Record<string, unknown>;
|
|
for (const field of [
|
|
"dependencies",
|
|
"devDependencies",
|
|
"optionalDependencies",
|
|
"peerDependencies",
|
|
]) {
|
|
const dependencies = manifest[field];
|
|
if (
|
|
typeof dependencies !== "object" ||
|
|
dependencies === null ||
|
|
Array.isArray(dependencies)
|
|
) {
|
|
continue;
|
|
}
|
|
const dependencyVersions = dependencies as Record<string, unknown>;
|
|
for (const [name, version] of Object.entries(dependencyVersions)) {
|
|
if (
|
|
typeof version === "string" &&
|
|
version.startsWith("workspace:")
|
|
) {
|
|
const workspaceVersion = workspaceVersions.get(name);
|
|
if (workspaceVersion === undefined) {
|
|
throw new Error(
|
|
`Cannot resolve workspace dependency ${name} in ${entryPath}.`
|
|
);
|
|
}
|
|
dependencyVersions[name] = workspaceVersion;
|
|
}
|
|
}
|
|
}
|
|
releaseSource = JSON.stringify(manifest, undefined, 2);
|
|
}
|
|
if (releaseSource !== source) {
|
|
await writeFile(entryPath, releaseSource, "utf8");
|
|
}
|
|
})
|
|
);
|
|
};
|
|
|
|
const packReleaseShapedCli = async ({
|
|
stagingDirectory,
|
|
packDirectory,
|
|
releaseVersion,
|
|
}: {
|
|
stagingDirectory: string;
|
|
packDirectory: string;
|
|
releaseVersion: string;
|
|
}) => {
|
|
const cliDirectory = join(import.meta.dirname, "..");
|
|
await Promise.all(
|
|
["bin.js", "package.json", "lib", "templates"].map((entry) =>
|
|
cp(join(cliDirectory, entry), join(stagingDirectory, entry), {
|
|
recursive: true,
|
|
})
|
|
)
|
|
);
|
|
if (typeof releaseVersion !== "string" || releaseVersion.length === 0) {
|
|
throw new Error("Could not resolve a published Webstudio release version.");
|
|
}
|
|
const packagesDirectory = join(import.meta.dirname, "../..");
|
|
const workspaceVersions = new Map<string, string>();
|
|
for (const entry of await readdir(packagesDirectory, {
|
|
withFileTypes: true,
|
|
})) {
|
|
if (entry.isDirectory() === false) {
|
|
continue;
|
|
}
|
|
try {
|
|
const manifest = JSON.parse(
|
|
await readFile(
|
|
join(packagesDirectory, entry.name, "package.json"),
|
|
"utf8"
|
|
)
|
|
) as { name?: string; version?: string };
|
|
if (manifest.name !== undefined && manifest.version !== undefined) {
|
|
workspaceVersions.set(
|
|
manifest.name,
|
|
manifest.version.replace(releaseVersionPlaceholder, releaseVersion)
|
|
);
|
|
}
|
|
} catch (error) {
|
|
if (
|
|
error instanceof Error &&
|
|
"code" in error &&
|
|
error.code === "ENOENT"
|
|
) {
|
|
continue;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
await replaceReleaseVersion(
|
|
stagingDirectory,
|
|
releaseVersion,
|
|
workspaceVersions
|
|
);
|
|
await execFileAsync("npm", ["pack", "--pack-destination", packDirectory], {
|
|
cwd: stagingDirectory,
|
|
});
|
|
return releaseVersion;
|
|
};
|
|
|
|
const createFixture = () => {
|
|
const assets = [
|
|
{
|
|
id: "asset-hero",
|
|
projectId,
|
|
name: "release-hero.png",
|
|
filename: "release-hero.png",
|
|
type: "image" as const,
|
|
size: 128,
|
|
format: "png",
|
|
createdAt: "2026-01-01T00:00:00.000Z",
|
|
description: "Release smoke hero",
|
|
meta: { width: 1200, height: 800 },
|
|
},
|
|
];
|
|
const basePages = [
|
|
{
|
|
id: "home",
|
|
name: "Home",
|
|
title: privacyCanary,
|
|
path: "",
|
|
rootInstanceId: "home-root",
|
|
meta: {},
|
|
},
|
|
{
|
|
id: "pricing",
|
|
name: "Pricing",
|
|
title: "Pricing",
|
|
path: "/pricing",
|
|
rootInstanceId: "pricing-root",
|
|
meta: {},
|
|
},
|
|
];
|
|
const extraPageCount =
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_LARGE_AUDIT === "1" ? 114 : 0;
|
|
const pages = [
|
|
...basePages,
|
|
...Array.from({ length: extraPageCount }, (_, index) => {
|
|
const number = index + 3;
|
|
return {
|
|
id: `audit-${number}`,
|
|
name: `Audit ${number}`,
|
|
title: `Audit ${number}`,
|
|
path: `/audit-${number}`,
|
|
rootInstanceId: `audit-${number}-root`,
|
|
meta: {},
|
|
};
|
|
}),
|
|
];
|
|
const persistedPages = {
|
|
meta: { siteName: "Release Smoke", contactEmail: "" },
|
|
compiler: { atomicStyles: true },
|
|
redirects: [],
|
|
homePageId: "home",
|
|
rootFolderId: "root-folder",
|
|
pages,
|
|
folders: [
|
|
{
|
|
id: "root-folder",
|
|
name: "Root",
|
|
slug: "",
|
|
children: pages.map((page) => page.id),
|
|
},
|
|
],
|
|
};
|
|
const instances: Array<[string, Instance]> = pages.map((page) => [
|
|
page.rootInstanceId,
|
|
{
|
|
type: "instance",
|
|
id: page.rootInstanceId,
|
|
component: "Box",
|
|
children:
|
|
page.id === "home"
|
|
? [
|
|
{ type: "text", value: `Release smoke ${page.name}` },
|
|
{ type: "id", value: "home-image" },
|
|
]
|
|
: [{ type: "text", value: `Release smoke ${page.name}` }],
|
|
},
|
|
]);
|
|
instances.push([
|
|
"home-image",
|
|
{
|
|
type: "instance",
|
|
id: "home-image",
|
|
component: "Image",
|
|
children: [],
|
|
},
|
|
]);
|
|
const props = [
|
|
{
|
|
id: "home-image-src",
|
|
instanceId: "home-image",
|
|
name: "src",
|
|
type: "asset" as const,
|
|
value: "asset-hero",
|
|
},
|
|
];
|
|
const build = {
|
|
id: buildId,
|
|
projectId,
|
|
version: projectVersion,
|
|
createdAt: "2026-01-01T00:00:00.000Z",
|
|
updatedAt: "2026-01-01T00:00:00.000Z",
|
|
pages: persistedPages,
|
|
props: props.map((prop) => [prop.id, prop]) as never,
|
|
instances: instances as never,
|
|
dataSources: [],
|
|
resources: [],
|
|
styleSources: [],
|
|
styleSourceSelections: [],
|
|
styles: [],
|
|
breakpoints: [],
|
|
projectSettings: { meta: {}, compiler: {} },
|
|
};
|
|
return {
|
|
data: {
|
|
bundleVersion,
|
|
origin: "https://assets.example",
|
|
projectDomain: "example.com",
|
|
projectTitle: "Release Smoke",
|
|
page: pages[0],
|
|
pages,
|
|
assets,
|
|
build,
|
|
},
|
|
build,
|
|
persistedPages,
|
|
instances,
|
|
assets,
|
|
props,
|
|
};
|
|
};
|
|
|
|
const waitForUrl = async (url: string, child: ChildProcess) => {
|
|
const deadline = Date.now() + 60_000;
|
|
while (Date.now() < deadline) {
|
|
if (child.exitCode !== null) {
|
|
throw new Error(`Preview exited with code ${child.exitCode}.`);
|
|
}
|
|
try {
|
|
const response = await fetch(url);
|
|
if (response.ok) {
|
|
return;
|
|
}
|
|
} catch {}
|
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
}
|
|
throw new Error(`Preview did not become ready at ${url}.`);
|
|
};
|
|
|
|
const stopProcessTree = async (child: ChildProcess | undefined) => {
|
|
if (child?.pid === undefined) {
|
|
return;
|
|
}
|
|
const pid = child.pid;
|
|
const usesProcessGroup = process.platform !== "win32";
|
|
const signal = (value: NodeJS.Signals) => {
|
|
try {
|
|
if (usesProcessGroup) {
|
|
process.kill(-pid, value);
|
|
} else {
|
|
child.kill(value);
|
|
}
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== "ESRCH") {
|
|
throw error;
|
|
}
|
|
}
|
|
};
|
|
const isRunning = () => {
|
|
try {
|
|
process.kill(usesProcessGroup ? -pid : pid, 0);
|
|
return true;
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code === "ESRCH") {
|
|
return false;
|
|
}
|
|
throw error;
|
|
}
|
|
};
|
|
const waitUntilStopped = async (timeoutMs: number) => {
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (isRunning() && Date.now() < deadline) {
|
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
}
|
|
return isRunning() === false;
|
|
};
|
|
|
|
signal("SIGTERM");
|
|
if (await waitUntilStopped(5_000)) {
|
|
return;
|
|
}
|
|
signal("SIGKILL");
|
|
if ((await waitUntilStopped(5_000)) === false) {
|
|
throw new Error(`Preview process group ${pid} did not stop.`);
|
|
}
|
|
};
|
|
|
|
const startFixtureApi = async (
|
|
fixture: ReturnType<typeof createFixture>,
|
|
{
|
|
initialState,
|
|
executeRuntime,
|
|
applyTransactions,
|
|
}: {
|
|
initialState: BuilderState;
|
|
executeRuntime: (input: {
|
|
id: string;
|
|
state: BuilderState;
|
|
input: unknown;
|
|
context: {
|
|
createId: () => string;
|
|
projectId: string;
|
|
projectVersion: number;
|
|
};
|
|
}) => unknown | Promise<unknown>;
|
|
applyTransactions: (
|
|
state: BuilderState,
|
|
transactions: readonly BuilderPatchTransaction[]
|
|
) => { state: BuilderState };
|
|
}
|
|
) => {
|
|
let version = projectVersion;
|
|
let state = initialState;
|
|
let generatedId = 0;
|
|
const getCurrentBuildSnapshot = () =>
|
|
createRuntimeFixtureBuildSnapshot({
|
|
state,
|
|
projectId,
|
|
buildId,
|
|
version,
|
|
});
|
|
const getCurrentSerializedBuild = () =>
|
|
createRuntimeFixtureSerializedBuild({
|
|
state,
|
|
baseBuild: fixture.build,
|
|
buildId,
|
|
version,
|
|
});
|
|
return startRuntimeFixtureApi(
|
|
async ({ pathname, response, operationPath, readInput }) => {
|
|
if (pathname.endsWith("/release-hero.png")) {
|
|
response.writeHead(200, { "content-type": "image/png" });
|
|
response.end(
|
|
Buffer.from(
|
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
|
|
"base64"
|
|
)
|
|
);
|
|
return undefined;
|
|
}
|
|
let data: unknown;
|
|
if (operationPath === "build.loadProjectBundleByProjectId") {
|
|
data = { ...fixture.data, build: getCurrentSerializedBuild() };
|
|
} else if (operationPath === "projects.get") {
|
|
data = {
|
|
id: projectId,
|
|
name: "Release smoke project",
|
|
createdAt: "2026-01-01T00:00:00.000Z",
|
|
updatedAt: "2026-01-01T00:00:00.000Z",
|
|
buildId,
|
|
version,
|
|
homePageId: "home",
|
|
features: {},
|
|
};
|
|
} else if (operationPath === "publish.create") {
|
|
data = { jobId: "release-smoke-staging-job" };
|
|
} else if (operationPath === "build.patch") {
|
|
const input = (await readInput()) as {
|
|
transactions?: BuilderPatchTransaction[];
|
|
};
|
|
state = applyTransactions(state, input.transactions ?? []).state;
|
|
version += 1;
|
|
data = { version };
|
|
} else if (operationPath === "build.get") {
|
|
data = getCurrentBuildSnapshot();
|
|
} else if (
|
|
operationPath !== "projects.permissions" &&
|
|
operationPath !== ""
|
|
) {
|
|
const input = await readInput();
|
|
const result = await executeRuntime({
|
|
id: operationPath,
|
|
state,
|
|
input,
|
|
context: {
|
|
createId: () => `fixture-${generatedId++}`,
|
|
projectId,
|
|
projectVersion: version,
|
|
},
|
|
});
|
|
if (
|
|
typeof result === "object" &&
|
|
result !== null &&
|
|
"kind" in result &&
|
|
(result as BuilderRuntimeMutation).kind === "mutation"
|
|
) {
|
|
const mutation = result as BuilderRuntimeMutation;
|
|
state = applyTransactions(state, [
|
|
{
|
|
id: `fixture-transaction-${generatedId++}`,
|
|
payload: mutation.payload,
|
|
},
|
|
]).state;
|
|
version += 1;
|
|
data = mutation.result;
|
|
} else {
|
|
data = result;
|
|
}
|
|
} else {
|
|
data = runtimeFixturePermissions;
|
|
}
|
|
return data;
|
|
}
|
|
);
|
|
};
|
|
|
|
const assertJsonStdout = async (stdout: string, label: string) => {
|
|
try {
|
|
JSON.parse(stdout);
|
|
} catch (error) {
|
|
throw new Error(`${label} did not print valid JSON.`, { cause: error });
|
|
}
|
|
};
|
|
|
|
const getRecord = (value: unknown, label: string) => {
|
|
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
throw new Error(`${label} is not an object.`);
|
|
}
|
|
return value as Record<string, unknown>;
|
|
};
|
|
|
|
const createMcpSmokeClient = ({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
name,
|
|
}: {
|
|
bin: string;
|
|
cwd: string;
|
|
env: NodeJS.ProcessEnv;
|
|
name: string;
|
|
}) => {
|
|
const transport = new StdioClientTransport({
|
|
command: bin,
|
|
args: ["mcp"],
|
|
cwd,
|
|
env: Object.fromEntries(
|
|
Object.entries(env).filter(
|
|
(entry): entry is [string, string] => typeof entry[1] === "string"
|
|
)
|
|
),
|
|
stderr: "pipe",
|
|
});
|
|
const client = new Client({ name, version: "1.0.0" });
|
|
return { client, transport };
|
|
};
|
|
|
|
const emptyFallbackCounts = () => ({
|
|
verbose: 0,
|
|
fullResource: 0,
|
|
source: 0,
|
|
});
|
|
|
|
const runCompactMeasurementSuite = async ({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
expectedDetail,
|
|
}: {
|
|
bin: string;
|
|
cwd: string;
|
|
env: NodeJS.ProcessEnv;
|
|
expectedDetail: "compact" | "verbose";
|
|
}) => {
|
|
const { client, transport } = createMcpSmokeClient({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
name: "webstudio-release-smoke-compact-measurement",
|
|
});
|
|
const measurements = new Map<
|
|
CompactSmokeOperation,
|
|
CompactOperationMeasurement
|
|
>();
|
|
const contractFailures: CompactSmokeOperation[] = [];
|
|
let canarySeen = false;
|
|
await client.connect(transport);
|
|
try {
|
|
for (const operation of compactSmokeOperations) {
|
|
let attempts = 0;
|
|
let failures = 0;
|
|
let bytes = 0;
|
|
let tokens = 0;
|
|
const fallbacks = emptyFallbackCounts();
|
|
let passed = false;
|
|
for (let attempt = 0; attempt < 2 && passed === false; attempt += 1) {
|
|
attempts += 1;
|
|
try {
|
|
const result = await client.callTool({
|
|
name: operation,
|
|
arguments: {
|
|
limit: 25,
|
|
...(expectedDetail === "verbose" ? { verbose: true } : {}),
|
|
},
|
|
});
|
|
const source = JSON.stringify(result.structuredContent);
|
|
canarySeen ||= source.includes(privacyCanary);
|
|
const structured = getRecord(
|
|
result.structuredContent,
|
|
`${operation} compact structured content`
|
|
);
|
|
const data = getRecord(
|
|
structured.data,
|
|
`${operation} compact response data`
|
|
);
|
|
if (result.isError === true || structured.ok !== true) {
|
|
throw new Error(`${expectedDetail} operation failed`);
|
|
}
|
|
bytes = Buffer.byteLength(source, "utf8");
|
|
tokens = countCompactTokens(source);
|
|
if (data.detail !== expectedDetail) {
|
|
contractFailures.push(operation);
|
|
if (data.detail === "verbose") {
|
|
fallbacks.verbose += 1;
|
|
}
|
|
}
|
|
passed = true;
|
|
} catch {
|
|
failures += 1;
|
|
}
|
|
}
|
|
measurements.set(operation, {
|
|
bytes,
|
|
tokens,
|
|
attempts,
|
|
retries: Math.max(0, attempts - 1),
|
|
failures,
|
|
fallbacks,
|
|
});
|
|
if (passed === false) {
|
|
contractFailures.push(operation);
|
|
}
|
|
}
|
|
} finally {
|
|
await client.close();
|
|
}
|
|
return { measurements, contractFailures, canarySeen };
|
|
};
|
|
|
|
const verifyModelEvaluationOutcome = async ({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
}: {
|
|
bin: string;
|
|
cwd: string;
|
|
env: NodeJS.ProcessEnv;
|
|
}) => {
|
|
const { client, transport } = createMcpSmokeClient({
|
|
bin,
|
|
cwd,
|
|
env,
|
|
name: "webstudio-release-smoke-model-verifier",
|
|
});
|
|
await client.connect(transport);
|
|
try {
|
|
const countItems = async (operation: "list-pages" | "list-assets") => {
|
|
let result = await client.callTool({
|
|
name: operation,
|
|
arguments: { limit: 25 },
|
|
});
|
|
if (result.isError === true) {
|
|
result = await client.callTool({ name: operation, arguments: {} });
|
|
}
|
|
const structured = getRecord(
|
|
result.structuredContent,
|
|
`${operation} model evaluation verification response`
|
|
);
|
|
if (result.isError === true || structured.ok === false) {
|
|
throw new Error(`${operation} model evaluation verification failed.`);
|
|
}
|
|
return getModelVerificationItemCount(operation, structured);
|
|
};
|
|
return {
|
|
pageCount: await countItems("list-pages"),
|
|
assetCount: await countItems("list-assets"),
|
|
};
|
|
} finally {
|
|
await client.close();
|
|
}
|
|
};
|
|
|
|
const runSingleModelEvaluation = async ({
|
|
targetName,
|
|
command,
|
|
subject,
|
|
bin,
|
|
projectDirectory,
|
|
configDirectory,
|
|
temp,
|
|
}: {
|
|
targetName: "candidate" | "baseline";
|
|
command: string;
|
|
subject: ReleaseTarget;
|
|
bin: string;
|
|
projectDirectory: string;
|
|
configDirectory: string;
|
|
temp: string;
|
|
}) => {
|
|
const evaluationDirectory = join(
|
|
temp,
|
|
`model-evaluation-${targetName}-project`
|
|
);
|
|
const evaluationConfigDirectory = join(
|
|
temp,
|
|
`model-evaluation-${targetName}-config`
|
|
);
|
|
const taskPath = join(temp, `model-evaluation-${targetName}-task.json`);
|
|
const submissionPath = join(
|
|
temp,
|
|
`model-evaluation-${targetName}-submission.json`
|
|
);
|
|
await Promise.all([
|
|
cp(projectDirectory, evaluationDirectory, { recursive: true }),
|
|
cp(configDirectory, evaluationConfigDirectory, { recursive: true }),
|
|
]);
|
|
const task = {
|
|
schemaVersion: 2,
|
|
taskId: modelEvaluationTaskId,
|
|
objective:
|
|
"Using only the packaged Webstudio MCP server, determine the exact number of pages and assets in the linked project. Choose the minimal tool sequence yourself.",
|
|
mcp: { command: bin, args: ["mcp"], cwd: evaluationDirectory },
|
|
resultPath: submissionPath,
|
|
resultContract: {
|
|
schemaVersion: 2,
|
|
taskId: modelEvaluationTaskId,
|
|
provider: "agent provider identifier",
|
|
model: "agent model identifier",
|
|
outcome: "completed or failed",
|
|
toolCallCount: "non-negative integer",
|
|
observations: {
|
|
pageCount: "non-negative integer",
|
|
assetCount: "non-negative integer",
|
|
},
|
|
},
|
|
};
|
|
await writeFile(taskPath, JSON.stringify(task, undefined, 2), "utf8");
|
|
const evaluationEnv = {
|
|
...process.env,
|
|
WEBSTUDIO_CONFIG_DIR: evaluationConfigDirectory,
|
|
WEBSTUDIO_RELEASE_SMOKE_AGENT_TASK: taskPath,
|
|
WEBSTUDIO_RELEASE_SMOKE_AGENT_RESULT: submissionPath,
|
|
WEBSTUDIO_RELEASE_SMOKE_MCP_COMMAND: bin,
|
|
WEBSTUDIO_RELEASE_SMOKE_MCP_CWD: evaluationDirectory,
|
|
WEBSTUDIO_RELEASE_SMOKE_MODEL_TARGET: targetName,
|
|
};
|
|
const execution = await runAgentCommand({
|
|
command,
|
|
cwd: evaluationDirectory,
|
|
env: evaluationEnv,
|
|
timeoutMs: 10 * 60_000,
|
|
});
|
|
const submission = modelEvaluationSubmissionSchema.parse(
|
|
JSON.parse(await readFile(submissionPath, "utf8"))
|
|
);
|
|
const verifiedObservations = await verifyModelEvaluationOutcome({
|
|
bin,
|
|
cwd: evaluationDirectory,
|
|
env: evaluationEnv,
|
|
});
|
|
const projectOutcome =
|
|
submission.observations.pageCount === verifiedObservations.pageCount &&
|
|
submission.observations.assetCount === verifiedObservations.assetCount;
|
|
const passed =
|
|
execution.exitCode === 0 &&
|
|
submission.outcome === "completed" &&
|
|
projectOutcome;
|
|
return {
|
|
passed,
|
|
run: {
|
|
subject: { source: subject.source, version: subject.version },
|
|
evidence: {
|
|
source: "configured-agent-command" as const,
|
|
provider: submission.provider,
|
|
model: submission.model,
|
|
commandSha256: createHash("sha256").update(command).digest("hex"),
|
|
durationMs: execution.durationMs,
|
|
exitCode: execution.exitCode,
|
|
toolCallCount: submission.toolCallCount,
|
|
},
|
|
observations: submission.observations,
|
|
checks: {
|
|
submissionContract: "passed" as const,
|
|
projectOutcome: projectOutcome
|
|
? ("passed" as const)
|
|
: ("failed" as const),
|
|
},
|
|
},
|
|
};
|
|
};
|
|
|
|
const runModelEvaluation = async ({
|
|
command,
|
|
reportPath,
|
|
candidate,
|
|
baseline,
|
|
candidateBin,
|
|
baselineBin,
|
|
candidateProjectDirectory,
|
|
baselineProjectDirectory,
|
|
candidateConfigDirectory,
|
|
baselineConfigDirectory,
|
|
temp,
|
|
}: {
|
|
command: string;
|
|
reportPath: string;
|
|
candidate: ReleaseTarget;
|
|
baseline: ReleaseTarget;
|
|
candidateBin: string;
|
|
baselineBin: string;
|
|
candidateProjectDirectory: string;
|
|
baselineProjectDirectory: string;
|
|
candidateConfigDirectory: string;
|
|
baselineConfigDirectory: string;
|
|
temp: string;
|
|
}) => {
|
|
const results = await Promise.allSettled([
|
|
runSingleModelEvaluation({
|
|
targetName: "candidate",
|
|
command,
|
|
subject: candidate,
|
|
bin: candidateBin,
|
|
projectDirectory: candidateProjectDirectory,
|
|
configDirectory: candidateConfigDirectory,
|
|
temp,
|
|
}),
|
|
runSingleModelEvaluation({
|
|
targetName: "baseline",
|
|
command,
|
|
subject: baseline,
|
|
bin: baselineBin,
|
|
projectDirectory: baselineProjectDirectory,
|
|
configDirectory: baselineConfigDirectory,
|
|
temp,
|
|
}),
|
|
]);
|
|
const rejected = results.find(
|
|
(result): result is PromiseRejectedResult => result.status === "rejected"
|
|
);
|
|
if (rejected !== undefined) {
|
|
throw rejected.reason;
|
|
}
|
|
const [candidateResult, baselineResult] = results.map(
|
|
(result) =>
|
|
(
|
|
result as PromiseFulfilledResult<
|
|
Awaited<ReturnType<typeof runSingleModelEvaluation>>
|
|
>
|
|
).value
|
|
);
|
|
if (candidateResult === undefined || baselineResult === undefined) {
|
|
throw new Error(
|
|
"Both packaged CLI model evaluations must produce evidence."
|
|
);
|
|
}
|
|
const passed = candidateResult.passed && baselineResult.passed;
|
|
const report = modelEvaluationReportSchema.parse({
|
|
schemaVersion: 2,
|
|
kind: "model-driven-packaged-cli-evaluation",
|
|
outcome: passed ? "passed" : "failed",
|
|
subjects: {
|
|
candidate: { source: candidate.source, version: candidate.version },
|
|
baseline: { source: baseline.source, version: baseline.version },
|
|
},
|
|
taskId: modelEvaluationTaskId,
|
|
runs: {
|
|
candidate: candidateResult.run,
|
|
baseline: baselineResult.run,
|
|
},
|
|
checks: {
|
|
bothOutcomes: passed ? "passed" : "failed",
|
|
canaryRedaction: "passed",
|
|
},
|
|
});
|
|
await writeModelEvaluationReport(reportPath, report, [privacyCanary]);
|
|
if (passed === false) {
|
|
throw new Error(
|
|
"Configured model-driven evaluation did not pass against both packaged CLIs."
|
|
);
|
|
}
|
|
};
|
|
|
|
const getRenderedNetworkTransferBytes = (manifest: Record<string, unknown>) =>
|
|
(Array.isArray(manifest.checks) ? manifest.checks : []).reduce(
|
|
(total, check) => {
|
|
if (
|
|
isPlainRecord(check) === false ||
|
|
isPlainRecord(check.layout) === false
|
|
) {
|
|
return total;
|
|
}
|
|
return (
|
|
total +
|
|
(Array.isArray(check.layout.resources)
|
|
? check.layout.resources
|
|
: []
|
|
).reduce(
|
|
(resourceTotal, resource) =>
|
|
resourceTotal +
|
|
(isPlainRecord(resource) &&
|
|
typeof resource.transferSize === "number"
|
|
? resource.transferSize
|
|
: 0),
|
|
0
|
|
)
|
|
);
|
|
},
|
|
0
|
|
);
|
|
|
|
const getRetainedArtifactBytes = async ({
|
|
manifest,
|
|
manifestPath,
|
|
projectDirectory,
|
|
}: {
|
|
manifest: Record<string, unknown>;
|
|
manifestPath: string;
|
|
projectDirectory: string;
|
|
}) => {
|
|
const artifactPaths = new Set([manifestPath]);
|
|
for (const collection of [manifest.checks, manifest.failures]) {
|
|
if (Array.isArray(collection) === false) {
|
|
continue;
|
|
}
|
|
for (const item of collection) {
|
|
if (isPlainRecord(item) && typeof item.screenshotPath === "string") {
|
|
artifactPaths.add(
|
|
item.screenshotPath.startsWith("/")
|
|
? item.screenshotPath
|
|
: join(projectDirectory, item.screenshotPath)
|
|
);
|
|
}
|
|
}
|
|
}
|
|
const sizes = await Promise.all(
|
|
Array.from(artifactPaths, async (path) =>
|
|
stat(path)
|
|
.then((file) => file.size)
|
|
.catch(() => 0)
|
|
)
|
|
);
|
|
return sizes.reduce((total, size) => total + size, 0);
|
|
};
|
|
|
|
const assertScreenshotEvidence = async ({
|
|
payload,
|
|
label,
|
|
path,
|
|
status,
|
|
viewport,
|
|
projectDirectory,
|
|
generatedSite = true,
|
|
origin = "http://127.0.0.1:5190",
|
|
}: {
|
|
payload: Record<string, unknown>;
|
|
label: string;
|
|
path: string;
|
|
status: number;
|
|
viewport: { width: number; height: number };
|
|
projectDirectory: string;
|
|
generatedSite?: boolean;
|
|
origin?: string;
|
|
}) => {
|
|
const data = getRecord(payload.data, `${label} data`);
|
|
const navigation = getRecord(data.navigation, `${label} navigation`);
|
|
const finalUrl = new URL(String(navigation.finalUrl));
|
|
if (
|
|
finalUrl.origin !== origin ||
|
|
finalUrl.pathname !== path ||
|
|
navigation.status !== status ||
|
|
(generatedSite && navigation.generatedSiteRootPresent !== true)
|
|
) {
|
|
throw new Error(
|
|
`${label} returned unexpected navigation evidence: ${JSON.stringify(navigation)}`
|
|
);
|
|
}
|
|
const actualViewport = getRecord(data.viewport, `${label} viewport`);
|
|
if (
|
|
actualViewport.width !== viewport.width ||
|
|
actualViewport.height !== viewport.height
|
|
) {
|
|
throw new Error(
|
|
`${label} returned unexpected viewport: ${JSON.stringify(actualViewport)}`
|
|
);
|
|
}
|
|
const output = String(data.output);
|
|
const screenshot = await readFile(
|
|
output.startsWith("/") ? output : join(projectDirectory, output)
|
|
);
|
|
const isPng = screenshot
|
|
.subarray(0, 8)
|
|
.equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]));
|
|
const isWebp =
|
|
screenshot.subarray(0, 4).toString("ascii") === "RIFF" &&
|
|
screenshot.subarray(8, 12).toString("ascii") === "WEBP";
|
|
if (screenshot.length === 0 || (isPng === false && isWebp === false)) {
|
|
throw new Error(`${label} did not create a valid PNG or WebP artifact.`);
|
|
}
|
|
};
|
|
|
|
const run = async () => {
|
|
Object.assign(globalThis, {
|
|
React,
|
|
});
|
|
const [pagesModule, state, namespaces, projectSession, runtimeRegistry] =
|
|
await Promise.all([
|
|
import("@webstudio-is/project-migrations/pages"),
|
|
import("@webstudio-is/project-build/state"),
|
|
import("@webstudio-is/project-build/contracts"),
|
|
import("@webstudio-is/project-build/project-session"),
|
|
import("@webstudio-is/project-build/runtime"),
|
|
]);
|
|
const temp = await mkdtemp(join(tmpdir(), "webstudio-release-smoke-"));
|
|
let preview: ChildProcess | undefined;
|
|
let closeApiServer: (() => Promise<void>) | undefined;
|
|
try {
|
|
const startedAt = Date.now();
|
|
const fixture = createFixture();
|
|
const initialState = state.createBuilderStateFromBuildData({
|
|
...fixture.build,
|
|
pages: pagesModule.migratePages(fixture.persistedPages),
|
|
assets: fixture.assets,
|
|
instances: fixture.instances.map(([, instance]) => instance),
|
|
props: fixture.props,
|
|
});
|
|
const fixtureApi = await startFixtureApi(fixture, {
|
|
initialState,
|
|
executeRuntime: runtimeRegistry.executeBuilderRuntimeOperation,
|
|
applyTransactions: state.applyBuilderPatchTransactions,
|
|
});
|
|
fixture.data.origin = fixtureApi.origin;
|
|
reportPhase("fixture-api", startedAt);
|
|
closeApiServer = fixtureApi.close;
|
|
const packDirectory = join(temp, "pack");
|
|
const stagingDirectory = join(temp, "staging");
|
|
const projectDirectory = join(temp, "candidate-project");
|
|
const baselineProjectDirectory = join(temp, "baseline-project");
|
|
const configDirectory = join(temp, "candidate-config");
|
|
const baselineConfigDirectory = join(temp, "baseline-config");
|
|
await Promise.all([
|
|
mkdir(packDirectory, { recursive: true }),
|
|
mkdir(stagingDirectory, { recursive: true }),
|
|
mkdir(join(projectDirectory, ".webstudio"), { recursive: true }),
|
|
mkdir(join(baselineProjectDirectory, ".webstudio"), { recursive: true }),
|
|
mkdir(join(configDirectory, "webstudio-nodejs"), { recursive: true }),
|
|
mkdir(join(baselineConfigDirectory, "webstudio-nodejs"), {
|
|
recursive: true,
|
|
}),
|
|
]);
|
|
const requestedCandidate =
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_CANDIDATE ?? "local";
|
|
let candidate: ReleaseTarget;
|
|
if (requestedCandidate === "local") {
|
|
const releaseVersion =
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_VERSION ??
|
|
(await resolveRegistryTarget("webstudio@latest", { readNpmJson }))
|
|
.version;
|
|
const packStartedAt = Date.now();
|
|
await packReleaseShapedCli({
|
|
stagingDirectory,
|
|
packDirectory,
|
|
releaseVersion,
|
|
});
|
|
reportPhase("package", packStartedAt);
|
|
const tarballName = (await readdir(packDirectory)).find((name) =>
|
|
name.endsWith(".tgz")
|
|
);
|
|
if (tarballName === undefined) {
|
|
throw new Error("npm pack did not produce a tarball.");
|
|
}
|
|
candidate = {
|
|
source: "local",
|
|
version: releaseVersion,
|
|
installSpec: join(packDirectory, tarballName),
|
|
};
|
|
} else {
|
|
candidate = await resolveRegistryTarget(requestedCandidate, {
|
|
readNpmJson,
|
|
});
|
|
}
|
|
const baseline = await resolveRegistryTarget(
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_BASELINE ?? "webstudio@latest",
|
|
{ readNpmJson }
|
|
);
|
|
reporter.setSubjects({
|
|
candidate: { source: candidate.source, version: candidate.version },
|
|
baseline: { source: baseline.source, version: baseline.version },
|
|
});
|
|
const expectedRegistryVersion =
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_EXPECT_REGISTRY_VERSION;
|
|
if (expectedRegistryVersion !== undefined) {
|
|
if (
|
|
candidate.source !== "registry" ||
|
|
candidate.version !== expectedRegistryVersion ||
|
|
candidate.integrity === undefined
|
|
) {
|
|
reporter.setRegistryArtifact("failed");
|
|
throw new Error(
|
|
`Expected exact registry artifact webstudio@${expectedRegistryVersion}, resolved ${candidate.source} ${candidate.version}.`
|
|
);
|
|
}
|
|
}
|
|
|
|
const candidateInstallStartedAt = Date.now();
|
|
await installReleaseTarget({
|
|
directory: projectDirectory,
|
|
target: candidate,
|
|
enforceArtifactContract: true,
|
|
});
|
|
if (expectedRegistryVersion !== undefined) {
|
|
reporter.setRegistryArtifact("passed");
|
|
}
|
|
reportPhase("candidate-install", candidateInstallStartedAt);
|
|
const baselineInstallStartedAt = Date.now();
|
|
await installReleaseTarget({
|
|
directory: baselineProjectDirectory,
|
|
target: baseline,
|
|
});
|
|
reportPhase("baseline-install", baselineInstallStartedAt);
|
|
|
|
const bin = join(projectDirectory, "node_modules", ".bin", "webstudio");
|
|
const baselineBin = join(
|
|
baselineProjectDirectory,
|
|
"node_modules",
|
|
".bin",
|
|
"webstudio"
|
|
);
|
|
const env = {
|
|
...process.env,
|
|
WEBSTUDIO_CONFIG_DIR: configDirectory,
|
|
};
|
|
const baselineEnv = {
|
|
...process.env,
|
|
WEBSTUDIO_CONFIG_DIR: baselineConfigDirectory,
|
|
};
|
|
const globalConfigPath = join(configDirectory, "webstudio-config.json");
|
|
const baselineGlobalConfigPath = join(
|
|
baselineConfigDirectory,
|
|
"webstudio-config.json"
|
|
);
|
|
await Promise.all([
|
|
mkdir(dirname(globalConfigPath), { recursive: true }),
|
|
mkdir(dirname(baselineGlobalConfigPath), { recursive: true }),
|
|
writeFile(globalConfigPath, "{}", "utf8"),
|
|
writeFile(baselineGlobalConfigPath, "{}", "utf8"),
|
|
]);
|
|
const workflowStartedAt = Date.now();
|
|
const runCliStep = (step: string, recovery: string, args: string[]) =>
|
|
runAgentWorkflowStep({
|
|
step,
|
|
recovery,
|
|
run: () =>
|
|
execFileAsync(bin, args, {
|
|
cwd: projectDirectory,
|
|
env,
|
|
maxBuffer: 20 * 1024 * 1024,
|
|
}),
|
|
});
|
|
|
|
const shareLink = `${fixtureApi.origin}/builder/${projectId}?authToken=fixture`;
|
|
const initialized = await runCliStep(
|
|
"link",
|
|
"Create a fresh editable API share link and rerun webstudio init --link <share-link> --json.",
|
|
["init", "--link", shareLink, "--json"]
|
|
);
|
|
await assertJsonStdout(initialized.stdout, "agent workflow link");
|
|
await runCliStep(
|
|
"sync",
|
|
"Verify the share link has API access, then run webstudio sync again.",
|
|
["sync"]
|
|
);
|
|
for (const { connect: client, print } of agentSmokeClients) {
|
|
const connected = await runCliStep(
|
|
`connect ${client}`,
|
|
`Run webstudio connect ${client} again and reload ${client}.`,
|
|
["connect", client, ...(print ? ["--print"] : [])]
|
|
);
|
|
if (
|
|
client === "codex" &&
|
|
connected.stdout.includes("codex mcp add webstudio --") === false
|
|
) {
|
|
throw new Error(
|
|
"Codex connection smoke did not print its registration command."
|
|
);
|
|
}
|
|
}
|
|
|
|
const builderState = state.createBuilderStateFromBuildData({
|
|
...fixture.build,
|
|
pages: pagesModule.migratePages(fixture.persistedPages),
|
|
assets: fixture.assets,
|
|
instances: fixture.instances.map(([, instance]) => instance),
|
|
props: fixture.props,
|
|
});
|
|
const loadedAt = "2026-01-01T00:00:00.000Z";
|
|
const freshness = Object.fromEntries(
|
|
namespaces.builderNamespaces.map((namespace) => [
|
|
namespace,
|
|
{
|
|
status: "fresh",
|
|
version: projectVersion,
|
|
source: "remote",
|
|
loadedAt,
|
|
},
|
|
])
|
|
);
|
|
const candidateSessionSnapshot = {
|
|
projectId,
|
|
buildId,
|
|
version: projectVersion,
|
|
state: state.createSerializedBuilderStateSnapshotFromState(builderState),
|
|
freshness,
|
|
compatibilityVersion: sessionVersion,
|
|
compatibility: {
|
|
...projectSession.createDefaultProjectSessionCompatibility(
|
|
sessionVersion
|
|
),
|
|
apiCompatibilityVersion: candidate.version,
|
|
},
|
|
revision: randomUUID(),
|
|
};
|
|
await writeFile(
|
|
join(projectDirectory, ".webstudio", "project-session.json"),
|
|
JSON.stringify(candidateSessionSnapshot),
|
|
"utf8"
|
|
);
|
|
await cp(
|
|
join(projectDirectory, ".webstudio"),
|
|
join(baselineProjectDirectory, ".webstudio"),
|
|
{ recursive: true, force: true }
|
|
);
|
|
await Promise.all([
|
|
writeFile(
|
|
join(baselineProjectDirectory, ".webstudio", "project-session.json"),
|
|
JSON.stringify({
|
|
...candidateSessionSnapshot,
|
|
compatibility: {
|
|
...candidateSessionSnapshot.compatibility,
|
|
apiCompatibilityVersion: baseline.version,
|
|
},
|
|
revision: randomUUID(),
|
|
}),
|
|
"utf8"
|
|
),
|
|
cp(globalConfigPath, baselineGlobalConfigPath, { force: true }),
|
|
]);
|
|
const agentCommand = process.env.WEBSTUDIO_RELEASE_SMOKE_AGENT_COMMAND;
|
|
if (
|
|
process.env.WEBSTUDIO_RELEASE_SMOKE_MODEL_EVAL_REQUIRED === "1" &&
|
|
(agentCommand === undefined || agentCommand === "")
|
|
) {
|
|
throw new Error(
|
|
"WEBSTUDIO_RELEASE_SMOKE_AGENT_COMMAND is required for model evaluation; no model evidence was created."
|
|
);
|
|
}
|
|
if (agentCommand !== undefined && agentCommand !== "") {
|
|
const agentReportPath = process.env.WEBSTUDIO_RELEASE_SMOKE_AGENT_REPORT;
|
|
if (agentReportPath === undefined || agentReportPath === "") {
|
|
throw new Error(
|
|
"WEBSTUDIO_RELEASE_SMOKE_AGENT_REPORT is required when a model evaluation command is configured."
|
|
);
|
|
}
|
|
const modelEvaluationStartedAt = Date.now();
|
|
await runModelEvaluation({
|
|
command: agentCommand,
|
|
reportPath: agentReportPath,
|
|
candidate,
|
|
baseline,
|
|
candidateBin: bin,
|
|
baselineBin,
|
|
candidateProjectDirectory: projectDirectory,
|
|
baselineProjectDirectory,
|
|
candidateConfigDirectory: configDirectory,
|
|
baselineConfigDirectory,
|
|
temp,
|
|
});
|
|
reportPhase("model-evaluation", modelEvaluationStartedAt);
|
|
}
|
|
const compactComparisonStartedAt = Date.now();
|
|
const candidateCompact = await runCompactMeasurementSuite({
|
|
bin,
|
|
cwd: projectDirectory,
|
|
env,
|
|
expectedDetail: "compact",
|
|
});
|
|
const candidateVerbose = await runCompactMeasurementSuite({
|
|
bin,
|
|
cwd: projectDirectory,
|
|
env,
|
|
expectedDetail: "verbose",
|
|
});
|
|
const baselineCompact = await runCompactMeasurementSuite({
|
|
bin: baselineBin,
|
|
cwd: baselineProjectDirectory,
|
|
env: baselineEnv,
|
|
expectedDetail: "compact",
|
|
});
|
|
for (const operation of compactSmokeOperations) {
|
|
const candidateMeasurement = candidateCompact.measurements.get(operation);
|
|
const verboseMeasurement = candidateVerbose.measurements.get(operation);
|
|
const baselineMeasurement = baselineCompact.measurements.get(operation);
|
|
if (
|
|
candidateMeasurement === undefined ||
|
|
verboseMeasurement === undefined ||
|
|
baselineMeasurement === undefined
|
|
) {
|
|
throw new Error(`Missing comparison measurement for ${operation}.`);
|
|
}
|
|
reporter.recordOperation(
|
|
operation,
|
|
candidateMeasurement,
|
|
verboseMeasurement,
|
|
baselineMeasurement
|
|
);
|
|
}
|
|
const candidateContractPassed =
|
|
candidateCompact.contractFailures.length === 0;
|
|
const candidateVerboseContractPassed =
|
|
candidateVerbose.contractFailures.length === 0;
|
|
const baselineContractPassed =
|
|
baselineCompact.contractFailures.length === 0;
|
|
reporter.setContract(
|
|
"candidateCompact",
|
|
candidateContractPassed ? "passed" : "failed"
|
|
);
|
|
reporter.setContract(
|
|
"candidateVerbose",
|
|
candidateVerboseContractPassed ? "passed" : "failed"
|
|
);
|
|
reporter.setContract(
|
|
"historicalBaselineCompact",
|
|
baselineContractPassed ? "passed" : "failed"
|
|
);
|
|
reportPhase("compact-comparison", compactComparisonStartedAt);
|
|
if (candidateContractPassed === false) {
|
|
throw new Error(
|
|
`Compact output contract failed for candidate [${candidateCompact.contractFailures.join(", ")}].`
|
|
);
|
|
}
|
|
if (candidateVerboseContractPassed === false) {
|
|
throw new Error(
|
|
`Verbose semantic baseline failed for candidate [${candidateVerbose.contractFailures.join(", ")}].`
|
|
);
|
|
}
|
|
if (reporter.candidatePassesCompactRollout() === false) {
|
|
const report = reporter.createReport("failed");
|
|
const totals = report.sameCandidate.totals;
|
|
throw new Error(
|
|
`Candidate compact rollout gate failed: compact=${totals.compact.tokens} tokens, verbose=${totals.verbose.tokens} tokens, budget=${report.candidateTokenBudget}, retryDelta=${totals.retryDelta}, failureDelta=${totals.failureDelta}. Compact must be strictly smaller with no retry or failure regression.`
|
|
);
|
|
}
|
|
if (candidateCompact.canarySeen === false) {
|
|
throw new Error(
|
|
"Privacy canary was not exercised by the candidate compact list-pages response."
|
|
);
|
|
}
|
|
reporter.markCanaryExercised();
|
|
const invoke = async (label: string, args: string[]) => {
|
|
const result = await execFileAsync(bin, args, {
|
|
cwd: projectDirectory,
|
|
env,
|
|
maxBuffer: 20 * 1024 * 1024,
|
|
});
|
|
await assertJsonStdout(result.stdout, label);
|
|
return JSON.parse(result.stdout) as Record<string, unknown>;
|
|
};
|
|
|
|
const stagingPublish = await invoke("agent workflow staging publish", [
|
|
"publish",
|
|
"deploy",
|
|
"--target",
|
|
"staging",
|
|
"--json",
|
|
]);
|
|
if (
|
|
stagingPublish.ok !== true ||
|
|
getRecord(stagingPublish.data, "staging publish data").jobId !==
|
|
"release-smoke-staging-job"
|
|
) {
|
|
throw new Error(
|
|
`Agent connection workflow stopped at "staging publish": ${JSON.stringify(stagingPublish)} Recovery: Verify edit permission and retry webstudio publish deploy --target staging --json.`
|
|
);
|
|
}
|
|
|
|
const docs = await readFile(
|
|
join(import.meta.dirname, "../src/docs/api-use-cases.md"),
|
|
"utf8"
|
|
);
|
|
const documentedAuditCommands = docs
|
|
.split("\n")
|
|
.filter((line) => line.startsWith("- webstudio audit "))
|
|
.map((line) =>
|
|
line
|
|
.slice("- webstudio ".length)
|
|
.trim()
|
|
.split(/\s+/)
|
|
.filter((arg) => arg !== "--json")
|
|
);
|
|
if (documentedAuditCommands.length === 0) {
|
|
throw new Error("No documented audit CLI examples were found.");
|
|
}
|
|
const helpStartedAt = Date.now();
|
|
for (const args of documentedAuditCommands) {
|
|
const result = await execFileAsync(bin, [...args, "--help"], {
|
|
cwd: projectDirectory,
|
|
env,
|
|
});
|
|
if (!result.stdout.includes("webstudio audit")) {
|
|
throw new Error(
|
|
`Packed audit help is not audit-specific: ${result.stdout}`
|
|
);
|
|
}
|
|
}
|
|
reportPhase("audit-help", helpStartedAt);
|
|
|
|
const previewStartedAt = Date.now();
|
|
preview = spawn(bin, ["preview", "--port", "5190"], {
|
|
cwd: projectDirectory,
|
|
env,
|
|
detached: process.platform !== "win32",
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
await waitForUrl("http://127.0.0.1:5190/", preview);
|
|
const previewManifest = JSON.parse(
|
|
await readFile(
|
|
join(projectDirectory, ".webstudio/preview/package.json"),
|
|
"utf8"
|
|
)
|
|
) as { dependencies?: Record<string, string> };
|
|
if (
|
|
previewManifest.dependencies?.ipx !== undefined ||
|
|
previewManifest.dependencies?.h3 !== undefined
|
|
) {
|
|
throw new Error(
|
|
"Generic React Router preview unexpectedly includes Docker-only image proxy dependencies."
|
|
);
|
|
}
|
|
reportPhase("preview-start", previewStartedAt);
|
|
const commandsStartedAt = Date.now();
|
|
const assertLocalDryRun = async (
|
|
label: string,
|
|
command: string,
|
|
input: Record<string, unknown>
|
|
) => {
|
|
const payload = await invoke(label, [
|
|
command,
|
|
JSON.stringify(input),
|
|
"--dry-run",
|
|
]);
|
|
const meta = getRecord(payload.meta, `${label} meta`);
|
|
const session = getRecord(meta.session, `${label} session`);
|
|
if (
|
|
payload.ok !== true ||
|
|
session.source !== "dry-run" ||
|
|
session.committed !== false ||
|
|
typeof session.transaction !== "object"
|
|
) {
|
|
throw new Error(
|
|
`${label} did not produce a local dry-run transaction: ${JSON.stringify(payload)}`
|
|
);
|
|
}
|
|
};
|
|
await invoke("packed list pages", ["list-pages"]);
|
|
await assertLocalDryRun("plain page metadata update", "update-page", {
|
|
pageId: "home",
|
|
values: {
|
|
title: "My title: seven pages & more",
|
|
meta: { description: "Ordinary prose, punctuation included." },
|
|
},
|
|
});
|
|
await assertLocalDryRun("cold page duplicate", "duplicate-page", {
|
|
pageId: "home",
|
|
name: "Home copy",
|
|
path: "/home-copy",
|
|
});
|
|
await assertLocalDryRun("local fragment insertion", "insert-fragment", {
|
|
parentInstanceId: "home-root",
|
|
fragment: '<ws.element ws:tag="p">Release smoke</ws.element>',
|
|
});
|
|
await assertLocalDryRun("local component insertion", "insert-component", {
|
|
parentInstanceId: "home-root",
|
|
component: "ws:element",
|
|
tag: "p",
|
|
});
|
|
|
|
await stopProcessTree(preview);
|
|
preview = undefined;
|
|
await rm(join(projectDirectory, ".webstudio", "project-session.json"), {
|
|
force: true,
|
|
});
|
|
const stdioStartedAt = Date.now();
|
|
for (const { name: clientName } of agentSmokeClients) {
|
|
await runAgentWorkflowStep({
|
|
step: `${clientName} connection smoke`,
|
|
recovery: `Reload ${clientName}, then verify its Webstudio MCP command starts in the linked project folder.`,
|
|
run: async () => {
|
|
const { client, transport } = createMcpSmokeClient({
|
|
bin,
|
|
cwd: projectDirectory,
|
|
env,
|
|
name: clientName,
|
|
});
|
|
transport.stderr?.on("data", () => {});
|
|
await client.connect(transport);
|
|
try {
|
|
const tools = await client.listTools();
|
|
if (
|
|
tools.tools.some(({ name }) => name === "meta.index") === false
|
|
) {
|
|
throw new Error("Webstudio connected but meta.index is missing.");
|
|
}
|
|
} finally {
|
|
await client.close();
|
|
}
|
|
},
|
|
});
|
|
}
|
|
const { client: stdioClient, transport: stdioTransport } =
|
|
createMcpSmokeClient({
|
|
bin,
|
|
cwd: projectDirectory,
|
|
env,
|
|
name: "webstudio-release-smoke",
|
|
});
|
|
let stdioLogs = "";
|
|
stdioTransport.stderr?.on("data", (chunk) => {
|
|
stdioLogs += String(chunk);
|
|
});
|
|
const callTool = async (
|
|
name: string,
|
|
args: Record<string, unknown> = {}
|
|
) => {
|
|
const result = await stdioClient.callTool({ name, arguments: args });
|
|
const structured = getRecord(
|
|
result.structuredContent,
|
|
`${name} structured content`
|
|
);
|
|
if (result.isError === true || structured.ok !== true) {
|
|
throw new Error(
|
|
`${name} failed over packed stdio MCP: ${JSON.stringify(structured)}`
|
|
);
|
|
}
|
|
return getRecord(structured.data, `${name} data`);
|
|
};
|
|
const listAllPages = async () => {
|
|
const pages: unknown[] = [];
|
|
let cursor: string | undefined;
|
|
do {
|
|
const result = await callTool("list-pages", { cursor, limit: 200 });
|
|
if (Array.isArray(result.pages) === false) {
|
|
throw new Error(
|
|
`list-pages returned invalid data: ${JSON.stringify(result)}`
|
|
);
|
|
}
|
|
pages.push(...result.pages);
|
|
cursor =
|
|
typeof result.nextCursor === "string" ? result.nextCursor : undefined;
|
|
} while (cursor !== undefined);
|
|
return pages;
|
|
};
|
|
await stdioClient.connect(stdioTransport);
|
|
try {
|
|
const inspection = await callTool("inspect");
|
|
if (inspection.id !== projectId) {
|
|
throw new Error(
|
|
`Inspect returned the wrong project: ${JSON.stringify(inspection)}`
|
|
);
|
|
}
|
|
const initialPages = await listAllPages();
|
|
if (initialPages.length !== fixture.data.pages.length) {
|
|
throw new Error(
|
|
`Cold stdio discovery returned unexpected pages: ${JSON.stringify(initialPages)}`
|
|
);
|
|
}
|
|
await callTool("refresh", {
|
|
namespaces: ["pages", "instances", "props", "assets"],
|
|
});
|
|
const assets = await callTool("list-assets");
|
|
if (Array.isArray(assets.items) === false || assets.items.length !== 1) {
|
|
throw new Error(
|
|
`Cold stdio asset discovery failed: ${JSON.stringify(assets)}`
|
|
);
|
|
}
|
|
|
|
const createdPages: Array<{
|
|
pageId: string;
|
|
name: string;
|
|
path: string;
|
|
}> = [];
|
|
const duplicate = await callTool("duplicate-page", {
|
|
pageId: "home",
|
|
...duplicatedPage,
|
|
});
|
|
createdPages.push({
|
|
pageId: String(duplicate.pageId),
|
|
...duplicatedPage,
|
|
});
|
|
for (const page of createdPageInputs) {
|
|
const created = await callTool("create-page", page);
|
|
createdPages.push({ pageId: String(created.pageId), ...page });
|
|
}
|
|
|
|
for (const page of createdPages) {
|
|
await callTool("update-page", {
|
|
pageId: page.pageId,
|
|
values: {
|
|
title: `${page.name}: release smoke & verification`,
|
|
meta: { description: `A plain-text description for ${page.name}.` },
|
|
},
|
|
});
|
|
const details = await callTool("get-page", { pageId: page.pageId });
|
|
const inserted = await callTool("insert-component", {
|
|
parentInstanceId: String(details.rootInstanceId),
|
|
component: "ws:element",
|
|
tag: "section",
|
|
});
|
|
const sectionId = String(
|
|
(inserted.rootInstanceIds as unknown[] | undefined)?.[0]
|
|
);
|
|
await callTool("update-props", {
|
|
updates: [
|
|
{
|
|
instanceId: sectionId,
|
|
name: "data-imported-page",
|
|
type: "string",
|
|
value: page.name,
|
|
},
|
|
],
|
|
});
|
|
}
|
|
|
|
await callTool("update-text", {
|
|
instanceId: "pricing-root",
|
|
childIndex: 0,
|
|
text: "Pricing imported content",
|
|
mode: "text",
|
|
});
|
|
|
|
for (let index = 0; index < 10; index += 1) {
|
|
await callTool("list-pages");
|
|
await callTool("update-page", {
|
|
pageId: "pricing",
|
|
values: { title: `Pricing session check ${index}` },
|
|
});
|
|
}
|
|
const committedPricing = await callTool("get-page", {
|
|
pageId: "pricing",
|
|
});
|
|
await callTool("refresh", { namespaces: ["pages"] });
|
|
const refreshedPricing = await callTool("get-page", {
|
|
pageId: "pricing",
|
|
});
|
|
if (refreshedPricing.title !== committedPricing.title) {
|
|
throw new Error(
|
|
`Committed page title did not survive refresh: before=${JSON.stringify(committedPricing)} after=${JSON.stringify(refreshedPricing)}`
|
|
);
|
|
}
|
|
|
|
const usage = await callTool("find-asset-usage", {
|
|
assetId: "asset-hero",
|
|
});
|
|
if (Array.isArray(usage.usages) === false || usage.usages.length < 2) {
|
|
throw new Error(
|
|
`Duplicated page lost its image asset reference: ${JSON.stringify(usage)}`
|
|
);
|
|
}
|
|
|
|
await callTool("preview.start", {
|
|
host: "127.0.0.1",
|
|
port: 5191,
|
|
source: "session",
|
|
});
|
|
const workflowScreenshot = await callTool("screenshot", {
|
|
path: "/work",
|
|
output: ".webstudio/workflow-work.png",
|
|
viewport: { width: 1280, height: 800 },
|
|
waitUntil: "load",
|
|
});
|
|
await assertScreenshotEvidence({
|
|
payload: { data: workflowScreenshot },
|
|
label: "seven-page stdio workflow screenshot",
|
|
path: "/work",
|
|
status: 200,
|
|
viewport: { width: 1280, height: 800 },
|
|
projectDirectory,
|
|
origin: "http://127.0.0.1:5191",
|
|
});
|
|
await callTool("preview.stop");
|
|
|
|
const workflowDuration = Date.now() - workflowStartedAt;
|
|
if (workflowDuration >= 10 * 60_000) {
|
|
throw new Error(
|
|
`Agent connection workflow exceeded ten minutes (${workflowDuration}ms). Recovery: Inspect the reported release-smoke phase timings and optimize the slowest blocked step.`
|
|
);
|
|
}
|
|
reportPhase("agent-workflow", workflowStartedAt);
|
|
|
|
const finalPages = await listAllPages();
|
|
const finalPageNames = finalPages.map(
|
|
(page) => getRecord(page, "final page").name
|
|
);
|
|
if (
|
|
finalPageNames.length !==
|
|
fixture.data.pages.length + createdPages.length ||
|
|
createdPages.some(
|
|
(page) => finalPageNames.includes(page.name) === false
|
|
)
|
|
) {
|
|
throw new Error(
|
|
`Seven-page workflow verification failed: ${JSON.stringify(finalPages)}`
|
|
);
|
|
}
|
|
} finally {
|
|
await stdioClient.close();
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
if (
|
|
stdioLogs.includes("API contract negotiated") === false ||
|
|
stdioLogs.includes('"event":"stdio_connection_closed"') === false ||
|
|
stdioLogs.includes('"event":"stdio_transport_error"')
|
|
) {
|
|
throw new Error(`Unexpected packed stdio lifecycle logs:\n${stdioLogs}`);
|
|
}
|
|
reportPhase("stdio-workflow", stdioStartedAt);
|
|
|
|
preview = spawn(bin, ["preview", "--port", "5190"], {
|
|
cwd: projectDirectory,
|
|
env,
|
|
detached: process.platform !== "win32",
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
await waitForUrl("http://127.0.0.1:5190/", preview);
|
|
|
|
const pricingScreenshot = await invoke("standalone screenshot", [
|
|
"screenshot",
|
|
"http://127.0.0.1:5190/pricing",
|
|
"--output",
|
|
".webstudio/pricing.png",
|
|
"--width",
|
|
"1440",
|
|
"--height",
|
|
"900",
|
|
"--json",
|
|
]);
|
|
await assertScreenshotEvidence({
|
|
payload: pricingScreenshot,
|
|
label: "standalone pricing screenshot",
|
|
path: "/pricing",
|
|
status: 200,
|
|
viewport: { width: 1440, height: 900 },
|
|
projectDirectory,
|
|
});
|
|
const homeScreenshot = await invoke("MCP screenshot", [
|
|
"mcp",
|
|
"single-op-call",
|
|
"screenshot",
|
|
JSON.stringify({
|
|
baseUrl: "http://127.0.0.1:5190",
|
|
path: "/",
|
|
viewport: { width: 375, height: 812 },
|
|
}),
|
|
]);
|
|
await assertScreenshotEvidence({
|
|
payload: homeScreenshot,
|
|
label: "MCP home screenshot",
|
|
path: "/",
|
|
status: 200,
|
|
viewport: { width: 375, height: 812 },
|
|
projectDirectory,
|
|
});
|
|
await invoke("MCP run", [
|
|
"mcp",
|
|
"run",
|
|
JSON.stringify([
|
|
{
|
|
tool: "screenshot",
|
|
input: {
|
|
baseUrl: "http://127.0.0.1:5190",
|
|
path: "/pricing",
|
|
viewport: { width: 1440, height: 900 },
|
|
},
|
|
},
|
|
]),
|
|
]);
|
|
const missingScreenshot = await invoke("missing route screenshot", [
|
|
"screenshot",
|
|
"http://127.0.0.1:5190/missing",
|
|
"--output",
|
|
".webstudio/missing.png",
|
|
"--width",
|
|
"375",
|
|
"--height",
|
|
"812",
|
|
"--json",
|
|
]);
|
|
await assertScreenshotEvidence({
|
|
payload: missingScreenshot,
|
|
label: "missing route screenshot",
|
|
path: "/missing",
|
|
status: 404,
|
|
viewport: { width: 375, height: 812 },
|
|
projectDirectory,
|
|
generatedSite: false,
|
|
});
|
|
const renderedAudit = await invoke("rendered audit", [
|
|
"audit",
|
|
"--rendered",
|
|
"--page-path",
|
|
"/",
|
|
"--json",
|
|
]);
|
|
const renderedAuditData = getRecord(
|
|
renderedAudit.data,
|
|
"rendered audit data"
|
|
);
|
|
const renderedAuditSummary = getRecord(
|
|
renderedAuditData.renderedCaptureSummary,
|
|
"rendered audit capture summary"
|
|
);
|
|
if (
|
|
renderedAudit.ok !== true ||
|
|
renderedAuditData.renderedCheckCount !== 1 ||
|
|
renderedAuditSummary.planned !== 1 ||
|
|
renderedAuditSummary.failed !== 0 ||
|
|
renderedAuditSummary.skipped !== 0
|
|
) {
|
|
throw new Error(
|
|
`Rendered audit was incomplete: ${JSON.stringify(renderedAuditData)}`
|
|
);
|
|
}
|
|
if (process.env.WEBSTUDIO_RELEASE_SMOKE_LARGE_AUDIT === "1") {
|
|
const expectedCaptureCount =
|
|
fixture.data.pages.length + additionalPageCount;
|
|
const planPayload = await invoke("large rendered audit plan", [
|
|
"audit",
|
|
"--rendered",
|
|
"--scopes",
|
|
"performance",
|
|
"--json",
|
|
]);
|
|
const planData = getRecord(planPayload.data, "large audit plan data");
|
|
const plan = getRecord(planData.renderedPlan, "large rendered plan");
|
|
if (
|
|
plan.captureCount !== expectedCaptureCount ||
|
|
typeof plan.confirmationToken !== "string"
|
|
) {
|
|
throw new Error(
|
|
`Large rendered audit did not return the expected ${expectedCaptureCount}-capture confirmation plan: ${JSON.stringify(plan)}`
|
|
);
|
|
}
|
|
const confirmedAuditStartedAt = Date.now();
|
|
const confirmedPayload = await invoke("confirmed large rendered audit", [
|
|
"audit",
|
|
"--rendered",
|
|
"--scopes",
|
|
"performance",
|
|
"--confirm-large-run",
|
|
"--confirmation-token",
|
|
plan.confirmationToken,
|
|
"--json",
|
|
]);
|
|
const confirmedData = getRecord(
|
|
confirmedPayload.data,
|
|
"confirmed large audit data"
|
|
);
|
|
const summary = getRecord(
|
|
confirmedData.renderedCaptureSummary,
|
|
"confirmed large audit capture summary"
|
|
);
|
|
const manifestSummary = getRecord(
|
|
confirmedData.renderedArtifactManifest,
|
|
"confirmed large audit artifact manifest"
|
|
);
|
|
const manifestPath = String(manifestSummary.path);
|
|
const absoluteManifestPath = manifestPath.startsWith("/")
|
|
? manifestPath
|
|
: join(projectDirectory, manifestPath);
|
|
const manifest = JSON.parse(
|
|
await readFile(absoluteManifestPath, "utf8")
|
|
) as Record<string, unknown>;
|
|
if (
|
|
confirmedPayload.ok !== true ||
|
|
confirmedData.renderedCheckCount !== expectedCaptureCount ||
|
|
summary.planned !== expectedCaptureCount ||
|
|
summary.failed !== 0 ||
|
|
summary.skipped !== 0
|
|
) {
|
|
throw new Error(
|
|
`Confirmed large rendered audit was incomplete: ${JSON.stringify({ summary, failures: manifest.failures })}`
|
|
);
|
|
}
|
|
const memory = process.memoryUsage();
|
|
const buildMetrics = isPlainRecord(manifest.generatedBuildMetrics)
|
|
? manifest.generatedBuildMetrics
|
|
: undefined;
|
|
reportPhase("rendered-audit", confirmedAuditStartedAt, {
|
|
rssBytes: memory.rss,
|
|
heapUsedBytes: memory.heapUsed,
|
|
networkTransferBytes: getRenderedNetworkTransferBytes(manifest),
|
|
retainedArtifactBytes: await getRetainedArtifactBytes({
|
|
manifest,
|
|
manifestPath: absoluteManifestPath,
|
|
projectDirectory,
|
|
}),
|
|
...(typeof buildMetrics?.bytes === "number"
|
|
? { generatedBuildBytes: buildMetrics.bytes }
|
|
: {}),
|
|
...(typeof buildMetrics?.gzipBytes === "number"
|
|
? { generatedBuildGzipBytes: buildMetrics.gzipBytes }
|
|
: {}),
|
|
});
|
|
const confirmedAuditDuration = Date.now() - confirmedAuditStartedAt;
|
|
if (confirmedAuditDuration > 120_000) {
|
|
throw new Error(
|
|
`Confirmed ${expectedCaptureCount}-capture packaged audit took ${confirmedAuditDuration}ms; expected at most 120000ms.`
|
|
);
|
|
}
|
|
const cachedAuditStartedAt = Date.now();
|
|
await invoke("cached rendered audit", [
|
|
"audit",
|
|
"--rendered",
|
|
"--page-path",
|
|
"/",
|
|
"--scopes",
|
|
"performance",
|
|
"--json",
|
|
]);
|
|
reportPhase("cached-audit", cachedAuditStartedAt);
|
|
const cachedAuditDuration = Date.now() - cachedAuditStartedAt;
|
|
if (cachedAuditDuration > 20_000) {
|
|
throw new Error(
|
|
`Cached rendered audit took ${cachedAuditDuration}ms; expected at most 20000ms.`
|
|
);
|
|
}
|
|
}
|
|
reportPhase("structured-commands", commandsStartedAt);
|
|
} finally {
|
|
await stopProcessTree(preview);
|
|
await closeApiServer?.();
|
|
await rm(temp, { recursive: true, force: true });
|
|
reportPhase("cleanup");
|
|
}
|
|
};
|
|
|
|
let outcome: "passed" | "failed" = "failed";
|
|
try {
|
|
await run();
|
|
outcome = "passed";
|
|
} finally {
|
|
const reportPath = process.env.WEBSTUDIO_RELEASE_SMOKE_REPORT;
|
|
if (reportPath !== undefined && reportPath !== "") {
|
|
await reporter.write(reportPath, outcome);
|
|
}
|
|
}
|