From cc10a4e893d446e1566dbbfdbe576d6b9aa9dbca Mon Sep 17 00:00:00 2001 From: Oleg Isonen Date: Thu, 23 Jul 2026 01:28:59 +0100 Subject: [PATCH] fix: harden asset resource publication --- .../asset-query-form-utils.test.ts | 8 + .../settings-panel/asset-query-form-utils.ts | 12 +- .../settings-panel/asset-query-form.tsx | 16 +- apps/builder/app/shared/db/canvas.server.ts | 137 ++++++++++++--- .../docs/asset-resource-architecture.md | 17 ++ packages/asset-resource/package.json | 28 ++- packages/asset-resource/src/canonical.test.ts | 33 ++-- packages/asset-resource/src/canonical.ts | 32 ++-- packages/asset-resource/src/field-catalog.ts | 7 + packages/asset-resource/src/index.ts | 162 +---------------- packages/asset-resource/src/indexing.ts | 5 + .../src/published-runtime.test.ts | 17 ++ .../asset-resource/src/published-runtime.ts | 44 ++++- .../asset-resource/src/query-execution.ts | 163 ++++++++++++++++++ .../src/query-validation.test.ts | 13 ++ .../asset-resource/src/query-validation.ts | 65 +++++++ packages/asset-resource/src/query.ts | 2 + .../src/canonical-metadata-backfill.test.ts | 15 +- .../src/canonical-metadata-backfill.ts | 29 +++- .../asset-uploader/src/query-preview.test.ts | 44 +++++ packages/asset-uploader/src/query-preview.ts | 15 +- .../resource-index-garbage-collection.test.ts | 4 +- .../src/resource-index-garbage-collection.ts | 11 +- .../src/resource-index-lifecycle.test.ts | 6 +- .../src/resource-index-lifecycle.ts | 5 +- .../src/resource-index-maintenance.ts | 10 +- packages/asset-uploader/src/upload.ts | 26 ++- .../cli/src/commands/api-command-metadata.ts | 11 ++ packages/cli/src/commands/api-command.ts | 39 +++++ packages/cli/src/commands/man.ts | 3 + packages/cli/src/framework-vike-ssg.ts | 41 +++-- packages/cli/src/framework.ts | 5 +- packages/cli/src/prebuild.test.ts | 140 +++++++++++++-- packages/cli/src/prebuild.ts | 151 +++++++++++++++- packages/cli/src/template-placeholders.d.ts | 4 +- .../defaults/app/route-templates/html.tsx | 3 +- .../defaults/app/route-templates/text.tsx | 3 +- .../defaults/app/route-templates/xml.tsx | 3 +- .../react-router/app/route-templates/html.tsx | 3 +- .../react-router/app/route-templates/text.tsx | 3 +- .../react-router/app/route-templates/xml.tsx | 3 +- .../templates/ssg/app/asset-resource-fetch.ts | 2 +- packages/http-client/src/index.test.ts | 4 + packages/http-client/src/index.ts | 6 +- .../supabase/tests/asset-resource-indexes.sql | 98 ++++++++++- .../migration.sql | 97 ++++++++++- packages/project-build/package.json | 1 + .../src/runtime/asset-resources.test.ts | 58 +++++++ .../src/runtime/asset-resources.ts | 61 ++++++- packages/sdk/src/asset-resource-config.ts | 3 +- packages/sdk/src/schema/asset-resource.ts | 9 + pnpm-lock.yaml | 3 + 52 files changed, 1367 insertions(+), 313 deletions(-) create mode 100644 packages/asset-resource/src/indexing.ts create mode 100644 packages/asset-resource/src/query-execution.ts create mode 100644 packages/asset-resource/src/query.ts create mode 100644 packages/project-build/src/runtime/asset-resources.test.ts diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts index f67956d7bf..3f61db22f6 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.test.ts @@ -146,5 +146,13 @@ describe("asset query resource body", () => { content: { mode: "none" }, }) ).toContain("$slug"); + expect( + getAssetQueryConfigurationError({ + query: "*[properties.slug == $slug]", + parameters: [{ name: "slug", value: "system.params." }], + resultLimit: 1, + content: { mode: "none" }, + }) + ).toContain("Webstudio expression"); }); }); diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts index 1b4368ea86..6148ba4774 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form-utils.ts @@ -7,6 +7,7 @@ import { type AssetResourceIndexStatus, } from "@webstudio-is/sdk"; import { validateAssetResourceQuery } from "@webstudio-is/asset-resource"; +import { getExpressionErrorMessages } from "@webstudio-is/project-build/runtime"; export { createAssetQueryResourceBody, parseAssetQueryResourceBody, @@ -58,7 +59,7 @@ export const getAssetQueryConfigurationError = ({ content, }: { query: string; - parameters: readonly { name: string }[]; + parameters: readonly { name: string; value?: string }[]; resultLimit: number; content: AssetResourceContentOptions; }) => { @@ -84,6 +85,15 @@ export const getAssetQueryConfigurationError = ({ if (new Set(names).size !== names.length) { return "Runtime parameter names must be unique."; } + if ( + parameters.some( + ({ value }) => + value !== undefined && + getExpressionErrorMessages({ expression: value }).length > 0 + ) + ) { + return "Enter a valid Webstudio expression for every runtime parameter."; + } const configuredNames = new Set(names); const missingName = referencedParameterNames.find( (name) => configuredNames.has(name) === false diff --git a/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx b/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx index 3a9700f313..596fb9382d 100644 --- a/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx +++ b/apps/builder/app/builder/features/settings-panel/asset-query-form.tsx @@ -306,6 +306,13 @@ const AssetQueryPreview = ({ } let ignore = false; let pollTimeout: ReturnType | undefined; + let pollAttempts = 0; + const schedulePoll = () => { + pollAttempts += 1; + if (pollAttempts < 10) { + pollTimeout = setTimeout(load, 1000); + } + }; const load = async () => { try { const response = await loadBuilderAssetIndexStatus(resourceId); @@ -316,12 +323,13 @@ const AssetQueryPreview = ({ const parsed = assetResourceIndexStatus.safeParse(data.status); const status = parsed.success ? parsed.data : undefined; onIndexStatusChange(status); - if (status?.state === "indexing") { - pollTimeout = setTimeout(load, 1000); + if (status?.state === "indexing" || status === undefined) { + schedulePoll(); } } catch { if (ignore === false) { onIndexStatusChange(undefined); + schedulePoll(); } } }; @@ -448,7 +456,9 @@ export const AssetQueryForm = ({ {} ); const [resultLimit, setResultLimit] = useState( - typeof parsedResultLimit === "number" ? parsedResultLimit : 100 + typeof parsedResultLimit === "number" + ? parsedResultLimit + : assetResourceLimits.defaultResultCount ); const parsedContent = assetResourceContentOptions.safeParse( evaluateExpressionWithinScope(parsedBody.contentExpression ?? "", {}) diff --git a/apps/builder/app/shared/db/canvas.server.ts b/apps/builder/app/shared/db/canvas.server.ts index 679f6ca815..ee8c797201 100644 --- a/apps/builder/app/shared/db/canvas.server.ts +++ b/apps/builder/app/shared/db/canvas.server.ts @@ -20,12 +20,15 @@ import { findPageByIdOrPath, getAllPages, getStyleDeclKey, + assetResourceLimits, + isStoredAssetQueryResource, type Asset, type AssetFolder, } from "@webstudio-is/sdk"; import { computeAssetResourceQueryHash, computeCanonicalAssetRevision, + serializeAssetResourceIndex, } from "@webstudio-is/asset-resource"; import { serializePages } from "@webstudio-is/project-migrations/pages"; import { loadById } from "@webstudio-is/project/index.server"; @@ -164,25 +167,42 @@ const addProjectMetadata = async ( ? undefined : await getUserById(context, project.userId); + const reachableResourceIds = new Set(); + for (const [, prop] of data.build.props) { + if (prop.type === "resource") { + reachableResourceIds.add(prop.value); + } + } + for (const [, dataSource] of data.build.dataSources) { + if (dataSource.type === "resource") { + reachableResourceIds.add(dataSource.resourceId); + } + } const assetQueryResources = data.build.resources .map(([, resource]) => resource) .flatMap((resource) => { + if (reachableResourceIds.has(resource.id) === false) { + return []; + } const query = getAssetResourceQuery(resource); - return query === undefined ? [] : [{ resourceId: resource.id, query }]; + if (query !== undefined) { + return [{ resourceId: resource.id, query }]; + } + if (isStoredAssetQueryResource(resource)) { + throw new Error( + `Assets resource ${JSON.stringify(resource.id)} has an invalid query configuration` + ); + } + return []; }); + if (assetQueryResources.length > assetResourceLimits.publishedResourceCount) { + throw new Error("Published Assets resource count exceeds the limit"); + } let assetResourceIndexes: PublishedProjectBundle["assetResourceIndexes"]; + let publishedAssets = data.assets; + let publishedAssetFolders = data.assetFolders; if (assetQueryResources.length > 0) { const assetClient = createAssetClientWithResourceIndexStore(); - await synchronizeCanonicalAssets({ - client: context.postgrest.client, - projectId: project.id, - assetClient, - }); - const { entries: canonicalEntries, metadataSnapshot } = - await loadCanonicalAssetFileSnapshot({ - client: context.postgrest.client, - projectId: project.id, - }); const indexedResources = await Promise.all( assetQueryResources.map(async ({ resourceId, query }) => ({ resourceId, @@ -190,24 +210,97 @@ const addProjectMetadata = async ( queryHash: await computeAssetResourceQueryHash(query), })) ); - const expectedAssetRevision = - await computeCanonicalAssetRevision(canonicalEntries); - assetResourceIndexes = - await prepareAssetResourceIndexSnapshotsForPublication({ + const currentBuild = await loadDevBuildByProjectId(context, project.id); + const currentQueries = new Map( + (currentBuild?.resources ?? []).map((resource) => [ + resource.id, + getAssetResourceQuery(resource), + ]) + ); + const currentResourceIds = indexedResources + .filter( + ({ resourceId, query }) => currentQueries.get(resourceId) === query + ) + .map(({ resourceId }) => resourceId); + const retainedFontIds = new Set( + data.assets + .filter((asset) => asset.type === "font") + .map((asset) => asset.id) + ); + for (let attempt = 0; attempt < 2; attempt += 1) { + const assetDataBefore = await loadAssetDataByProject(project.id, context); + await synchronizeCanonicalAssets({ client: context.postgrest.client, - store: assetClient.resourceIndexStore, projectId: project.id, - resources: indexedResources, - entries: canonicalEntries, - read: assetClient.resourceIndexStore.read, - referenceId: data.build.id, - assetRevision: expectedAssetRevision, - metadataSnapshot, + assetClient, }); + const { entries: canonicalEntries, metadataSnapshot } = + await loadCanonicalAssetFileSnapshot({ + client: context.postgrest.client, + projectId: project.id, + }); + const expectedAssetRevision = + await computeCanonicalAssetRevision(canonicalEntries); + const preparedIndexes = + await prepareAssetResourceIndexSnapshotsForPublication({ + client: context.postgrest.client, + store: assetClient.resourceIndexStore, + projectId: project.id, + resources: indexedResources, + entries: canonicalEntries, + read: assetClient.resourceIndexStore.read, + referenceId: data.build.id, + assetRevision: expectedAssetRevision, + metadataSnapshot, + currentResourceIds, + }); + const [assetDataAfter, latestCanonical] = await Promise.all([ + loadAssetDataByProject(project.id, context), + loadCanonicalAssetFileSnapshot({ + client: context.postgrest.client, + projectId: project.id, + }), + ]); + const latestAssetRevision = await computeCanonicalAssetRevision( + latestCanonical.entries + ); + const assetsStayedStable = + JSON.stringify(assetDataBefore) === JSON.stringify(assetDataAfter) && + expectedAssetRevision === latestAssetRevision; + if (assetsStayedStable === false) { + if (attempt === 0) { + continue; + } + throw new Error("Assets changed while preparing publication; retry"); + } + assetResourceIndexes = preparedIndexes; + publishedAssets = assetDataAfter.assets.filter( + (asset) => asset.type !== "font" || retainedFontIds.has(asset.id) + ); + publishedAssetFolders = assetDataAfter.assetFolders; + break; + } + if (assetResourceIndexes === undefined) { + throw new Error("Asset resource indexes were not prepared"); + } + const totalIndexBytes = assetResourceIndexes.reduce( + (total, snapshot) => + total + + new TextEncoder().encode(serializeAssetResourceIndex(snapshot.index)) + .byteLength, + 0 + ); + if (totalIndexBytes > assetResourceLimits.publishedIndexBytes) { + throw new Error( + "Published Assets resource indexes exceed the byte limit" + ); + } } return { ...data, + assets: publishedAssets, + assetFolders: publishedAssetFolders, bundleVersion, user: user ? { email: user.email } : undefined, projectDomain: project.domain, diff --git a/apps/builder/docs/asset-resource-architecture.md b/apps/builder/docs/asset-resource-architecture.md index c5d3639554..a5b85a1238 100644 --- a/apps/builder/docs/asset-resource-architecture.md +++ b/apps/builder/docs/asset-resource-architecture.md @@ -286,6 +286,10 @@ claims are 15-minute leases. A later cleanup worker rotates and resumes an expired claim, so a crash before or after object deletion cannot permanently strand the revision row. +Each inline cleanup invocation processes at most one bounded batch. Production +must also invoke the same collector from scheduled background work so cleanup +does not depend on later Builder or publication traffic. + ## Schemaless frontmatter and publication Frontmatter fields have no built-in publication meaning. Names such as `draft`, @@ -313,6 +317,9 @@ single source for Builder, indexer, generated runtime, and test limits. V1 uses: | Serialized result | 1 MiB | | Candidate documents | 1,000 | | Serialized index | 16 MiB | +| Published query resources | 32 | +| Published indexes combined | 32 MiB | +| Parsed indexes per isolate | 4, least-recently used | | Frontmatter per file | 64 KiB | | Frontmatter structure | Depth 8, 256 fields | | One frontmatter string | 16 KiB UTF-8 | @@ -648,6 +655,16 @@ server-only adapter reads immutable indexes and ranged Markdown bytes from the generated `public` directory, so prerendering requires neither a live HTTP endpoint nor bundling content into JavaScript. Range hydration uses positioned file reads rather than loading the complete source asset before slicing it. +Dynamic SSG pages are enumerated when every required `:parameter` is bound +directly from `system.params.` and its GROQ parameter is compared directly +with one canonical document field, for example +`properties.slug == $slug`. Prebuild fails explicitly when a dynamic route has +no safely enumerable Assets resource instead of silently omitting its pages. + +Redeploying a historical build preserves that build's stored query and +application structure but evaluates it against the project's latest assets. +This matches the existing non-versioned asset contract; it is not an exact +historical content rollback. Public index construction applies only the configured GROQ candidate selection. Prebuild materializes Markdown through the same static asset pipeline as other diff --git a/packages/asset-resource/package.json b/packages/asset-resource/package.json index 559bbd07d3..220989d881 100644 --- a/packages/asset-resource/package.json +++ b/packages/asset-resource/package.json @@ -11,7 +11,7 @@ "benchmark:groq": "node scripts/benchmark-groq.mjs", "benchmark:index": "NODE_OPTIONS=--conditions=webstudio pnpm tsx scripts/benchmark-index.mjs", "benchmark:scale": "NODE_OPTIONS=--conditions=webstudio pnpm tsx scripts/benchmark-scale.mjs", - "build": "rm -rf lib && esbuild src/index.ts --outdir=lib --bundle --format=esm --packages=external", + "build": "rm -rf lib && esbuild src/index.ts src/published-runtime.ts src/query.ts src/markdown.ts src/indexing.ts --outdir=lib --bundle --format=esm --packages=external", "dts": "tsc --project tsconfig.dts.json", "test": "vitest run", "typecheck": "tsc --noEmit" @@ -35,10 +35,34 @@ "types": "./lib/types/index.d.ts", "import": "./lib/index.js", "default": "./src/index.ts" + }, + "./runtime": { + "webstudio": "./src/published-runtime.ts", + "types": "./lib/types/published-runtime.d.ts", + "import": "./lib/published-runtime.js", + "default": "./src/published-runtime.ts" + }, + "./query": { + "webstudio": "./src/query.ts", + "types": "./lib/types/query.d.ts", + "import": "./lib/query.js", + "default": "./src/query.ts" + }, + "./markdown": { + "webstudio": "./src/markdown.ts", + "types": "./lib/types/markdown.d.ts", + "import": "./lib/markdown.js", + "default": "./src/markdown.ts" + }, + "./indexing": { + "webstudio": "./src/indexing.ts", + "types": "./lib/types/indexing.d.ts", + "import": "./lib/indexing.js", + "default": "./src/indexing.ts" } }, "files": [ - "lib/*", + "lib/**/*", "!*.test.*" ] } diff --git a/packages/asset-resource/src/canonical.test.ts b/packages/asset-resource/src/canonical.test.ts index 2c3a9257ca..de1ade8047 100644 --- a/packages/asset-resource/src/canonical.test.ts +++ b/packages/asset-resource/src/canonical.test.ts @@ -63,17 +63,28 @@ describe("normalizeAssetFileDocument", () => { }); }); - test.each(["", ".", "..", "nested/name", "nested\\name"])( - "rejects invalid path segment %j", - (name) => { - expect(() => - normalizeAssetFileDocument({ - asset: { ...asset, name }, - properties: {}, - }) - ).toThrow("one normalized path segment"); - } - ); + test.each([ + [".", "%2E"], + ["..", "%2E%2E"], + ["nested/name", "nested%2Fname"], + ["nested\\name", "nested%5Cname"], + ])("encodes user-defined path segment %j", (name, path) => { + expect( + normalizeAssetFileDocument({ + asset: { ...asset, name, folderId: undefined, folderNames: undefined }, + properties: {}, + }).path + ).toBe(path); + }); + + test("rejects an empty asset name", () => { + expect(() => + normalizeAssetFileDocument({ + asset: { ...asset, name: "" }, + properties: {}, + }) + ).toThrow(); + }); test("rejects JSON-incompatible property values", () => { expect(() => diff --git a/packages/asset-resource/src/canonical.ts b/packages/asset-resource/src/canonical.ts index cd1560d0ef..dc6a53b019 100644 --- a/packages/asset-resource/src/canonical.ts +++ b/packages/asset-resource/src/canonical.ts @@ -107,16 +107,18 @@ export const createCanonicalAssetFileEntry = ({ }; }; -const assertPathSegment = (segment: string, label: string) => { - if ( - segment.length === 0 || - segment === "." || - segment === ".." || - segment.includes("/") || - segment.includes("\\") - ) { - throw new Error(`${label} must be one normalized path segment`); +// Asset and folder names are user data, not filesystem paths. Encode each name +// independently so slashes and dot segments cannot change path structure while +// preserving the original name in the document for display and exact matching. +const encodeAssetPathSegment = (segment: string) => { + const encoded = encodeURIComponent(segment); + if (encoded === ".") { + return "%2E"; } + if (encoded === "..") { + return "%2E%2E"; + } + return encoded; }; /** @@ -127,22 +129,19 @@ export const normalizeAssetFileDocument = ({ asset, properties, excerpt, + metadataError, }: { asset: AssetFileMetadataInput; properties: Record; excerpt?: string; + metadataError?: AssetFileDocument["metadataError"]; }): AssetFileDocument => { - assertPathSegment(asset.name, "Asset name"); const folderNames = asset.folderNames ?? []; if ((asset.folderId === undefined) !== (folderNames.length === 0)) { throw new Error( "Asset folder ID and complete folder-name path must be provided together" ); } - for (const folderName of folderNames) { - assertPathSegment(folderName, "Asset folder name"); - } - const extensionSeparator = asset.name.lastIndexOf("."); const extension = asset.extension ?? @@ -153,7 +152,9 @@ export const normalizeAssetFileDocument = ({ extensionSeparator <= 0 ? asset.name : asset.name.slice(0, extensionSeparator); - const path = [...folderNames, asset.name].join("/"); + const path = [...folderNames, asset.name] + .map(encodeAssetPathSegment) + .join("/"); return assetFileDocument.parse({ _id: asset.id, @@ -169,5 +170,6 @@ export const normalizeAssetFileDocument = ({ contentRef: asset.contentRef, properties, ...(excerpt === undefined ? {} : { excerpt }), + ...(metadataError === undefined ? {} : { metadataError }), }); }; diff --git a/packages/asset-resource/src/field-catalog.ts b/packages/asset-resource/src/field-catalog.ts index 83b210d48d..9a0b2bb7ad 100644 --- a/packages/asset-resource/src/field-catalog.ts +++ b/packages/asset-resource/src/field-catalog.ts @@ -79,6 +79,13 @@ const getStandardFieldContributions = ( ...(document.excerpt === undefined ? [] : [{ path: "excerpt", type: "string" } as const]), + ...(document.metadataError === undefined + ? [] + : [ + { path: "metadataError", type: "object" } as const, + { path: "metadataError.code", type: "string" } as const, + { path: "metadataError.message", type: "string" } as const, + ]), ]; type MutableField = { diff --git a/packages/asset-resource/src/index.ts b/packages/asset-resource/src/index.ts index 6b7274fd22..3dbc76a7a4 100644 --- a/packages/asset-resource/src/index.ts +++ b/packages/asset-resource/src/index.ts @@ -1,19 +1,9 @@ -import { evaluate, type ExprNode } from "groq-js/1"; import { - assetResourceLimits, assetResourceQueryRequest, - assetResourceQuerySuccess, - type AssetFileDocument, type AssetResourceQueryInput, - type AssetResourceQueryRequest, type ResourceRequest, } from "@webstudio-is/sdk"; import { assetsQueryResourceUrl } from "@webstudio-is/sdk/runtime"; -import { validateAssetResourceQuery } from "./query-validation"; -import { - hydrateAssetResourceResult, - type AssetResourceContentReader, -} from "./hydration"; export * from "./markdown"; export * from "./canonical"; @@ -23,6 +13,7 @@ export * from "./candidate-selection"; export * from "./index-storage"; export * from "./query-validation"; export * from "./hydration"; +export * from "./query-execution"; export * from "./published-runtime"; export const createAssetResourceRequest = ( @@ -36,154 +27,3 @@ export const createAssetResourceRequest = ( headers: [{ name: "content-type", value: "application/json" }], body: assetResourceQueryRequest.parse(request), }); - -const evaluateAssetResourceQuery = async ({ - tree, - documents, - parameters = {}, -}: { - tree: ExprNode; - documents: readonly AssetFileDocument[]; - parameters?: Record; -}) => { - const value = await evaluate(tree, { - dataset: documents, - params: parameters, - }); - return await value.get(); -}; - -export class AssetResourceQueryExecutionError extends Error { - readonly code: - | "MISSING_PARAMETER" - | "RESULT_LIMIT_EXCEEDED" - | "RESULT_SIZE_EXCEEDED"; - readonly details?: Record; - - constructor({ - code, - message, - details, - }: { - code: AssetResourceQueryExecutionError["code"]; - message: string; - details?: AssetResourceQueryExecutionError["details"]; - }) { - super(message); - this.name = "AssetResourceQueryExecutionError"; - this.code = code; - this.details = details; - } -} - -const getResultCount = (result: unknown) => { - if (Array.isArray(result)) { - return result.length; - } - return result === null || result === undefined ? 0 : 1; -}; - -export const executeAssetResourceQuery = async ({ - request, - documents, - queryHash, - indexRevision, - assetRevision, -}: { - request: AssetResourceQueryRequest; - documents: readonly AssetFileDocument[]; - queryHash: string; - indexRevision: string; - assetRevision: string; -}) => { - const validated = validateAssetResourceQuery(request.query); - for (const parameterName of validated.parameterNames) { - if (Object.hasOwn(request.parameters, parameterName) === false) { - throw new AssetResourceQueryExecutionError({ - code: "MISSING_PARAMETER", - message: `Asset resource parameter $${parameterName} is required`, - details: { parameter: parameterName }, - }); - } - } - if (documents.length > assetResourceLimits.candidateDocuments) { - throw new AssetResourceQueryExecutionError({ - code: "RESULT_LIMIT_EXCEEDED", - message: "Asset resource candidate document limit was exceeded", - details: { - candidateCount: documents.length, - candidateLimit: assetResourceLimits.candidateDocuments, - }, - }); - } - - const rawResult = await evaluateAssetResourceQuery({ - tree: validated.tree, - documents, - parameters: request.parameters, - }); - const result = rawResult ?? null; - const resultCount = getResultCount(result); - if (resultCount > request.resultLimit) { - throw new AssetResourceQueryExecutionError({ - code: "RESULT_LIMIT_EXCEEDED", - message: "Asset resource query returned too many results", - details: { resultCount, resultLimit: request.resultLimit }, - }); - } - const resultBytes = new TextEncoder().encode(JSON.stringify(result)).length; - if (resultBytes > assetResourceLimits.resultBytes) { - throw new AssetResourceQueryExecutionError({ - code: "RESULT_SIZE_EXCEEDED", - message: "Asset resource query result exceeds the byte limit", - details: { - resultBytes, - resultByteLimit: assetResourceLimits.resultBytes, - }, - }); - } - - return assetResourceQuerySuccess.parse({ - ok: true, - result, - content: {}, - meta: { - queryHash, - indexRevision, - assetRevision, - resultCount, - hydratedFileCount: 0, - hydratedBytes: 0, - }, - }); -}; - -export const executeAndHydrateAssetResourceQuery = async ({ - read, - ...input -}: Parameters[0] & { - read?: AssetResourceContentReader; -}) => { - const response = await executeAssetResourceQuery(input); - if (input.request.content.mode === "none") { - return response; - } - if (read === undefined) { - throw new Error("Asset content reader is required for hydration"); - } - const hydration = await hydrateAssetResourceResult({ - result: response.result, - documents: input.documents, - options: input.request.content, - read, - }); - return { - ...response, - content: hydration.content, - meta: { - ...response.meta, - hydratedFileCount: hydration.hydratedFileCount, - hydratedBytes: hydration.hydratedBytes, - }, - }; -}; diff --git a/packages/asset-resource/src/indexing.ts b/packages/asset-resource/src/indexing.ts new file mode 100644 index 0000000000..f3a7fc1a6c --- /dev/null +++ b/packages/asset-resource/src/indexing.ts @@ -0,0 +1,5 @@ +export * from "./canonical"; +export * from "./field-catalog"; +export * from "./resource-index"; +export * from "./candidate-selection"; +export * from "./index-storage"; diff --git a/packages/asset-resource/src/published-runtime.test.ts b/packages/asset-resource/src/published-runtime.test.ts index f5684ce5fb..dfdd6e2c04 100644 --- a/packages/asset-resource/src/published-runtime.test.ts +++ b/packages/asset-resource/src/published-runtime.test.ts @@ -132,6 +132,22 @@ describe("published asset resource runtime", () => { ); }); + test("bounds parsed indexes retained by one isolate", async () => { + const { manifest, fetchAsset } = await createRuntime(); + for (let index = 0; index < 5; index += 1) { + const runtimeFetch = createPublishedAssetResourceFetch({ + baseUrl: "https://site.example", + deploymentId: `deployment-${index}`, + manifest, + fetchAsset, + }); + const response = await runtimeFetch(createQueryRequest()); + expect(response?.status).toBe(200); + } + + expect(__testing.getParsedIndexCacheSize()).toBe(4); + }); + test("hydrates exactly the selected complete Markdown file", async () => { const { runtimeFetch, fetchAsset } = await createRuntime(); const response = await runtimeFetch(createQueryRequest({ mode: "full" })); @@ -142,6 +158,7 @@ describe("published asset resource runtime", () => { }); expect(fetchAsset).toHaveBeenCalledWith("/assets/post.md", { headers: expect.any(Headers), + signal: expect.any(AbortSignal), }); }); diff --git a/packages/asset-resource/src/published-runtime.ts b/packages/asset-resource/src/published-runtime.ts index bdd6ca9fc3..8d85bdfd3e 100644 --- a/packages/asset-resource/src/published-runtime.ts +++ b/packages/asset-resource/src/published-runtime.ts @@ -1,4 +1,5 @@ import { + assetResourceLimits, assetResourceQueryFailure, assetResourceQueryRequest, type AssetResourceQueryFailure, @@ -9,9 +10,9 @@ import { } from "@webstudio-is/sdk/runtime"; import { AssetResourceQueryExecutionError, - computeAssetResourceQueryHash, executeAndHydrateAssetResourceQuery, -} from "./index"; +} from "./query-execution"; +import { computeAssetResourceQueryHash } from "./resource-index"; import { AssetResourceHydrationError } from "./hydration"; import { verifyAssetResourceIndex } from "./resource-index"; import { sha256Hex } from "./sha256"; @@ -58,7 +59,11 @@ const loadIndex = ({ }) => { const cacheKey = getParsedIndexCacheKey({ deploymentId, entry }); let pending = parsedIndexCache.get(cacheKey); - if (pending === undefined) { + if (pending !== undefined) { + // Map insertion order doubles as the least-recently-used order. + parsedIndexCache.delete(cacheKey); + parsedIndexCache.set(cacheKey, pending); + } else { pending = (async () => { const response = await fetchAsset(entry.indexPath); if (response.ok === false) { @@ -77,6 +82,13 @@ const loadIndex = ({ })(); parsedIndexCache.set(cacheKey, pending); pending.catch(() => parsedIndexCache.delete(cacheKey)); + while (parsedIndexCache.size > assetResourceLimits.runtimeCachedIndexes) { + const oldestKey = parsedIndexCache.keys().next().value; + if (oldestKey === undefined) { + break; + } + parsedIndexCache.delete(oldestKey); + } } return pending; }; @@ -235,7 +247,21 @@ export const createPublishedAssetResourceFetch = ({ } try { + if (request.signal.aborted) { + return failure({ + code: "REQUEST_CANCELLED", + message: "Published asset resource query was cancelled", + status: 499, + }); + } const index = await loadIndex({ deploymentId, entry, fetchAsset }); + if (request.signal.aborted) { + return failure({ + code: "REQUEST_CANCELLED", + message: "Published asset resource query was cancelled", + status: 499, + }); + } const result = await executeAndHydrateAssetResourceQuery({ request: parsedRequest, documents: index.documents, @@ -252,7 +278,7 @@ export const createPublishedAssetResourceFetch = ({ } const assetResponse = await fetchAsset( `/assets/${encodeURIComponent(contentRef)}`, - { headers } + { headers, signal: request.signal } ); if (assetResponse.ok === false || assetResponse.body === null) { throw new Error("Selected published asset content was not found"); @@ -266,7 +292,7 @@ export const createPublishedAssetResourceFetch = ({ }, }); const resultResponse = jsonResponse(result); - if (cacheKey !== undefined) { + if (cacheKey !== undefined && request.signal.aborted === false) { resultResponse.headers.set( "cache-control", request.headers.get("cache-control") as string @@ -277,6 +303,13 @@ export const createPublishedAssetResourceFetch = ({ } return resultResponse; } catch (error) { + if (request.signal.aborted) { + return failure({ + code: "REQUEST_CANCELLED", + message: "Published asset resource query was cancelled", + status: 499, + }); + } if ( error instanceof AssetResourceQueryExecutionError || error instanceof AssetResourceHydrationError @@ -353,4 +386,5 @@ export const createGeneratedAssetResourceFetch = async ({ export const __testing = { clearParsedIndexCache: () => parsedIndexCache.clear(), + getParsedIndexCacheSize: () => parsedIndexCache.size, }; diff --git a/packages/asset-resource/src/query-execution.ts b/packages/asset-resource/src/query-execution.ts new file mode 100644 index 0000000000..faa2393dce --- /dev/null +++ b/packages/asset-resource/src/query-execution.ts @@ -0,0 +1,163 @@ +import { evaluate, type ExprNode } from "groq-js/1"; +import { + assetResourceLimits, + assetResourceQuerySuccess, + type AssetFileDocument, + type AssetResourceQueryRequest, +} from "@webstudio-is/sdk"; +import { validateAssetResourceQuery } from "./query-validation"; +import { + hydrateAssetResourceResult, + type AssetResourceContentReader, +} from "./hydration"; + +const evaluateAssetResourceQuery = async ({ + tree, + documents, + parameters = {}, +}: { + tree: ExprNode; + documents: readonly AssetFileDocument[]; + parameters?: Record; +}) => { + const value = await evaluate(tree, { + dataset: documents, + params: parameters, + }); + return await value.get(); +}; + +export class AssetResourceQueryExecutionError extends Error { + readonly code: + | "MISSING_PARAMETER" + | "RESULT_LIMIT_EXCEEDED" + | "RESULT_SIZE_EXCEEDED"; + readonly details?: Record; + + constructor({ + code, + message, + details, + }: { + code: AssetResourceQueryExecutionError["code"]; + message: string; + details?: AssetResourceQueryExecutionError["details"]; + }) { + super(message); + this.name = "AssetResourceQueryExecutionError"; + this.code = code; + this.details = details; + } +} + +const getResultCount = (result: unknown) => { + if (Array.isArray(result)) { + return result.length; + } + return result === null || result === undefined ? 0 : 1; +}; + +export const executeAssetResourceQuery = async ({ + request, + documents, + queryHash, + indexRevision, + assetRevision, +}: { + request: AssetResourceQueryRequest; + documents: readonly AssetFileDocument[]; + queryHash: string; + indexRevision: string; + assetRevision: string; +}) => { + const validated = validateAssetResourceQuery(request.query); + for (const parameterName of validated.parameterNames) { + if (Object.hasOwn(request.parameters, parameterName) === false) { + throw new AssetResourceQueryExecutionError({ + code: "MISSING_PARAMETER", + message: `Asset resource parameter $${parameterName} is required`, + details: { parameter: parameterName }, + }); + } + } + if (documents.length > assetResourceLimits.candidateDocuments) { + throw new AssetResourceQueryExecutionError({ + code: "RESULT_LIMIT_EXCEEDED", + message: "Asset resource candidate document limit was exceeded", + details: { + candidateCount: documents.length, + candidateLimit: assetResourceLimits.candidateDocuments, + }, + }); + } + + const rawResult = await evaluateAssetResourceQuery({ + tree: validated.tree, + documents, + parameters: request.parameters, + }); + const result = rawResult ?? null; + const resultCount = getResultCount(result); + if (resultCount > request.resultLimit) { + throw new AssetResourceQueryExecutionError({ + code: "RESULT_LIMIT_EXCEEDED", + message: "Asset resource query returned too many results", + details: { resultCount, resultLimit: request.resultLimit }, + }); + } + const resultBytes = new TextEncoder().encode(JSON.stringify(result)).length; + if (resultBytes > assetResourceLimits.resultBytes) { + throw new AssetResourceQueryExecutionError({ + code: "RESULT_SIZE_EXCEEDED", + message: "Asset resource query result exceeds the byte limit", + details: { + resultBytes, + resultByteLimit: assetResourceLimits.resultBytes, + }, + }); + } + + return assetResourceQuerySuccess.parse({ + ok: true, + result, + content: {}, + meta: { + queryHash, + indexRevision, + assetRevision, + resultCount, + hydratedFileCount: 0, + hydratedBytes: 0, + }, + }); +}; + +export const executeAndHydrateAssetResourceQuery = async ({ + read, + ...input +}: Parameters[0] & { + read?: AssetResourceContentReader; +}) => { + const response = await executeAssetResourceQuery(input); + if (input.request.content.mode === "none") { + return response; + } + if (read === undefined) { + throw new Error("Asset content reader is required for hydration"); + } + const hydration = await hydrateAssetResourceResult({ + result: response.result, + documents: input.documents, + options: input.request.content, + read, + }); + return { + ...response, + content: hydration.content, + meta: { + ...response.meta, + hydratedFileCount: hydration.hydratedFileCount, + hydratedBytes: hydration.hydratedBytes, + }, + }; +}; diff --git a/packages/asset-resource/src/query-validation.test.ts b/packages/asset-resource/src/query-validation.test.ts index 12fdd32272..bfbc81c2d5 100644 --- a/packages/asset-resource/src/query-validation.test.ts +++ b/packages/asset-resource/src/query-validation.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "vitest"; import { assetResourceLimits } from "@webstudio-is/sdk"; import { + getAssetResourceParameterFieldPaths, AssetResourceQueryValidationError, getAssetResourceReferencedFieldPaths, getAssetResourceReferencedFieldPathsFromTree, @@ -21,6 +22,18 @@ const expectValidationCode = ( }; describe("asset resource query validation", () => { + test("finds unambiguous fields compared with route parameters", () => { + expect( + getAssetResourceParameterFieldPaths( + "*[properties.slug == $slug && $locale == properties.locale]" + ) + ).toEqual( + new Map([ + ["locale", ["properties", "locale"]], + ["slug", ["properties", "slug"]], + ]) + ); + }); test("returns one parsed analysis for a valid parameterized query", () => { expect( validateAssetResourceQuery( diff --git a/packages/asset-resource/src/query-validation.ts b/packages/asset-resource/src/query-validation.ts index 5c7bb5b303..bc0bb9c08b 100644 --- a/packages/asset-resource/src/query-validation.ts +++ b/packages/asset-resource/src/query-validation.ts @@ -147,6 +147,71 @@ const getAttributePath = (node: unknown): string | undefined => { : appendAssetFieldPath(basePath, attribute.name); }; +const getAttributePathSegments = (node: unknown): string[] | undefined => { + if ( + typeof node !== "object" || + node === null || + Reflect.get(node, "type") !== "AccessAttribute" || + typeof Reflect.get(node, "name") !== "string" + ) { + return; + } + const base = Reflect.get(node, "base"); + const baseSegments = base === undefined ? [] : getAttributePathSegments(base); + if (baseSegments === undefined) { + return; + } + return [...baseSegments, Reflect.get(node, "name") as string]; +}; + +const getParameterName = (node: unknown) => + typeof node === "object" && + node !== null && + Reflect.get(node, "type") === "Parameter" && + typeof Reflect.get(node, "name") === "string" + ? (Reflect.get(node, "name") as string) + : undefined; + +/** + * Returns the document field directly compared with each GROQ parameter. + * Ambiguous parameters are omitted instead of guessing a prerender identity. + */ +export const getAssetResourceParameterFieldPaths = (query: string) => { + const paths = new Map(); + const add = (parameterName: string, path: string[]) => { + const previous = paths.get(parameterName); + if (previous === undefined && paths.has(parameterName) === false) { + paths.set(parameterName, path); + return; + } + if (JSON.stringify(previous) !== JSON.stringify(path)) { + paths.set(parameterName, undefined); + } + }; + visitGroqAst(validateAssetResourceQuery(query).tree, (node) => { + if (node.type !== "OpCall" || Reflect.get(node, "op") !== "==") { + return; + } + const left = Reflect.get(node, "left"); + const right = Reflect.get(node, "right"); + const leftParameter = getParameterName(left); + const rightParameter = getParameterName(right); + const leftPath = getAttributePathSegments(left); + const rightPath = getAttributePathSegments(right); + if (leftParameter !== undefined && rightPath !== undefined) { + add(leftParameter, rightPath); + } + if (rightParameter !== undefined && leftPath !== undefined) { + add(rightParameter, leftPath); + } + }); + return new Map( + [...paths] + .filter((entry): entry is [string, string[]] => entry[1] !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + ); +}; + export const getAssetResourceReferencedFieldPathsFromTree = ( tree: ExprNode ) => { diff --git a/packages/asset-resource/src/query.ts b/packages/asset-resource/src/query.ts new file mode 100644 index 0000000000..3a3b19c0fc --- /dev/null +++ b/packages/asset-resource/src/query.ts @@ -0,0 +1,2 @@ +export * from "./query-validation"; +export * from "./query-execution"; diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts index a2509438dd..74a9113195 100644 --- a/packages/asset-uploader/src/canonical-metadata-backfill.test.ts +++ b/packages/asset-uploader/src/canonical-metadata-backfill.test.ts @@ -342,7 +342,7 @@ describe("canonical asset metadata synchronization", () => { ).resolves.toEqual({ status: "deleted" }); }); - test("keeps existing canonical metadata when changed Markdown has invalid YAML", async () => { + test("persists standard metadata and a parse error for invalid YAML", async () => { const source = "---\ntitle: [broken\n---\nBody"; const bytes = encoder.encode(source); const readFile = vi.fn(async () => ({ @@ -369,7 +369,16 @@ describe("canonical asset metadata synchronization", () => { }, ]) ), - db.get("AssetFolder", () => json([])) + db.get("AssetFolder", () => json([])), + db.post("rpc/replace_asset_file_metadata", async ({ request }) => { + const input = (await request.json()) as ReplaceMetadataRpcArgs; + expect(input.p_document).toMatchObject({ + _id: "asset-1", + properties: {}, + metadataError: { code: "FRONTMATTER_INVALID" }, + }); + return json(true); + }) ); await expect( @@ -379,7 +388,7 @@ describe("canonical asset metadata synchronization", () => { client: testContext.postgrest.client, assetClient: { uploadFile: vi.fn(), readFile }, }) - ).rejects.toMatchObject({ code: "FRONTMATTER_INVALID" }); + ).resolves.toEqual(expect.anything()); expect(readFile).toHaveBeenCalledOnce(); }); diff --git a/packages/asset-uploader/src/canonical-metadata-backfill.ts b/packages/asset-uploader/src/canonical-metadata-backfill.ts index fc2b92eb8c..4e68bee1e2 100644 --- a/packages/asset-uploader/src/canonical-metadata-backfill.ts +++ b/packages/asset-uploader/src/canonical-metadata-backfill.ts @@ -2,6 +2,7 @@ import { createCanonicalAssetFileEntry, extractMarkdownBodyAndExcerpt, extractMarkdownFrontmatter, + MarkdownMetadataError, normalizeAssetFileDocument, } from "@webstudio-is/asset-resource"; import { @@ -113,12 +114,14 @@ const createCanonicalDocument = ({ revision, properties, excerpt, + metadataError, }: { asset: UploadedAssetRow; hierarchy: AssetFolderHierarchy; revision: string; properties: Record; excerpt?: string; + metadataError?: AssetFileDocument["metadataError"]; }) => { const name = formatAssetName({ name: asset.file.name, @@ -140,6 +143,7 @@ const createCanonicalDocument = ({ }, properties, ...(excerpt === undefined || excerpt.length === 0 ? {} : { excerpt }), + ...(metadataError === undefined ? {} : { metadataError }), }); }; @@ -171,6 +175,7 @@ const indexCanonicalAsset = async ({ }) => { let properties: Record = {}; let excerpt: string | undefined; + let metadataError: AssetFileDocument["metadataError"]; if (markdownExtension.test(asset.file.name)) { const prefixLength = Math.min( asset.file.size, @@ -182,12 +187,22 @@ const indexCanonicalAsset = async ({ : await assetClient .readFile(asset.file.name, { offset: 0, length: prefixLength }) .then((stored) => readPrefix(stored.data, prefixLength)); - const markdown = await Promise.all([ - extractMarkdownFrontmatter(bytes), - extractMarkdownBodyAndExcerpt(bytes), - ]); - properties = markdown[0].properties; - excerpt = markdown[1].excerpt; + try { + properties = (await extractMarkdownFrontmatter(bytes)).properties; + } catch (error) { + if (error instanceof MarkdownMetadataError === false) { + throw error; + } + metadataError = { code: error.code, message: error.message }; + } + try { + excerpt = (await extractMarkdownBodyAndExcerpt(bytes)).excerpt; + } catch (error) { + if (error instanceof MarkdownMetadataError === false) { + throw error; + } + metadataError ??= { code: error.code, message: error.message }; + } } const revision = createAssetContentRevision({ storageName: asset.file.name, @@ -200,6 +215,7 @@ const indexCanonicalAsset = async ({ revision, properties, excerpt, + metadataError, }); await replaceCanonicalAssetFileEntry({ client, @@ -311,6 +327,7 @@ export const synchronizeCanonicalAssets = async ({ revision, properties: current.document.properties, excerpt: current.document.excerpt, + metadataError: current.document.metadataError, }); if ( assetEntries.length === 1 && diff --git a/packages/asset-uploader/src/query-preview.test.ts b/packages/asset-uploader/src/query-preview.test.ts index e6589ba983..d5ffef113f 100644 --- a/packages/asset-uploader/src/query-preview.test.ts +++ b/packages/asset-uploader/src/query-preview.test.ts @@ -209,4 +209,48 @@ describe("previewAssetResourceQuery", () => { length: 8, }); }); + + test("applies static candidate selection before the candidate limit", async () => { + hasProjectPermit.mockResolvedValue(true); + loadCanonicalAssetFileEntries.mockResolvedValue( + Array.from({ length: 1001 }, (_, index) => { + const selected = index === 1000; + const assetId = `asset-${index}`; + return { + projectId, + assetId, + revision, + document: { + _id: assetId, + _type: "asset.file" as const, + name: `${index}.md`, + path: selected ? "blog/selected.md" : `other/${index}.md`, + key: `${index}`, + extension: "md", + mimeType: "text/markdown", + size: 1, + revision, + contentRef: `assets/${assetId}`, + properties: {}, + }, + fieldContributions: [], + }; + }) + ); + + const result = await previewAssetResourceQuery({ + projectId, + request: { + query: '*[path match "blog/**"]{_id}', + parameters: {}, + resultLimit: 1, + content: { mode: "none" }, + }, + context, + assetClient: { readFile: vi.fn() }, + dependencies, + }); + + expect(result.result).toEqual([{ _id: "asset-1000" }]); + }); }); diff --git a/packages/asset-uploader/src/query-preview.ts b/packages/asset-uploader/src/query-preview.ts index 3c2f1bcf68..51b3e91b4f 100644 --- a/packages/asset-uploader/src/query-preview.ts +++ b/packages/asset-uploader/src/query-preview.ts @@ -1,6 +1,6 @@ import { + buildAssetResourceIndex, computeAssetResourceQueryHash, - computeCanonicalAssetRevision, executeAndHydrateAssetResourceQuery, } from "@webstudio-is/asset-resource"; import type { AssetResourceQueryRequest } from "@webstudio-is/sdk"; @@ -32,13 +32,18 @@ export const previewAssetResourceQuery = async ({ "You don't have access to preview this project's asset resources", dependencies, }); - const assetRevision = await computeCanonicalAssetRevision(entries); + const index = await buildAssetResourceIndex({ + projectId, + resourceId: "preview", + query: request.query, + entries, + }); return await executeAndHydrateAssetResourceQuery({ request, - documents: entries.map(({ document }) => document), + documents: index.documents, queryHash: await computeAssetResourceQueryHash(request.query), - indexRevision: `metadata:${assetRevision}`, - assetRevision, + indexRevision: `metadata:${index.integrity.checksum}`, + assetRevision: index.assetRevision, read: assetClient.readFile, }); }; diff --git a/packages/asset-uploader/src/resource-index-garbage-collection.test.ts b/packages/asset-uploader/src/resource-index-garbage-collection.test.ts index deac3df870..9b75adfbc5 100644 --- a/packages/asset-uploader/src/resource-index-garbage-collection.test.ts +++ b/packages/asset-uploader/src/resource-index-garbage-collection.test.ts @@ -105,7 +105,7 @@ describe("resource index garbage collection", () => { ).resolves.toEqual({ claimed: 1, deleted: 0, missing: 1 }); }); - test("drains every available batch", async () => { + test("processes at most one batch", async () => { let claims = 0; server.use( db.post("rpc/claim_asset_resource_index_garbage", () => { @@ -133,7 +133,7 @@ describe("resource index garbage collection", () => { limit: 2, }); - expect(claims).toBe(2); + expect(claims).toBe(1); expect(remove).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/asset-uploader/src/resource-index-garbage-collection.ts b/packages/asset-uploader/src/resource-index-garbage-collection.ts index d7a2872351..0b909329f3 100644 --- a/packages/asset-uploader/src/resource-index-garbage-collection.ts +++ b/packages/asset-uploader/src/resource-index-garbage-collection.ts @@ -109,16 +109,7 @@ export const collectAssetResourceIndexGarbageBestEffort = async ({ limit?: number; }) => { try { - while (true) { - const result = await collectAssetResourceIndexGarbage({ - client, - store, - limit, - }); - if (result.claimed < limit) { - break; - } - } + await collectAssetResourceIndexGarbage({ client, store, limit }); } catch (error) { console.error("Asset resource index cleanup failed", error); } diff --git a/packages/asset-uploader/src/resource-index-lifecycle.test.ts b/packages/asset-uploader/src/resource-index-lifecycle.test.ts index cd739a2127..7b766fce81 100644 --- a/packages/asset-uploader/src/resource-index-lifecycle.test.ts +++ b/packages/asset-uploader/src/resource-index-lifecycle.test.ts @@ -59,7 +59,7 @@ describe("asset resource query lifecycle", () => { ).toBeUndefined(); }); - test("rebuilds all current queries from one canonical load", async () => { + test("rebuilds only created and changed queries from one canonical load", async () => { const previous = [ resource("changed", "*[false]"), resource("unchanged", "*[]"), @@ -83,11 +83,11 @@ describe("asset resource query lifecycle", () => { }) ).resolves.toEqual({ deletedResourceIds: [], - updatedResourceIds: ["changed", "created", "unchanged"], + updatedResourceIds: ["changed", "created"], }); expect(loadCanonicalAssetFileSnapshot).toHaveBeenCalledOnce(); expect(synchronizeCanonicalAssets).toHaveBeenCalledOnce(); - expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(3); + expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(2); expect(collectAssetResourceIndexGarbageBestEffort).toHaveBeenCalledOnce(); }); diff --git a/packages/asset-uploader/src/resource-index-lifecycle.ts b/packages/asset-uploader/src/resource-index-lifecycle.ts index d521af7c8d..6a79d0d123 100644 --- a/packages/asset-uploader/src/resource-index-lifecycle.ts +++ b/packages/asset-uploader/src/resource-index-lifecycle.ts @@ -62,7 +62,10 @@ export const synchronizeAssetResourceIndexQueries = async ({ .map(([resourceId]) => resourceId) .sort(); const changed = [...current] - .filter(([, query]) => query !== undefined) + .filter( + ([resourceId, query]) => + query !== undefined && previous.get(resourceId) !== query + ) .map(([resourceId, query]) => ({ resourceId, query: query as string })) .sort((left, right) => left.resourceId.localeCompare(right.resourceId)); diff --git a/packages/asset-uploader/src/resource-index-maintenance.ts b/packages/asset-uploader/src/resource-index-maintenance.ts index 54a4635fd8..945cc43532 100644 --- a/packages/asset-uploader/src/resource-index-maintenance.ts +++ b/packages/asset-uploader/src/resource-index-maintenance.ts @@ -211,6 +211,7 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({ metadataSnapshot, read, referenceId, + currentResourceIds = [], }: { client: Client; store: ImmutableAssetResourceIndexStore; @@ -225,6 +226,7 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({ metadataSnapshot: CanonicalAssetMetadataSnapshot; read: (name: string) => Promise<{ data: AsyncIterable }>; referenceId: string; + currentResourceIds?: readonly string[]; }): Promise => { if (resources.length === 0) { return []; @@ -241,12 +243,14 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({ const states = new Map( (statesResult.data ?? []).map((state) => [state.resourceId, state]) ); + const explicitlyCurrent = new Set(currentResourceIds); const currentResources = resources.filter((resource) => { const state = states.get(resource.resourceId); return ( - state !== undefined && - state.deletedAt === null && - state.queryHash === resource.queryHash + explicitlyCurrent.has(resource.resourceId) || + (state !== undefined && + state.deletedAt === null && + state.queryHash === resource.queryHash) ); }); await reconcileAssetResourceIndexesForPublication({ diff --git a/packages/asset-uploader/src/upload.ts b/packages/asset-uploader/src/upload.ts index 2f16871015..f627484c3c 100644 --- a/packages/asset-uploader/src/upload.ts +++ b/packages/asset-uploader/src/upload.ts @@ -12,13 +12,14 @@ import { type Asset, } from "@webstudio-is/sdk"; import type { Database } from "@webstudio-is/postgrest/index.server"; -import type { AssetUploadClient } from "./client"; +import type { AssetClient, AssetUploadClient } from "./client"; import type { AssetDataOverride } from "./utils/get-asset-data"; import { createUniqueAssetFilename } from "./utils/get-unique-filename"; import { sanitizeS3Key } from "./utils/sanitize-s3-key"; import { formatAsset } from "./utils/format-asset"; import { assertPostgrestSuccess } from "./patch-utils"; import type { UploadTicket } from "./types"; +import { synchronizeAssetResourceStateAfterAssetChange } from "./resource-index-maintenance"; type UploadData = { projectId: string; @@ -514,12 +515,33 @@ export const uploadFile = async ( if (typeof projectId !== "string") { throw Error("File uploader project is missing"); } - return await getUploadedAsset({ + const asset = await getUploadedAsset({ name, projectId, context, file, }); + const readableClient = client as Partial; + if ( + readableClient.readFile !== undefined && + readableClient.resourceIndexStore !== undefined + ) { + try { + await synchronizeAssetResourceStateAfterAssetChange({ + client: context.postgrest.client, + assetClient: readableClient as AssetClient & { + resourceIndexStore: NonNullable; + }, + projectId, + assetId: asset.id, + }); + } catch (error) { + // The upload has committed. Derived metadata and indexes are repaired + // by the next preview or publication if this best-effort update fails. + console.error("Asset upload resource synchronization failed", error); + } + } + return asset; } catch (error) { await cleanupUploadError(name, context, onUploadError); throw error; diff --git a/packages/cli/src/commands/api-command-metadata.ts b/packages/cli/src/commands/api-command-metadata.ts index 55f7fbc3e5..87214096ed 100644 --- a/packages/cli/src/commands/api-command-metadata.ts +++ b/packages/cli/src/commands/api-command-metadata.ts @@ -247,6 +247,17 @@ export const apiCommandMetadata = publicApiOperations.map((operation) => ({ export const highLevelCliCommands = [ { command: "audit", operation: "audit" }, { command: "permissions", operation: "permissions" }, + { command: "get-assets-resource", operation: "get-assets-resource" }, + { command: "create-assets-resource", operation: "create-assets-resource" }, + { command: "update-assets-resource", operation: "update-assets-resource" }, + { + command: "get-asset-resource-index-status", + operation: "get-asset-resource-index-status", + }, + { + command: "rebuild-asset-resource-index", + operation: "rebuild-asset-resource-index", + }, { command: "get-marketplace-product", operation: "get-marketplace-product", diff --git a/packages/cli/src/commands/api-command.ts b/packages/cli/src/commands/api-command.ts index 53b3bf0cd2..5a8973d8d6 100644 --- a/packages/cli/src/commands/api-command.ts +++ b/packages/cli/src/commands/api-command.ts @@ -2877,6 +2877,45 @@ const apiCommandHandlers: Partial> = { dependencies ); }, + "get-assets-resource": async (options, connection, dependencies) => + runProjectSessionCommand( + "get-assets-resource", + { resourceId: requireOption(options.resource, "--resource") }, + connection, + dependencies + ), + "create-assets-resource": async (options, connection, dependencies) => + runProjectSessionCommand( + "create-assets-resource", + await readInputObject(dependencies, options), + connection, + dependencies + ), + "update-assets-resource": async (options, connection, dependencies) => + runProjectSessionCommand( + "update-assets-resource", + await readInputObject(dependencies, options), + connection, + dependencies + ), + "get-asset-resource-index-status": async ( + options, + connection, + dependencies + ) => + runProjectSessionCommand( + "get-asset-resource-index-status", + { resourceId: requireOption(options.resource, "--resource") }, + connection, + dependencies + ), + "rebuild-asset-resource-index": async (options, connection, dependencies) => + runProjectSessionCommand( + "rebuild-asset-resource-index", + { resourceId: requireOption(options.resource, "--resource") }, + connection, + dependencies + ), "create-resource": async (options, connection, dependencies) => { const input = { resource: (await getResourceFields( diff --git a/packages/cli/src/commands/man.ts b/packages/cli/src/commands/man.ts index 6bd92e56de..b7ccc6276a 100644 --- a/packages/cli/src/commands/man.ts +++ b/packages/cli/src/commands/man.ts @@ -472,6 +472,9 @@ const apiCommandsByArea = { projectQuality: cliCommandMetadata .filter(({ cliCommand }) => cliCommand === "audit") .map(({ cliCommand }) => cliCommand), + assetsAndIndexes: cliCommandMetadata + .filter(({ cliCommand }) => cliCommand.includes("asset")) + .map(({ cliCommand }) => cliCommand), publishAndDomains: cliCommandMetadata .filter( ({ cliCommand }) => diff --git a/packages/cli/src/framework-vike-ssg.ts b/packages/cli/src/framework-vike-ssg.ts index 33dcc55c56..610d29c957 100644 --- a/packages/cli/src/framework-vike-ssg.ts +++ b/packages/cli/src/framework-vike-ssg.ts @@ -1,6 +1,6 @@ import { join } from "node:path"; import { readFile } from "node:fs/promises"; -import { isPathnamePattern } from "@webstudio-is/sdk"; +import { isPathnamePattern, matchPathnameParams } from "@webstudio-is/sdk"; import { baseComponentImportSource, createFrameworkComponentRegistry, @@ -16,7 +16,16 @@ const generateVikeRoute = (pagePath: string) => { if (pagePath === "/") { return "index"; } - return pagePath; + let route = pagePath; + const matches = [...matchPathnameParams(pagePath)].reverse(); + for (const match of matches) { + const name = match.groups?.name; + if (name === undefined || match.index === undefined) { + continue; + } + route = `${route.slice(0, match.index)}@${name}${route.slice(match.index + match[0].length)}`; + } + return route; }; export const createFramework = async ( @@ -49,25 +58,37 @@ export const createFramework = async ( select: `${baseComponentImportSource}:Select`, a: `${baseComponentImportSource}:Link`, }, - html: ({ pagePath }: { pagePath: string }) => { - // ignore dynamic pages in static export - if (isPathnamePattern(pagePath)) { - return []; + html: ({ pagePath, prerenderPaths = [] }) => { + const dynamic = isPathnamePattern(pagePath); + if (dynamic && prerenderPaths.length === 0) { + throw new Error( + `Dynamic SSG page ${JSON.stringify(pagePath)} has no enumerable Assets query paths` + ); } - return [ + const route = generateVikeRoute(pagePath); + const entries = [ { - file: join("pages", generateVikeRoute(pagePath), "+Page.tsx"), + file: join("pages", route, "+Page.tsx"), template: htmlPageTemplate, }, { - file: join("pages", generateVikeRoute(pagePath), "+Head.tsx"), + file: join("pages", route, "+Head.tsx"), template: htmlHeadTemplate, }, { - file: join("pages", generateVikeRoute(pagePath), "+data.ts"), + file: join("pages", route, "+data.ts"), template: htmlDataTemplate, }, ]; + if (dynamic) { + entries.push({ + file: join("pages", route, "+onBeforePrerenderStart.ts"), + template: `export const onBeforePrerenderStart = () => ${JSON.stringify( + prerenderPaths + )};\n`, + }); + } + return entries; }, xml: () => [], text: () => [], diff --git a/packages/cli/src/framework.ts b/packages/cli/src/framework.ts index e23f96ff9e..281439dabb 100644 --- a/packages/cli/src/framework.ts +++ b/packages/cli/src/framework.ts @@ -25,7 +25,10 @@ export type Framework = { components: Record; // instance.tag: "importSource:importSpecifier" tags: Record; - html: (params: { pagePath: string }) => FrameworkTemplateEntry[]; + html: (params: { + pagePath: string; + prerenderPaths?: readonly string[]; + }) => FrameworkTemplateEntry[]; xml: (params: { pagePath: string }) => FrameworkTemplateEntry[]; text: (params: { pagePath: string }) => FrameworkTemplateEntry[]; defaultSitemap: () => FrameworkTemplateEntry[]; diff --git a/packages/cli/src/prebuild.test.ts b/packages/cli/src/prebuild.test.ts index 96dd4bf94a..552fd145cd 100644 --- a/packages/cli/src/prebuild.test.ts +++ b/packages/cli/src/prebuild.test.ts @@ -430,7 +430,9 @@ test("materializes immutable resource indexes as public JSON with a reference-on "app/__generated__/$resources.asset-query-runtime.ts", "utf8" ); - expect(runtimeModule).toContain('from "@webstudio-is/asset-resource"'); + expect(runtimeModule).toContain( + 'from "@webstudio-is/asset-resource/runtime"' + ); expect(runtimeModule).not.toContain('"documents"'); expect(runtimeModule).not.toContain('"properties"'); }); @@ -1016,7 +1018,21 @@ describe("prebuild", () => { path.includes("$slug") ) ).toHaveLength(1); - }); + await symlink(join(originalCwd, "node_modules"), "node_modules", "dir"); + await runGeneratedCommand("react-router", ["build"]); + const serverBundle = ( + await Promise.all( + ( + await getFilePaths("build/server") + ) + .filter((path) => path.endsWith(".js")) + .map((path) => readFile(path, "utf8")) + ) + ).join("\n"); + expect(serverBundle).not.toContain("Published post"); + expect(serverBundle).not.toContain("draft secret"); + expect(serverBundle).not.toContain("post-revision"); + }, 30_000); test("uses pass-through images in the base react-router template", async () => { await prebuild({ assets: false, template: ["react-router"] }); @@ -1087,7 +1103,7 @@ describe("prebuild", () => { ); }); - test("selects ssg templates and skips dynamic routes", async () => { + test("selects ssg templates", async () => { await writeSiteData( createSiteData({ pages: [ @@ -1099,14 +1115,6 @@ describe("prebuild", () => { rootInstanceId: "root", meta: {}, }, - { - id: "post", - name: "Post", - title: "Post", - path: "/blog/:slug", - rootInstanceId: "root", - meta: {}, - }, ], }) ); @@ -1122,12 +1130,6 @@ describe("prebuild", () => { await expect(readFile("pages/index/+data.ts", "utf8")).resolves.toContain( "../app/__generated__/_index.server" ); - await expect( - readFile("pages/blog/:slug/+Page.tsx", "utf8") - ).rejects.toThrow("ENOENT"); - await expect( - readFile("app/__generated__/[blog].$slug._index.tsx", "utf8") - ).resolves.toContain("export { Page }"); await expect(readFile("pages/index/+data.ts", "utf8")).resolves.toContain( "createSsgAssetResourceFetch" ); @@ -1137,6 +1139,110 @@ describe("prebuild", () => { ); }); + test("prerenders dynamic SSG paths from parameterized Assets resources", async () => { + const query = + '*[properties.slug == $slug][0]{_id, "title": properties.title}'; + const index = await createAssetResourceIndex({ + format: "webstudio-resource-index", + version: 1, + resourceId: "post", + query, + assetRevision: `sha256:${"a".repeat(64)}`, + queryMode: "parameterized", + parameterNames: ["slug"], + documents: [ + { + _id: "post-1", + _type: "asset.file", + name: "post.md", + path: "blog/post.md", + key: "post", + extension: "md", + mimeType: "text/markdown", + size: 1, + revision: "post-revision", + contentRef: "post.md", + properties: { slug: "hello-world", title: "Hello" }, + }, + ], + }); + const siteData = { + ...createSiteData({ + pages: [ + { + id: "home", + name: "Home", + title: "Home", + path: "", + rootInstanceId: "home-root", + meta: {}, + }, + { + id: "post", + name: "Post", + title: "Post", + path: "/blog/:slug", + rootInstanceId: "post-root", + meta: {}, + }, + ], + instances: [ + ["home-root", { id: "home-root", component: "Box", children: [] }], + ["post-root", { id: "post-root", component: "Box", children: [] }], + ], + }), + assetResourceIndexes: [ + { + resourceId: index.resourceId, + revision: index.integrity.checksum, + index, + }, + ], + }; + siteData.build.resources = [ + [ + "post", + { + id: "post", + name: "Post", + control: "system", + method: "post", + url: '"/$resources/assets/query"', + headers: [], + body: createAssetQueryResourceBody({ + query, + parameters: [{ name: "slug", value: "system.params.slug" }], + resultLimit: 1, + }), + }, + ], + ] as never; + siteData.build.dataSources = [ + [ + "post-data", + { + id: "post-data", + type: "resource", + name: "post", + resourceId: "post", + scopeInstanceId: "post-root", + }, + ], + ] as never; + await writeSiteData(siteData); + + await prebuild({ assets: false, template: ["ssg"] }); + await expect( + readFile("pages/blog/@slug/+onBeforePrerenderStart.ts", "utf8") + ).resolves.toContain("/blog/hello-world"); + await symlink(join(originalCwd, "node_modules"), "node_modules", "dir"); + await runGeneratedCommand("vite", ["build"]); + await runGeneratedCommand("vike", ["prerender"]); + await expect( + readFile("dist/client/blog/hello-world/index.html", "utf8") + ).resolves.toContain(""); + }, 30_000); + test("prerenders SSG pages with asset query data", async () => { const query = '*[]{"title": properties.title}'; const index = await createAssetResourceIndex({ diff --git a/packages/cli/src/prebuild.ts b/packages/cli/src/prebuild.ts index 78d57de6ba..2c3eae5e8e 100644 --- a/packages/cli/src/prebuild.ts +++ b/packages/cli/src/prebuild.ts @@ -36,6 +36,7 @@ import { createScope, findTreeInstanceIds, getAllPages, + getAssetResourceQuery, getPagePath, getPublishablePages, generateResources, @@ -50,8 +51,11 @@ import { elementComponent, toRuntimeAsset, assetResourceContentOptions, + assetResourceLimits, + matchPathnameParams, parseAssetQueryResourceBody, } from "@webstudio-is/sdk"; +import { getAssetResourceParameterFieldPaths } from "@webstudio-is/asset-resource"; import { migratePages } from "@webstudio-is/project-migrations/pages"; import { collectFontFamiliesFromStyleDecls } from "@webstudio-is/project-build/runtime"; import { @@ -99,6 +103,141 @@ type SiteDataByPage = { }; }; +const getExpressionMemberPath = (node: unknown): string[] | undefined => { + if (typeof node !== "object" || node === null) { + return; + } + if ( + Reflect.get(node, "type") === "Identifier" && + typeof Reflect.get(node, "name") === "string" + ) { + return [Reflect.get(node, "name") as string]; + } + if (Reflect.get(node, "type") !== "MemberExpression") { + return; + } + const base = getExpressionMemberPath(Reflect.get(node, "object")); + if (base === undefined) { + return; + } + const computed = Reflect.get(node, "computed") === true; + const property = Reflect.get(node, "property"); + const name = computed + ? Reflect.get(property, "value") + : Reflect.get(property, "name"); + return typeof name === "string" ? [...base, name] : undefined; +}; + +const getBoundSystemRouteParameter = (expression: string) => { + try { + const program = parse(`(${expression})`, { ecmaVersion: "latest" }); + const statement = program.body[0]; + const node = + statement?.type === "ExpressionStatement" + ? statement.expression + : undefined; + const path = getExpressionMemberPath(node); + return path?.length === 3 && path[0] === "system" && path[1] === "params" + ? path[2] + : undefined; + } catch { + return; + } +}; + +const getDocumentPathValue = (document: unknown, path: readonly string[]) => { + let value = document; + for (const segment of path) { + if (typeof value !== "object" || value === null) { + return; + } + value = Reflect.get(value, segment); + } + return value; +}; + +export const getAssetResourcePrerenderPaths = ({ + pagePath, + resources, + indexes, +}: { + pagePath: string; + resources: readonly [string, Resource][]; + indexes: PublishedProjectBundle["assetResourceIndexes"]; +}) => { + const pathParameters = [...matchPathnameParams(pagePath)]; + if ( + pathParameters.length === 0 || + pathParameters.some( + (match) => + match.groups?.name === undefined || (match.groups.modifier ?? "") !== "" + ) + ) { + return []; + } + const routeParameterNames = new Set( + pathParameters.map((match) => match.groups?.name as string) + ); + const indexByResourceId = new Map( + (indexes ?? []).map((snapshot) => [snapshot.resourceId, snapshot.index]) + ); + const paths = new Set(); + for (const [, resource] of resources) { + const index = indexByResourceId.get(resource.id); + if (index === undefined) { + continue; + } + const query = getAssetResourceQuery(resource); + if (query === undefined) { + continue; + } + const parameterFields = getAssetResourceParameterFieldPaths(query); + const routeFields = new Map(); + for (const binding of parseAssetQueryResourceBody(resource.body) + .parameters) { + const routeParameter = getBoundSystemRouteParameter(binding.value); + const fieldPath = parameterFields.get(binding.name); + if ( + routeParameter !== undefined && + routeParameterNames.has(routeParameter) && + fieldPath !== undefined + ) { + routeFields.set(routeParameter, fieldPath); + } + } + if (routeFields.size !== routeParameterNames.size) { + continue; + } + for (const document of index.documents) { + const values = new Map(); + for (const [name, fieldPath] of routeFields) { + const value = getDocumentPathValue(document, fieldPath); + if ( + (typeof value === "string" && value.length > 0) || + (typeof value === "number" && Number.isFinite(value)) || + typeof value === "boolean" + ) { + values.set(name, String(value)); + } + } + if (values.size !== routeParameterNames.size) { + continue; + } + let path = pagePath; + for (const match of [...pathParameters].reverse()) { + const name = match.groups?.name as string; + const value = values.get(name) as string; + path = `${path.slice(0, match.index)}${encodeURIComponent(value)}${path.slice((match.index ?? 0) + match[0].length)}`; + } + paths.add(path); + if (paths.size > assetResourceLimits.candidateDocuments) { + throw new Error("Dynamic SSG path count exceeds the Assets limit"); + } + } + } + return [...paths].sort(); +}; + const mergeJsonInto = async (sourcePath: string, destinationPath: string) => { const sourceJson = await readFile(sourcePath, "utf8"); const destinationJson = await readFile(destinationPath, "utf8").catch( @@ -215,7 +354,7 @@ export const generateAssetQueryRuntimeModule = ({ if (manifest.length === 0) { return `export const createGeneratedAssetResourceFetch = async ({ fallback }: ${inputType}): Promise => fallback;\n`; } - return `import { createGeneratedAssetResourceFetch as createRuntimeFetch } from "@webstudio-is/asset-resource"; + return `import { createGeneratedAssetResourceFetch as createRuntimeFetch } from "@webstudio-is/asset-resource/runtime"; const deploymentId = ${JSON.stringify(deploymentId)}; const manifest = ${JSON.stringify(manifest, null, 2)}; @@ -970,7 +1109,15 @@ export const prebuild = async (options: { await writeGeneratedFile(serverFile, serverExports); const getTemplates = framework[documentType]; - for (const { file, template } of getTemplates({ pagePath })) { + const prerenderPaths = getAssetResourcePrerenderPaths({ + pagePath, + resources: pageData.build.resources, + indexes: siteData.assetResourceIndexes, + }); + for (const { file, template } of getTemplates({ + pagePath, + prerenderPaths, + })) { const content = template .replaceAll("__CONSTANTS__", importFrom("./app/constants.mjs", file)) .replaceAll( diff --git a/packages/cli/src/template-placeholders.d.ts b/packages/cli/src/template-placeholders.d.ts index 34678a644b..d5ec53db77 100644 --- a/packages/cli/src/template-placeholders.d.ts +++ b/packages/cli/src/template-placeholders.d.ts @@ -1,5 +1,5 @@ declare module "__ASSET_QUERY_MANIFEST__" { - import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource"; + import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource/runtime"; export const assetQueryDeploymentId: string; export const assetQueryManifest: PublishedAssetResourceManifestEntry[]; @@ -14,7 +14,7 @@ declare module "__ASSET_QUERY_RUNTIME__" { } declare module "__ASSET_RESOURCE_FETCH__" { - import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource"; + import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource/runtime"; export const createSsgAssetResourceFetch: (options: { deploymentId: string; diff --git a/packages/cli/templates/defaults/app/route-templates/html.tsx b/packages/cli/templates/defaults/app/route-templates/html.tsx index 3a88a70808..5d0246cdaf 100644 --- a/packages/cli/templates/defaults/app/route-templates/html.tsx +++ b/packages/cli/templates/defaults/app/route-templates/html.tsx @@ -132,7 +132,8 @@ export const loader = async (arg: LoaderFunctionArgs) => { const resources = await loadResources( generatedFetch, getResources({ system }).data, - url + url, + { signal: arg.request.signal } ); const pageMeta = getPageMeta({ system, resources }); diff --git a/packages/cli/templates/defaults/app/route-templates/text.tsx b/packages/cli/templates/defaults/app/route-templates/text.tsx index d8cec689df..05813ac665 100644 --- a/packages/cli/templates/defaults/app/route-templates/text.tsx +++ b/packages/cli/templates/defaults/app/route-templates/text.tsx @@ -91,7 +91,8 @@ export const loader = async (arg: LoaderFunctionArgs) => { const resources = await loadResources( generatedFetch, getResources({ system }).data, - url + url, + { signal: arg.request.signal } ); const pageMeta = getPageMeta({ system, resources }); diff --git a/packages/cli/templates/defaults/app/route-templates/xml.tsx b/packages/cli/templates/defaults/app/route-templates/xml.tsx index 2f91078a73..dc9005ac8d 100644 --- a/packages/cli/templates/defaults/app/route-templates/xml.tsx +++ b/packages/cli/templates/defaults/app/route-templates/xml.tsx @@ -99,7 +99,8 @@ export const loader = async (arg: LoaderFunctionArgs) => { const resources = await loadResources( generatedFetch, getResources({ system }).data, - url + url, + { signal: arg.request.signal } ); const pageMeta = getPageMeta({ system, resources }); diff --git a/packages/cli/templates/react-router/app/route-templates/html.tsx b/packages/cli/templates/react-router/app/route-templates/html.tsx index ea30c7265f..36b0228e7e 100644 --- a/packages/cli/templates/react-router/app/route-templates/html.tsx +++ b/packages/cli/templates/react-router/app/route-templates/html.tsx @@ -131,7 +131,8 @@ export const loader = async (arg: LoaderFunctionArgs) => { const resources = await loadResources( generatedFetch, getResources({ system }).data, - url + url, + { signal: arg.request.signal } ); const pageMeta = getPageMeta({ system, resources }); diff --git a/packages/cli/templates/react-router/app/route-templates/text.tsx b/packages/cli/templates/react-router/app/route-templates/text.tsx index 88e3391e08..092f9b894b 100644 --- a/packages/cli/templates/react-router/app/route-templates/text.tsx +++ b/packages/cli/templates/react-router/app/route-templates/text.tsx @@ -91,7 +91,8 @@ export const loader = async (arg: LoaderFunctionArgs) => { const resources = await loadResources( generatedFetch, getResources({ system }).data, - url + url, + { signal: arg.request.signal } ); const pageMeta = getPageMeta({ system, resources }); diff --git a/packages/cli/templates/react-router/app/route-templates/xml.tsx b/packages/cli/templates/react-router/app/route-templates/xml.tsx index 5cecf6a16f..9d72682c4a 100644 --- a/packages/cli/templates/react-router/app/route-templates/xml.tsx +++ b/packages/cli/templates/react-router/app/route-templates/xml.tsx @@ -99,7 +99,8 @@ export const loader = async (arg: LoaderFunctionArgs) => { const resources = await loadResources( generatedFetch, getResources({ system }).data, - url + url, + { signal: arg.request.signal } ); const pageMeta = getPageMeta({ system, resources }); diff --git a/packages/cli/templates/ssg/app/asset-resource-fetch.ts b/packages/cli/templates/ssg/app/asset-resource-fetch.ts index a4f36f2295..2b6b7ed7ea 100644 --- a/packages/cli/templates/ssg/app/asset-resource-fetch.ts +++ b/packages/cli/templates/ssg/app/asset-resource-fetch.ts @@ -1,6 +1,6 @@ import { open } from "node:fs/promises"; import { resolve, sep } from "node:path"; -import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource"; +import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource/runtime"; const resolvePublicAsset = (path: string) => { const publicDirectory = resolve("public"); diff --git a/packages/http-client/src/index.test.ts b/packages/http-client/src/index.test.ts index 0e6ef3edfa..f67d8881ae 100644 --- a/packages/http-client/src/index.test.ts +++ b/packages/http-client/src/index.test.ts @@ -1647,6 +1647,7 @@ test("imports project bundle through staged upload", async () => { projectId: "project-id", data: { largeContent: "x".repeat(3 * 1024 * 1024 + 1), + assetResourceIndexes: [{ marker: "derived-index-marker" }], } as unknown as PublishedProjectBundle, }) ).resolves.toEqual({ version: 2 }); @@ -1657,6 +1658,9 @@ test("imports project bundle through staged upload", async () => { } expect(uploadChunks).toHaveLength(2); + expect(Buffer.concat(uploadChunks).toString("utf8")).not.toContain( + "derived-index-marker" + ); expect(JSON.stringify(trpcBody)).toContain('"uploadId":"upload-id"'); expect(JSON.stringify(trpcBody)).not.toContain("largeContent"); }); diff --git a/packages/http-client/src/index.ts b/packages/http-client/src/index.ts index 2076ebe85f..828acbd8d9 100644 --- a/packages/http-client/src/index.ts +++ b/packages/http-client/src/index.ts @@ -1811,7 +1811,11 @@ const uploadProjectBundleData = async ( ) => { const { sourceOrigin } = parseBuilderUrl(params.origin); const endpoint = new URL(stagedUploadPath, sourceOrigin); - const data = JSON.stringify(params.data); + // Resource indexes are derived deployment artifacts. Import rebuilds them + // from the destination project's assets and must not transport stale blobs. + const { assetResourceIndexes: _assetResourceIndexes, ...portableData } = + params.data; + const data = JSON.stringify(portableData); if (new TextEncoder().encode(data).byteLength > maxProjectBundleSize) { throw new Error( `Project bundle is too large to import. Maximum size is ${formatMebibytes(maxProjectBundleSize)}.` diff --git a/packages/postgrest/supabase/tests/asset-resource-indexes.sql b/packages/postgrest/supabase/tests/asset-resource-indexes.sql index dfca118cc7..4ade08e7d5 100644 --- a/packages/postgrest/supabase/tests/asset-resource-indexes.sql +++ b/packages/postgrest/supabase/tests/asset-resource-indexes.sql @@ -160,13 +160,30 @@ VALUES ('resource-index-test.md', 'file', 10, 'UPLOADED'); INSERT INTO "Asset" (id, "projectId", name) VALUES ('resource-index-test-asset', 'resource-index-test-project', 'resource-index-test.md'); +SELECT throws_ok( + $$ + INSERT INTO "AssetFileMetadata" ( + "projectId", "assetId", revision, document, "fieldContributions" + ) VALUES ( + 'resource-index-test-project', + 'resource-index-test-asset', + 'file:resource-index-test.md:invalid', + '{}'::JSONB, + '[]'::JSONB + ) + $$, + '23514', + NULL, + 'Canonical metadata requires explicit matching identity fields' +); + INSERT INTO "AssetFileMetadata" ( "projectId", "assetId", revision, document, "fieldContributions" ) VALUES ( 'resource-index-test-project', 'resource-index-test-asset', 'file:resource-index-test.md:1', - '{}'::JSONB, + '{"_id":"resource-index-test-asset","revision":"file:resource-index-test.md:1"}'::JSONB, '[]'::JSONB ); @@ -545,10 +562,89 @@ SELECT is( 'Repeated query deletion is idempotent' ); +INSERT INTO "AssetResourceIndexState" ( + "projectId", "resourceId", query, "queryHash", "assetRevision", + "buildAttemptId", "buildStatus" +) VALUES ( + 'resource-index-test-project', + 'project-delete-test', + '*[]', + 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'project-delete-attempt', + 'BUILDING' +); + +INSERT INTO "AssetResourceIndexRevision" ( + "projectId", "resourceId", revision, "queryHash", "assetRevision", + checksum, "objectKey" +) VALUES ( + 'resource-index-test-project', + 'project-delete-test', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'projects/test/resources/project-delete-test.json' +); + +UPDATE "AssetResourceIndexState" +SET "activeRevision" = + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + "buildStatus" = 'ACTIVE' +WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'project-delete-test'; + +UPDATE "Project" +SET "isDeleted" = TRUE +WHERE id = 'resource-index-test-project'; + +SELECT ok( + ( + SELECT "activeRevision" IS NULL AND "deletedAt" IS NOT NULL + FROM "AssetResourceIndexState" + WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'project-delete-test' + ), + 'Soft project deletion retires active resource indexes' +); + +SELECT ok( + ( + SELECT "unreferencedAt" IS NOT NULL + FROM "AssetResourceIndexRevision" + WHERE "projectId" = 'resource-index-test-project' + AND "resourceId" = 'project-delete-test' + ), + 'Soft project deletion makes immutable indexes collectible' +); + DELETE FROM "Build" WHERE "id" = 'resource-index-test-build' AND "projectId" = 'resource-index-test-project'; +SELECT throws_ok( + $$ DELETE FROM "Project" WHERE id = 'resource-index-test-project' $$, + '23503', + NULL, + 'Hard project deletion cannot orphan immutable resource indexes' +); + +CREATE TEMP TABLE project_delete_garbage AS +SELECT * FROM claim_asset_resource_index_garbage('2100-01-01', 10) +WHERE "resourceId" = 'project-delete-test'; + +SELECT is( + finalize_asset_resource_index_garbage( + (SELECT "projectId" FROM project_delete_garbage), + (SELECT "resourceId" FROM project_delete_garbage), + (SELECT revision FROM project_delete_garbage), + (SELECT "gcClaimId" FROM project_delete_garbage) + ), + TRUE, + 'Project deletion cleanup finalizes its immutable resource index' +); + DELETE FROM "Project" WHERE "id" = 'resource-index-test-project'; SELECT is( diff --git a/packages/prisma-client/prisma/migrations/20260718120000_asset_resources/migration.sql b/packages/prisma-client/prisma/migrations/20260718120000_asset_resources/migration.sql index 135d77584a..2f37ddaef0 100644 --- a/packages/prisma-client/prisma/migrations/20260718120000_asset_resources/migration.sql +++ b/packages/prisma-client/prisma/migrations/20260718120000_asset_resources/migration.sql @@ -15,11 +15,13 @@ CREATE TABLE "AssetFileMetadata" ( CONSTRAINT "AssetFileMetadata_pkey" PRIMARY KEY ("projectId", "assetId", "revision"), CONSTRAINT "AssetFileMetadata_document_identity_check" - CHECK ( + CHECK (( jsonb_typeof("document") = 'object' + AND jsonb_typeof("document"->'_id') = 'string' AND "document"->>'_id' = "assetId" + AND jsonb_typeof("document"->'revision') = 'string' AND "document"->>'revision' = "revision" - ), + ) IS TRUE), CONSTRAINT "AssetFileMetadata_field_contributions_check" CHECK (jsonb_typeof("fieldContributions") = 'array'), CONSTRAINT "AssetFileMetadata_assetId_projectId_fkey" @@ -27,10 +29,6 @@ CREATE TABLE "AssetFileMetadata" ( ON DELETE CASCADE ON UPDATE CASCADE ); -CREATE INDEX "AssetFileMetadata_projectId_assetId_idx" - ON "AssetFileMetadata"("projectId", "assetId"); - - -- Canonical metadata lifecycle CREATE FUNCTION replace_asset_file_metadata( @@ -244,6 +242,15 @@ CREATE INDEX "AssetResourceIndexRevision_lookup_idx" "assetRevision" ); +CREATE INDEX "AssetResourceIndexRevision_garbage_idx" + ON "AssetResourceIndexRevision"( + "unreferencedAt", + "projectId", + "resourceId", + "revision" + ) + WHERE "unreferencedAt" IS NOT NULL; + ALTER TABLE "AssetResourceIndexState" ADD CONSTRAINT "AssetResourceIndexState_activeRevision_fkey" FOREIGN KEY ("projectId", "resourceId", "activeRevision") @@ -254,6 +261,58 @@ ALTER TABLE "AssetResourceIndexState" ) ON DELETE NO ACTION ON UPDATE CASCADE; +-- Derived index objects must enter garbage collection before project rows can +-- disappear. Normal project deletion is soft; the guard prevents a later hard +-- delete from losing the only durable object keys. +CREATE FUNCTION retire_asset_resource_indexes_on_project_delete() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF OLD."isDeleted" IS DISTINCT FROM TRUE AND NEW."isDeleted" = TRUE THEN + UPDATE public."AssetResourceIndexState" + SET "activeRevision" = NULL, + "buildStatus" = 'STALE', + "buildError" = NULL, + "deletedAt" = COALESCE("deletedAt", CURRENT_TIMESTAMP), + "updatedAt" = CURRENT_TIMESTAMP + WHERE "projectId" = NEW.id; + + UPDATE public."AssetResourceIndexRevision" + SET "unreferencedAt" = COALESCE("unreferencedAt", CURRENT_TIMESTAMP) + WHERE "projectId" = NEW.id; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER "Project_retire_asset_resource_indexes" +AFTER UPDATE OF "isDeleted" ON "Project" +FOR EACH ROW +EXECUTE FUNCTION retire_asset_resource_indexes_on_project_delete(); + +CREATE FUNCTION protect_asset_resource_objects_before_project_delete() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM public."AssetResourceIndexRevision" + WHERE "projectId" = OLD.id + ) THEN + RAISE EXCEPTION + 'Project asset resource indexes must be collected before hard deletion' + USING ERRCODE = '23503'; + END IF; + RETURN OLD; +END; +$$; + +CREATE TRIGGER "Project_protect_asset_resource_objects" +BEFORE DELETE ON "Project" +FOR EACH ROW +EXECUTE FUNCTION protect_asset_resource_objects_before_project_delete(); + -- Resource index build and activation lifecycle @@ -975,7 +1034,16 @@ CREATE FUNCTION finalize_asset_resource_index_garbage( RETURNS BOOLEAN LANGUAGE plpgsql AS $$ +DECLARE + removed BOOLEAN; BEGIN + PERFORM pg_catalog.pg_advisory_xact_lock( + pg_catalog.hashtextextended( + pg_catalog.jsonb_build_array(p_project_id, p_resource_id)::TEXT, + 0 + ) + ); + DELETE FROM public."AssetResourceIndexRevision" AS candidate WHERE candidate."projectId" = p_project_id AND candidate."resourceId" = p_resource_id @@ -997,7 +1065,22 @@ BEGIN AND state."resourceId" = candidate."resourceId" AND state."activeRevision" = candidate.revision ); - RETURN FOUND; + removed := FOUND; + + IF removed THEN + DELETE FROM public."AssetResourceIndexState" AS state + WHERE state."projectId" = p_project_id + AND state."resourceId" = p_resource_id + AND state."deletedAt" IS NOT NULL + AND state."activeRevision" IS NULL + AND NOT EXISTS ( + SELECT 1 FROM public."AssetResourceIndexRevision" AS revision + WHERE revision."projectId" = state."projectId" + AND revision."resourceId" = state."resourceId" + ); + END IF; + + RETURN removed; END; $$; diff --git a/packages/project-build/package.json b/packages/project-build/package.json index 8120f32061..bbace1269d 100644 --- a/packages/project-build/package.json +++ b/packages/project-build/package.json @@ -67,6 +67,7 @@ "dependencies": { "@emotion/hash": "^0.9.2", "@modelcontextprotocol/sdk": "^1.10.0", + "@webstudio-is/asset-resource": "workspace:*", "@webstudio-is/authorization-token": "workspace:*", "@webstudio-is/css-data": "workspace:*", "@webstudio-is/css-engine": "workspace:*", diff --git a/packages/project-build/src/runtime/asset-resources.test.ts b/packages/project-build/src/runtime/asset-resources.test.ts new file mode 100644 index 0000000000..45d158fc77 --- /dev/null +++ b/packages/project-build/src/runtime/asset-resources.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, test } from "vitest"; +import { + assetsResourceCreateInput, + assetsResourceUpdateInput, +} from "./asset-resources"; + +const baseQuery = { + resultLimit: 20, + content: { mode: "none" as const }, + parameters: [], +}; + +describe("Assets resource mutation input", () => { + test("rejects invalid GROQ on create", () => { + const result = assetsResourceCreateInput.safeParse({ + name: "Posts", + scopeInstanceId: "root", + query: { ...baseQuery, groq: "*[" }, + }); + + expect(result.success).toBe(false); + }); + + test("rejects missing parameter bindings on create and update", () => { + const query = { + ...baseQuery, + groq: "*[properties.slug == $slug]", + }; + + expect( + assetsResourceCreateInput.safeParse({ + name: "Posts", + scopeInstanceId: "root", + query, + }).success + ).toBe(false); + expect( + assetsResourceUpdateInput.safeParse({ + resourceId: "posts", + values: { query }, + }).success + ).toBe(false); + }); + + test("accepts a binding for every GROQ parameter", () => { + expect( + assetsResourceCreateInput.safeParse({ + name: "Post", + scopeInstanceId: "root", + query: { + ...baseQuery, + groq: "*[properties.slug == $slug]", + parameters: [{ name: "slug", value: "system.params.slug" }], + }, + }).success + ).toBe(true); + }); +}); diff --git a/packages/project-build/src/runtime/asset-resources.ts b/packages/project-build/src/runtime/asset-resources.ts index 35c8e4e6f3..aacfd53020 100644 --- a/packages/project-build/src/runtime/asset-resources.ts +++ b/packages/project-build/src/runtime/asset-resources.ts @@ -16,6 +16,7 @@ import { assetsQueryResourceUrl, assetsResourceUrl, } from "@webstudio-is/sdk/runtime"; +import { validateAssetResourceQuery } from "@webstudio-is/asset-resource"; import { z } from "zod"; import type { BuilderState } from "../state/builder-state"; import type { BuilderRuntimeContext } from "./context"; @@ -67,16 +68,58 @@ const assetResourceConfigurationFields = { content: assetResourceContentOptions.default({ mode: "none" }), }; -const storedAssetQueryConfigurationInput = z.object( - assetResourceConfigurationFields -); +const refineAssetQueryConfiguration = ( + configuration: { + query: string; + parameters: readonly { name: string }[]; + }, + context: z.RefinementCtx +) => { + let parameterNames: readonly string[]; + try { + parameterNames = validateAssetResourceQuery( + configuration.query + ).parameterNames; + } catch (error) { + context.addIssue({ + code: "custom", + path: ["query"], + message: + error instanceof Error ? error.message : "Asset query is invalid.", + }); + return; + } + const bindings = new Set( + configuration.parameters.map((parameter) => parameter.name) + ); + for (const parameterName of parameterNames) { + if (bindings.has(parameterName) === false) { + context.addIssue({ + code: "custom", + path: ["parameters"], + message: `Asset query parameter $${parameterName} requires a binding.`, + }); + } + } +}; -export const assetsQueryConfigurationInput = z.object({ - groq: assetResourceConfigurationFields.query, - parameters: assetResourceConfigurationFields.parameters, - resultLimit: assetResourceConfigurationFields.resultLimit, - content: assetResourceConfigurationFields.content, -}); +const storedAssetQueryConfigurationInput = z + .object(assetResourceConfigurationFields) + .superRefine(refineAssetQueryConfiguration); + +export const assetsQueryConfigurationInput = z + .object({ + groq: assetResourceConfigurationFields.query, + parameters: assetResourceConfigurationFields.parameters, + resultLimit: assetResourceConfigurationFields.resultLimit, + content: assetResourceConfigurationFields.content, + }) + .superRefine((configuration, context) => + refineAssetQueryConfiguration( + { query: configuration.groq, parameters: configuration.parameters }, + context + ) + ); export const assetsResourceListInput = z.object({ scopeInstanceId: z.string().optional(), diff --git a/packages/sdk/src/asset-resource-config.ts b/packages/sdk/src/asset-resource-config.ts index b116e16e08..3e12fbb7d8 100644 --- a/packages/sdk/src/asset-resource-config.ts +++ b/packages/sdk/src/asset-resource-config.ts @@ -1,5 +1,6 @@ import { generateObjectExpression, parseObjectExpression } from "./expression"; import { assetsQueryResourceUrl, assetsResourceUrl } from "./resource-loader"; +import { assetResourceLimits } from "./schema/asset-resource"; import type { Resource } from "./schema/resources"; export type AssetQueryParameterBinding = { @@ -34,7 +35,7 @@ export const parseAssetQueryResourceBody = (body: string | undefined) => { export const createAssetQueryResourceBody = ({ query, parameters, - resultLimit = 100, + resultLimit = assetResourceLimits.defaultResultCount, contentExpression = '{ "mode": "none" }', }: { query: string; diff --git a/packages/sdk/src/schema/asset-resource.ts b/packages/sdk/src/schema/asset-resource.ts index 87c9f1c86a..a38a420639 100644 --- a/packages/sdk/src/schema/asset-resource.ts +++ b/packages/sdk/src/schema/asset-resource.ts @@ -37,6 +37,12 @@ export const assetFileDocument = z.strictObject({ contentRef: z.string().min(1), properties: z.record(z.string(), z.json()), excerpt: z.string().optional(), + metadataError: z + .strictObject({ + code: z.string().min(1), + message: z.string().min(1), + }) + .optional(), }); export type AssetFileDocument = z.infer; @@ -138,6 +144,9 @@ export const assetResourceLimits = { resultBytes: 1024 * 1024, candidateDocuments: 1000, indexBytes: 16 * 1024 * 1024, + publishedResourceCount: 32, + publishedIndexBytes: 32 * 1024 * 1024, + runtimeCachedIndexes: 4, frontmatterBytes: 64 * 1024, frontmatterDepth: 8, frontmatterFields: 256, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4af81c051a..7c5b875bb4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2047,6 +2047,9 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.10.0 version: 1.10.0 + '@webstudio-is/asset-resource': + specifier: workspace:* + version: link:../asset-resource '@webstudio-is/authorization-token': specifier: workspace:* version: link:../authorization-token