fix: harden asset resource publication

This commit is contained in:
Oleg Isonen committed 2026-07-30 10:35:51 +01:00
1 parent eaad08bb13
commit cc10a4e893
52 files changed
+1367 -313

No files matched your search

@@ -146,5 +146,13 @@ describe("asset query resource body", () => {
content: { mode: "none" },
})
).toContain("$slug");
expect(
getAssetQueryConfigurationError({
query: "*[properties.slug == $slug]",
parameters: [{ name: "slug", value: "system.params." }],
resultLimit: 1,
content: { mode: "none" },
})
).toContain("Webstudio expression");
});
});
@@ -7,6 +7,7 @@ import {
type AssetResourceIndexStatus,
} from "@webstudio-is/sdk";
import { validateAssetResourceQuery } from "@webstudio-is/asset-resource";
import { getExpressionErrorMessages } from "@webstudio-is/project-build/runtime";
export {
createAssetQueryResourceBody,
parseAssetQueryResourceBody,
@@ -58,7 +59,7 @@ export const getAssetQueryConfigurationError = ({
content,
}: {
query: string;
parameters: readonly { name: string }[];
parameters: readonly { name: string; value?: string }[];
resultLimit: number;
content: AssetResourceContentOptions;
}) => {
@@ -84,6 +85,15 @@ export const getAssetQueryConfigurationError = ({
if (new Set(names).size !== names.length) {
return "Runtime parameter names must be unique.";
}
if (
parameters.some(
({ value }) =>
value !== undefined &&
getExpressionErrorMessages({ expression: value }).length > 0
)
) {
return "Enter a valid Webstudio expression for every runtime parameter.";
}
const configuredNames = new Set(names);
const missingName = referencedParameterNames.find(
(name) => configuredNames.has(name) === false
@@ -306,6 +306,13 @@ const AssetQueryPreview = ({
}
let ignore = false;
let pollTimeout: ReturnType<typeof setTimeout> | undefined;
let pollAttempts = 0;
const schedulePoll = () => {
pollAttempts += 1;
if (pollAttempts < 10) {
pollTimeout = setTimeout(load, 1000);
}
};
const load = async () => {
try {
const response = await loadBuilderAssetIndexStatus(resourceId);
@@ -316,12 +323,13 @@ const AssetQueryPreview = ({
const parsed = assetResourceIndexStatus.safeParse(data.status);
const status = parsed.success ? parsed.data : undefined;
onIndexStatusChange(status);
if (status?.state === "indexing") {
pollTimeout = setTimeout(load, 1000);
if (status?.state === "indexing" || status === undefined) {
schedulePoll();
}
} catch {
if (ignore === false) {
onIndexStatusChange(undefined);
schedulePoll();
}
}
};
@@ -448,7 +456,9 @@ export const AssetQueryForm = ({
{}
);
const [resultLimit, setResultLimit] = useState(
typeof parsedResultLimit === "number" ? parsedResultLimit : 100
typeof parsedResultLimit === "number"
? parsedResultLimit
: assetResourceLimits.defaultResultCount
);
const parsedContent = assetResourceContentOptions.safeParse(
evaluateExpressionWithinScope(parsedBody.contentExpression ?? "", {})
+115 -22
View File
@@ -20,12 +20,15 @@ import {
findPageByIdOrPath,
getAllPages,
getStyleDeclKey,
assetResourceLimits,
isStoredAssetQueryResource,
type Asset,
type AssetFolder,
} from "@webstudio-is/sdk";
import {
computeAssetResourceQueryHash,
computeCanonicalAssetRevision,
serializeAssetResourceIndex,
} from "@webstudio-is/asset-resource";
import { serializePages } from "@webstudio-is/project-migrations/pages";
import { loadById } from "@webstudio-is/project/index.server";
@@ -164,25 +167,42 @@ const addProjectMetadata = async (
? undefined
: await getUserById(context, project.userId);
const reachableResourceIds = new Set<string>();
for (const [, prop] of data.build.props) {
if (prop.type === "resource") {
reachableResourceIds.add(prop.value);
}
}
for (const [, dataSource] of data.build.dataSources) {
if (dataSource.type === "resource") {
reachableResourceIds.add(dataSource.resourceId);
}
}
const assetQueryResources = data.build.resources
.map(([, resource]) => resource)
.flatMap((resource) => {
if (reachableResourceIds.has(resource.id) === false) {
return [];
}
const query = getAssetResourceQuery(resource);
return query === undefined ? [] : [{ resourceId: resource.id, query }];
if (query !== undefined) {
return [{ resourceId: resource.id, query }];
}
if (isStoredAssetQueryResource(resource)) {
throw new Error(
`Assets resource ${JSON.stringify(resource.id)} has an invalid query configuration`
);
}
return [];
});
if (assetQueryResources.length > assetResourceLimits.publishedResourceCount) {
throw new Error("Published Assets resource count exceeds the limit");
}
let assetResourceIndexes: PublishedProjectBundle["assetResourceIndexes"];
let publishedAssets = data.assets;
let publishedAssetFolders = data.assetFolders;
if (assetQueryResources.length > 0) {
const assetClient = createAssetClientWithResourceIndexStore();
await synchronizeCanonicalAssets({
client: context.postgrest.client,
projectId: project.id,
assetClient,
});
const { entries: canonicalEntries, metadataSnapshot } =
await loadCanonicalAssetFileSnapshot({
client: context.postgrest.client,
projectId: project.id,
});
const indexedResources = await Promise.all(
assetQueryResources.map(async ({ resourceId, query }) => ({
resourceId,
@@ -190,24 +210,97 @@ const addProjectMetadata = async (
queryHash: await computeAssetResourceQueryHash(query),
}))
);
const expectedAssetRevision =
await computeCanonicalAssetRevision(canonicalEntries);
assetResourceIndexes =
await prepareAssetResourceIndexSnapshotsForPublication({
const currentBuild = await loadDevBuildByProjectId(context, project.id);
const currentQueries = new Map(
(currentBuild?.resources ?? []).map((resource) => [
resource.id,
getAssetResourceQuery(resource),
])
);
const currentResourceIds = indexedResources
.filter(
({ resourceId, query }) => currentQueries.get(resourceId) === query
)
.map(({ resourceId }) => resourceId);
const retainedFontIds = new Set(
data.assets
.filter((asset) => asset.type === "font")
.map((asset) => asset.id)
);
for (let attempt = 0; attempt < 2; attempt += 1) {
const assetDataBefore = await loadAssetDataByProject(project.id, context);
await synchronizeCanonicalAssets({
client: context.postgrest.client,
store: assetClient.resourceIndexStore,
projectId: project.id,
resources: indexedResources,
entries: canonicalEntries,
read: assetClient.resourceIndexStore.read,
referenceId: data.build.id,
assetRevision: expectedAssetRevision,
metadataSnapshot,
assetClient,
});
const { entries: canonicalEntries, metadataSnapshot } =
await loadCanonicalAssetFileSnapshot({
client: context.postgrest.client,
projectId: project.id,
});
const expectedAssetRevision =
await computeCanonicalAssetRevision(canonicalEntries);
const preparedIndexes =
await prepareAssetResourceIndexSnapshotsForPublication({
client: context.postgrest.client,
store: assetClient.resourceIndexStore,
projectId: project.id,
resources: indexedResources,
entries: canonicalEntries,
read: assetClient.resourceIndexStore.read,
referenceId: data.build.id,
assetRevision: expectedAssetRevision,
metadataSnapshot,
currentResourceIds,
});
const [assetDataAfter, latestCanonical] = await Promise.all([
loadAssetDataByProject(project.id, context),
loadCanonicalAssetFileSnapshot({
client: context.postgrest.client,
projectId: project.id,
}),
]);
const latestAssetRevision = await computeCanonicalAssetRevision(
latestCanonical.entries
);
const assetsStayedStable =
JSON.stringify(assetDataBefore) === JSON.stringify(assetDataAfter) &&
expectedAssetRevision === latestAssetRevision;
if (assetsStayedStable === false) {
if (attempt === 0) {
continue;
}
throw new Error("Assets changed while preparing publication; retry");
}
assetResourceIndexes = preparedIndexes;
publishedAssets = assetDataAfter.assets.filter(
(asset) => asset.type !== "font" || retainedFontIds.has(asset.id)
);
publishedAssetFolders = assetDataAfter.assetFolders;
break;
}
if (assetResourceIndexes === undefined) {
throw new Error("Asset resource indexes were not prepared");
}
const totalIndexBytes = assetResourceIndexes.reduce(
(total, snapshot) =>
total +
new TextEncoder().encode(serializeAssetResourceIndex(snapshot.index))
.byteLength,
0
);
if (totalIndexBytes > assetResourceLimits.publishedIndexBytes) {
throw new Error(
"Published Assets resource indexes exceed the byte limit"
);
}
}
return {
...data,
assets: publishedAssets,
assetFolders: publishedAssetFolders,
bundleVersion,
user: user ? { email: user.email } : undefined,
projectDomain: project.domain,
@@ -286,6 +286,10 @@ claims are 15-minute leases. A later
cleanup worker rotates and resumes an expired claim, so a crash before or after
object deletion cannot permanently strand the revision row.
Each inline cleanup invocation processes at most one bounded batch. Production
must also invoke the same collector from scheduled background work so cleanup
does not depend on later Builder or publication traffic.
## Schemaless frontmatter and publication
Frontmatter fields have no built-in publication meaning. Names such as `draft`,
@@ -313,6 +317,9 @@ single source for Builder, indexer, generated runtime, and test limits. V1 uses:
| Serialized result | 1 MiB |
| Candidate documents | 1,000 |
| Serialized index | 16 MiB |
| Published query resources | 32 |
| Published indexes combined | 32 MiB |
| Parsed indexes per isolate | 4, least-recently used |
| Frontmatter per file | 64 KiB |
| Frontmatter structure | Depth 8, 256 fields |
| One frontmatter string | 16 KiB UTF-8 |
@@ -648,6 +655,16 @@ server-only adapter reads immutable indexes and ranged Markdown bytes from the
generated `public` directory, so prerendering requires neither a live HTTP
endpoint nor bundling content into JavaScript. Range hydration uses positioned
file reads rather than loading the complete source asset before slicing it.
Dynamic SSG pages are enumerated when every required `:parameter` is bound
directly from `system.params.<name>` and its GROQ parameter is compared directly
with one canonical document field, for example
`properties.slug == $slug`. Prebuild fails explicitly when a dynamic route has
no safely enumerable Assets resource instead of silently omitting its pages.
Redeploying a historical build preserves that build's stored query and
application structure but evaluates it against the project's latest assets.
This matches the existing non-versioned asset contract; it is not an exact
historical content rollback.
Public index construction applies only the configured GROQ candidate selection.
Prebuild materializes Markdown through the same static asset pipeline as other
+26 -2
View File
@@ -11,7 +11,7 @@
"benchmark:groq": "node scripts/benchmark-groq.mjs",
"benchmark:index": "NODE_OPTIONS=--conditions=webstudio pnpm tsx scripts/benchmark-index.mjs",
"benchmark:scale": "NODE_OPTIONS=--conditions=webstudio pnpm tsx scripts/benchmark-scale.mjs",
"build": "rm -rf lib && esbuild src/index.ts --outdir=lib --bundle --format=esm --packages=external",
"build": "rm -rf lib && esbuild src/index.ts src/published-runtime.ts src/query.ts src/markdown.ts src/indexing.ts --outdir=lib --bundle --format=esm --packages=external",
"dts": "tsc --project tsconfig.dts.json",
"test": "vitest run",
"typecheck": "tsc --noEmit"
@@ -35,10 +35,34 @@
"types": "./lib/types/index.d.ts",
"import": "./lib/index.js",
"default": "./src/index.ts"
},
"./runtime": {
"webstudio": "./src/published-runtime.ts",
"types": "./lib/types/published-runtime.d.ts",
"import": "./lib/published-runtime.js",
"default": "./src/published-runtime.ts"
},
"./query": {
"webstudio": "./src/query.ts",
"types": "./lib/types/query.d.ts",
"import": "./lib/query.js",
"default": "./src/query.ts"
},
"./markdown": {
"webstudio": "./src/markdown.ts",
"types": "./lib/types/markdown.d.ts",
"import": "./lib/markdown.js",
"default": "./src/markdown.ts"
},
"./indexing": {
"webstudio": "./src/indexing.ts",
"types": "./lib/types/indexing.d.ts",
"import": "./lib/indexing.js",
"default": "./src/indexing.ts"
}
},
"files": [
"lib/*",
"lib/**/*",
"!*.test.*"
]
}
+22 -11
View File
@@ -63,17 +63,28 @@ describe("normalizeAssetFileDocument", () => {
});
});
test.each(["", ".", "..", "nested/name", "nested\\name"])(
"rejects invalid path segment %j",
(name) => {
expect(() =>
normalizeAssetFileDocument({
asset: { ...asset, name },
properties: {},
})
).toThrow("one normalized path segment");
}
);
test.each([
[".", "%2E"],
["..", "%2E%2E"],
["nested/name", "nested%2Fname"],
["nested\\name", "nested%5Cname"],
])("encodes user-defined path segment %j", (name, path) => {
expect(
normalizeAssetFileDocument({
asset: { ...asset, name, folderId: undefined, folderNames: undefined },
properties: {},
}).path
).toBe(path);
});
test("rejects an empty asset name", () => {
expect(() =>
normalizeAssetFileDocument({
asset: { ...asset, name: "" },
properties: {},
})
).toThrow();
});
test("rejects JSON-incompatible property values", () => {
expect(() =>
+17 -15
View File
@@ -107,16 +107,18 @@ export const createCanonicalAssetFileEntry = ({
};
};
const assertPathSegment = (segment: string, label: string) => {
if (
segment.length === 0 ||
segment === "." ||
segment === ".." ||
segment.includes("/") ||
segment.includes("\\")
) {
throw new Error(`${label} must be one normalized path segment`);
// Asset and folder names are user data, not filesystem paths. Encode each name
// independently so slashes and dot segments cannot change path structure while
// preserving the original name in the document for display and exact matching.
const encodeAssetPathSegment = (segment: string) => {
const encoded = encodeURIComponent(segment);
if (encoded === ".") {
return "%2E";
}
if (encoded === "..") {
return "%2E%2E";
}
return encoded;
};
/**
@@ -127,22 +129,19 @@ export const normalizeAssetFileDocument = ({
asset,
properties,
excerpt,
metadataError,
}: {
asset: AssetFileMetadataInput;
properties: Record<string, unknown>;
excerpt?: string;
metadataError?: AssetFileDocument["metadataError"];
}): AssetFileDocument => {
assertPathSegment(asset.name, "Asset name");
const folderNames = asset.folderNames ?? [];
if ((asset.folderId === undefined) !== (folderNames.length === 0)) {
throw new Error(
"Asset folder ID and complete folder-name path must be provided together"
);
}
for (const folderName of folderNames) {
assertPathSegment(folderName, "Asset folder name");
}
const extensionSeparator = asset.name.lastIndexOf(".");
const extension =
asset.extension ??
@@ -153,7 +152,9 @@ export const normalizeAssetFileDocument = ({
extensionSeparator <= 0
? asset.name
: asset.name.slice(0, extensionSeparator);
const path = [...folderNames, asset.name].join("/");
const path = [...folderNames, asset.name]
.map(encodeAssetPathSegment)
.join("/");
return assetFileDocument.parse({
_id: asset.id,
@@ -169,5 +170,6 @@ export const normalizeAssetFileDocument = ({
contentRef: asset.contentRef,
properties,
...(excerpt === undefined ? {} : { excerpt }),
...(metadataError === undefined ? {} : { metadataError }),
});
};
@@ -79,6 +79,13 @@ const getStandardFieldContributions = (
...(document.excerpt === undefined
? []
: [{ path: "excerpt", type: "string" } as const]),
...(document.metadataError === undefined
? []
: [
{ path: "metadataError", type: "object" } as const,
{ path: "metadataError.code", type: "string" } as const,
{ path: "metadataError.message", type: "string" } as const,
]),
];
type MutableField = {
+1 -161
View File
@@ -1,19 +1,9 @@
import { evaluate, type ExprNode } from "groq-js/1";
import {
assetResourceLimits,
assetResourceQueryRequest,
assetResourceQuerySuccess,
type AssetFileDocument,
type AssetResourceQueryInput,
type AssetResourceQueryRequest,
type ResourceRequest,
} from "@webstudio-is/sdk";
import { assetsQueryResourceUrl } from "@webstudio-is/sdk/runtime";
import { validateAssetResourceQuery } from "./query-validation";
import {
hydrateAssetResourceResult,
type AssetResourceContentReader,
} from "./hydration";
export * from "./markdown";
export * from "./canonical";
@@ -23,6 +13,7 @@ export * from "./candidate-selection";
export * from "./index-storage";
export * from "./query-validation";
export * from "./hydration";
export * from "./query-execution";
export * from "./published-runtime";
export const createAssetResourceRequest = (
@@ -36,154 +27,3 @@ export const createAssetResourceRequest = (
headers: [{ name: "content-type", value: "application/json" }],
body: assetResourceQueryRequest.parse(request),
});
const evaluateAssetResourceQuery = async ({
tree,
documents,
parameters = {},
}: {
tree: ExprNode;
documents: readonly AssetFileDocument[];
parameters?: Record<string, unknown>;
}) => {
const value = await evaluate(tree, {
dataset: documents,
params: parameters,
});
return await value.get();
};
export class AssetResourceQueryExecutionError extends Error {
readonly code:
| "MISSING_PARAMETER"
| "RESULT_LIMIT_EXCEEDED"
| "RESULT_SIZE_EXCEEDED";
readonly details?: Record<string, number | string>;
constructor({
code,
message,
details,
}: {
code: AssetResourceQueryExecutionError["code"];
message: string;
details?: AssetResourceQueryExecutionError["details"];
}) {
super(message);
this.name = "AssetResourceQueryExecutionError";
this.code = code;
this.details = details;
}
}
const getResultCount = (result: unknown) => {
if (Array.isArray(result)) {
return result.length;
}
return result === null || result === undefined ? 0 : 1;
};
export const executeAssetResourceQuery = async ({
request,
documents,
queryHash,
indexRevision,
assetRevision,
}: {
request: AssetResourceQueryRequest;
documents: readonly AssetFileDocument[];
queryHash: string;
indexRevision: string;
assetRevision: string;
}) => {
const validated = validateAssetResourceQuery(request.query);
for (const parameterName of validated.parameterNames) {
if (Object.hasOwn(request.parameters, parameterName) === false) {
throw new AssetResourceQueryExecutionError({
code: "MISSING_PARAMETER",
message: `Asset resource parameter $${parameterName} is required`,
details: { parameter: parameterName },
});
}
}
if (documents.length > assetResourceLimits.candidateDocuments) {
throw new AssetResourceQueryExecutionError({
code: "RESULT_LIMIT_EXCEEDED",
message: "Asset resource candidate document limit was exceeded",
details: {
candidateCount: documents.length,
candidateLimit: assetResourceLimits.candidateDocuments,
},
});
}
const rawResult = await evaluateAssetResourceQuery({
tree: validated.tree,
documents,
parameters: request.parameters,
});
const result = rawResult ?? null;
const resultCount = getResultCount(result);
if (resultCount > request.resultLimit) {
throw new AssetResourceQueryExecutionError({
code: "RESULT_LIMIT_EXCEEDED",
message: "Asset resource query returned too many results",
details: { resultCount, resultLimit: request.resultLimit },
});
}
const resultBytes = new TextEncoder().encode(JSON.stringify(result)).length;
if (resultBytes > assetResourceLimits.resultBytes) {
throw new AssetResourceQueryExecutionError({
code: "RESULT_SIZE_EXCEEDED",
message: "Asset resource query result exceeds the byte limit",
details: {
resultBytes,
resultByteLimit: assetResourceLimits.resultBytes,
},
});
}
return assetResourceQuerySuccess.parse({
ok: true,
result,
content: {},
meta: {
queryHash,
indexRevision,
assetRevision,
resultCount,
hydratedFileCount: 0,
hydratedBytes: 0,
},
});
};
export const executeAndHydrateAssetResourceQuery = async ({
read,
...input
}: Parameters<typeof executeAssetResourceQuery>[0] & {
read?: AssetResourceContentReader;
}) => {
const response = await executeAssetResourceQuery(input);
if (input.request.content.mode === "none") {
return response;
}
if (read === undefined) {
throw new Error("Asset content reader is required for hydration");
}
const hydration = await hydrateAssetResourceResult({
result: response.result,
documents: input.documents,
options: input.request.content,
read,
});
return {
...response,
content: hydration.content,
meta: {
...response.meta,
hydratedFileCount: hydration.hydratedFileCount,
hydratedBytes: hydration.hydratedBytes,
},
};
};
+5
View File
@@ -0,0 +1,5 @@
export * from "./canonical";
export * from "./field-catalog";
export * from "./resource-index";
export * from "./candidate-selection";
export * from "./index-storage";
@@ -132,6 +132,22 @@ describe("published asset resource runtime", () => {
);
});
test("bounds parsed indexes retained by one isolate", async () => {
const { manifest, fetchAsset } = await createRuntime();
for (let index = 0; index < 5; index += 1) {
const runtimeFetch = createPublishedAssetResourceFetch({
baseUrl: "https://site.example",
deploymentId: `deployment-${index}`,
manifest,
fetchAsset,
});
const response = await runtimeFetch(createQueryRequest());
expect(response?.status).toBe(200);
}
expect(__testing.getParsedIndexCacheSize()).toBe(4);
});
test("hydrates exactly the selected complete Markdown file", async () => {
const { runtimeFetch, fetchAsset } = await createRuntime();
const response = await runtimeFetch(createQueryRequest({ mode: "full" }));
@@ -142,6 +158,7 @@ describe("published asset resource runtime", () => {
});
expect(fetchAsset).toHaveBeenCalledWith("/assets/post.md", {
headers: expect.any(Headers),
signal: expect.any(AbortSignal),
});
});
@@ -1,4 +1,5 @@
import {
assetResourceLimits,
assetResourceQueryFailure,
assetResourceQueryRequest,
type AssetResourceQueryFailure,
@@ -9,9 +10,9 @@ import {
} from "@webstudio-is/sdk/runtime";
import {
AssetResourceQueryExecutionError,
computeAssetResourceQueryHash,
executeAndHydrateAssetResourceQuery,
} from "./index";
} from "./query-execution";
import { computeAssetResourceQueryHash } from "./resource-index";
import { AssetResourceHydrationError } from "./hydration";
import { verifyAssetResourceIndex } from "./resource-index";
import { sha256Hex } from "./sha256";
@@ -58,7 +59,11 @@ const loadIndex = ({
}) => {
const cacheKey = getParsedIndexCacheKey({ deploymentId, entry });
let pending = parsedIndexCache.get(cacheKey);
if (pending === undefined) {
if (pending !== undefined) {
// Map insertion order doubles as the least-recently-used order.
parsedIndexCache.delete(cacheKey);
parsedIndexCache.set(cacheKey, pending);
} else {
pending = (async () => {
const response = await fetchAsset(entry.indexPath);
if (response.ok === false) {
@@ -77,6 +82,13 @@ const loadIndex = ({
})();
parsedIndexCache.set(cacheKey, pending);
pending.catch(() => parsedIndexCache.delete(cacheKey));
while (parsedIndexCache.size > assetResourceLimits.runtimeCachedIndexes) {
const oldestKey = parsedIndexCache.keys().next().value;
if (oldestKey === undefined) {
break;
}
parsedIndexCache.delete(oldestKey);
}
}
return pending;
};
@@ -235,7 +247,21 @@ export const createPublishedAssetResourceFetch = ({
}
try {
if (request.signal.aborted) {
return failure({
code: "REQUEST_CANCELLED",
message: "Published asset resource query was cancelled",
status: 499,
});
}
const index = await loadIndex({ deploymentId, entry, fetchAsset });
if (request.signal.aborted) {
return failure({
code: "REQUEST_CANCELLED",
message: "Published asset resource query was cancelled",
status: 499,
});
}
const result = await executeAndHydrateAssetResourceQuery({
request: parsedRequest,
documents: index.documents,
@@ -252,7 +278,7 @@ export const createPublishedAssetResourceFetch = ({
}
const assetResponse = await fetchAsset(
`/assets/${encodeURIComponent(contentRef)}`,
{ headers }
{ headers, signal: request.signal }
);
if (assetResponse.ok === false || assetResponse.body === null) {
throw new Error("Selected published asset content was not found");
@@ -266,7 +292,7 @@ export const createPublishedAssetResourceFetch = ({
},
});
const resultResponse = jsonResponse(result);
if (cacheKey !== undefined) {
if (cacheKey !== undefined && request.signal.aborted === false) {
resultResponse.headers.set(
"cache-control",
request.headers.get("cache-control") as string
@@ -277,6 +303,13 @@ export const createPublishedAssetResourceFetch = ({
}
return resultResponse;
} catch (error) {
if (request.signal.aborted) {
return failure({
code: "REQUEST_CANCELLED",
message: "Published asset resource query was cancelled",
status: 499,
});
}
if (
error instanceof AssetResourceQueryExecutionError ||
error instanceof AssetResourceHydrationError
@@ -353,4 +386,5 @@ export const createGeneratedAssetResourceFetch = async ({
export const __testing = {
clearParsedIndexCache: () => parsedIndexCache.clear(),
getParsedIndexCacheSize: () => parsedIndexCache.size,
};
@@ -0,0 +1,163 @@
import { evaluate, type ExprNode } from "groq-js/1";
import {
assetResourceLimits,
assetResourceQuerySuccess,
type AssetFileDocument,
type AssetResourceQueryRequest,
} from "@webstudio-is/sdk";
import { validateAssetResourceQuery } from "./query-validation";
import {
hydrateAssetResourceResult,
type AssetResourceContentReader,
} from "./hydration";
const evaluateAssetResourceQuery = async ({
tree,
documents,
parameters = {},
}: {
tree: ExprNode;
documents: readonly AssetFileDocument[];
parameters?: Record<string, unknown>;
}) => {
const value = await evaluate(tree, {
dataset: documents,
params: parameters,
});
return await value.get();
};
export class AssetResourceQueryExecutionError extends Error {
readonly code:
| "MISSING_PARAMETER"
| "RESULT_LIMIT_EXCEEDED"
| "RESULT_SIZE_EXCEEDED";
readonly details?: Record<string, number | string>;
constructor({
code,
message,
details,
}: {
code: AssetResourceQueryExecutionError["code"];
message: string;
details?: AssetResourceQueryExecutionError["details"];
}) {
super(message);
this.name = "AssetResourceQueryExecutionError";
this.code = code;
this.details = details;
}
}
const getResultCount = (result: unknown) => {
if (Array.isArray(result)) {
return result.length;
}
return result === null || result === undefined ? 0 : 1;
};
export const executeAssetResourceQuery = async ({
request,
documents,
queryHash,
indexRevision,
assetRevision,
}: {
request: AssetResourceQueryRequest;
documents: readonly AssetFileDocument[];
queryHash: string;
indexRevision: string;
assetRevision: string;
}) => {
const validated = validateAssetResourceQuery(request.query);
for (const parameterName of validated.parameterNames) {
if (Object.hasOwn(request.parameters, parameterName) === false) {
throw new AssetResourceQueryExecutionError({
code: "MISSING_PARAMETER",
message: `Asset resource parameter $${parameterName} is required`,
details: { parameter: parameterName },
});
}
}
if (documents.length > assetResourceLimits.candidateDocuments) {
throw new AssetResourceQueryExecutionError({
code: "RESULT_LIMIT_EXCEEDED",
message: "Asset resource candidate document limit was exceeded",
details: {
candidateCount: documents.length,
candidateLimit: assetResourceLimits.candidateDocuments,
},
});
}
const rawResult = await evaluateAssetResourceQuery({
tree: validated.tree,
documents,
parameters: request.parameters,
});
const result = rawResult ?? null;
const resultCount = getResultCount(result);
if (resultCount > request.resultLimit) {
throw new AssetResourceQueryExecutionError({
code: "RESULT_LIMIT_EXCEEDED",
message: "Asset resource query returned too many results",
details: { resultCount, resultLimit: request.resultLimit },
});
}
const resultBytes = new TextEncoder().encode(JSON.stringify(result)).length;
if (resultBytes > assetResourceLimits.resultBytes) {
throw new AssetResourceQueryExecutionError({
code: "RESULT_SIZE_EXCEEDED",
message: "Asset resource query result exceeds the byte limit",
details: {
resultBytes,
resultByteLimit: assetResourceLimits.resultBytes,
},
});
}
return assetResourceQuerySuccess.parse({
ok: true,
result,
content: {},
meta: {
queryHash,
indexRevision,
assetRevision,
resultCount,
hydratedFileCount: 0,
hydratedBytes: 0,
},
});
};
export const executeAndHydrateAssetResourceQuery = async ({
read,
...input
}: Parameters<typeof executeAssetResourceQuery>[0] & {
read?: AssetResourceContentReader;
}) => {
const response = await executeAssetResourceQuery(input);
if (input.request.content.mode === "none") {
return response;
}
if (read === undefined) {
throw new Error("Asset content reader is required for hydration");
}
const hydration = await hydrateAssetResourceResult({
result: response.result,
documents: input.documents,
options: input.request.content,
read,
});
return {
...response,
content: hydration.content,
meta: {
...response.meta,
hydratedFileCount: hydration.hydratedFileCount,
hydratedBytes: hydration.hydratedBytes,
},
};
};
@@ -1,6 +1,7 @@
import { describe, expect, test } from "vitest";
import { assetResourceLimits } from "@webstudio-is/sdk";
import {
getAssetResourceParameterFieldPaths,
AssetResourceQueryValidationError,
getAssetResourceReferencedFieldPaths,
getAssetResourceReferencedFieldPathsFromTree,
@@ -21,6 +22,18 @@ const expectValidationCode = (
};
describe("asset resource query validation", () => {
test("finds unambiguous fields compared with route parameters", () => {
expect(
getAssetResourceParameterFieldPaths(
"*[properties.slug == $slug && $locale == properties.locale]"
)
).toEqual(
new Map([
["locale", ["properties", "locale"]],
["slug", ["properties", "slug"]],
])
);
});
test("returns one parsed analysis for a valid parameterized query", () => {
expect(
validateAssetResourceQuery(
@@ -147,6 +147,71 @@ const getAttributePath = (node: unknown): string | undefined => {
: appendAssetFieldPath(basePath, attribute.name);
};
const getAttributePathSegments = (node: unknown): string[] | undefined => {
if (
typeof node !== "object" ||
node === null ||
Reflect.get(node, "type") !== "AccessAttribute" ||
typeof Reflect.get(node, "name") !== "string"
) {
return;
}
const base = Reflect.get(node, "base");
const baseSegments = base === undefined ? [] : getAttributePathSegments(base);
if (baseSegments === undefined) {
return;
}
return [...baseSegments, Reflect.get(node, "name") as string];
};
const getParameterName = (node: unknown) =>
typeof node === "object" &&
node !== null &&
Reflect.get(node, "type") === "Parameter" &&
typeof Reflect.get(node, "name") === "string"
? (Reflect.get(node, "name") as string)
: undefined;
/**
* Returns the document field directly compared with each GROQ parameter.
* Ambiguous parameters are omitted instead of guessing a prerender identity.
*/
export const getAssetResourceParameterFieldPaths = (query: string) => {
const paths = new Map<string, string[] | undefined>();
const add = (parameterName: string, path: string[]) => {
const previous = paths.get(parameterName);
if (previous === undefined && paths.has(parameterName) === false) {
paths.set(parameterName, path);
return;
}
if (JSON.stringify(previous) !== JSON.stringify(path)) {
paths.set(parameterName, undefined);
}
};
visitGroqAst(validateAssetResourceQuery(query).tree, (node) => {
if (node.type !== "OpCall" || Reflect.get(node, "op") !== "==") {
return;
}
const left = Reflect.get(node, "left");
const right = Reflect.get(node, "right");
const leftParameter = getParameterName(left);
const rightParameter = getParameterName(right);
const leftPath = getAttributePathSegments(left);
const rightPath = getAttributePathSegments(right);
if (leftParameter !== undefined && rightPath !== undefined) {
add(leftParameter, rightPath);
}
if (rightParameter !== undefined && leftPath !== undefined) {
add(rightParameter, leftPath);
}
});
return new Map(
[...paths]
.filter((entry): entry is [string, string[]] => entry[1] !== undefined)
.sort(([left], [right]) => left.localeCompare(right))
);
};
export const getAssetResourceReferencedFieldPathsFromTree = (
tree: ExprNode
) => {
+2
View File
@@ -0,0 +1,2 @@
export * from "./query-validation";
export * from "./query-execution";
@@ -342,7 +342,7 @@ describe("canonical asset metadata synchronization", () => {
).resolves.toEqual({ status: "deleted" });
});
test("keeps existing canonical metadata when changed Markdown has invalid YAML", async () => {
test("persists standard metadata and a parse error for invalid YAML", async () => {
const source = "---\ntitle: [broken\n---\nBody";
const bytes = encoder.encode(source);
const readFile = vi.fn<AssetClient["readFile"]>(async () => ({
@@ -369,7 +369,16 @@ describe("canonical asset metadata synchronization", () => {
},
])
),
db.get("AssetFolder", () => json([]))
db.get("AssetFolder", () => json([])),
db.post("rpc/replace_asset_file_metadata", async ({ request }) => {
const input = (await request.json()) as ReplaceMetadataRpcArgs;
expect(input.p_document).toMatchObject({
_id: "asset-1",
properties: {},
metadataError: { code: "FRONTMATTER_INVALID" },
});
return json(true);
})
);
await expect(
@@ -379,7 +388,7 @@ describe("canonical asset metadata synchronization", () => {
client: testContext.postgrest.client,
assetClient: { uploadFile: vi.fn(), readFile },
})
).rejects.toMatchObject({ code: "FRONTMATTER_INVALID" });
).resolves.toEqual(expect.anything());
expect(readFile).toHaveBeenCalledOnce();
});
@@ -2,6 +2,7 @@ import {
createCanonicalAssetFileEntry,
extractMarkdownBodyAndExcerpt,
extractMarkdownFrontmatter,
MarkdownMetadataError,
normalizeAssetFileDocument,
} from "@webstudio-is/asset-resource";
import {
@@ -113,12 +114,14 @@ const createCanonicalDocument = ({
revision,
properties,
excerpt,
metadataError,
}: {
asset: UploadedAssetRow;
hierarchy: AssetFolderHierarchy;
revision: string;
properties: Record<string, unknown>;
excerpt?: string;
metadataError?: AssetFileDocument["metadataError"];
}) => {
const name = formatAssetName({
name: asset.file.name,
@@ -140,6 +143,7 @@ const createCanonicalDocument = ({
},
properties,
...(excerpt === undefined || excerpt.length === 0 ? {} : { excerpt }),
...(metadataError === undefined ? {} : { metadataError }),
});
};
@@ -171,6 +175,7 @@ const indexCanonicalAsset = async ({
}) => {
let properties: Record<string, unknown> = {};
let excerpt: string | undefined;
let metadataError: AssetFileDocument["metadataError"];
if (markdownExtension.test(asset.file.name)) {
const prefixLength = Math.min(
asset.file.size,
@@ -182,12 +187,22 @@ const indexCanonicalAsset = async ({
: await assetClient
.readFile(asset.file.name, { offset: 0, length: prefixLength })
.then((stored) => readPrefix(stored.data, prefixLength));
const markdown = await Promise.all([
extractMarkdownFrontmatter(bytes),
extractMarkdownBodyAndExcerpt(bytes),
]);
properties = markdown[0].properties;
excerpt = markdown[1].excerpt;
try {
properties = (await extractMarkdownFrontmatter(bytes)).properties;
} catch (error) {
if (error instanceof MarkdownMetadataError === false) {
throw error;
}
metadataError = { code: error.code, message: error.message };
}
try {
excerpt = (await extractMarkdownBodyAndExcerpt(bytes)).excerpt;
} catch (error) {
if (error instanceof MarkdownMetadataError === false) {
throw error;
}
metadataError ??= { code: error.code, message: error.message };
}
}
const revision = createAssetContentRevision({
storageName: asset.file.name,
@@ -200,6 +215,7 @@ const indexCanonicalAsset = async ({
revision,
properties,
excerpt,
metadataError,
});
await replaceCanonicalAssetFileEntry({
client,
@@ -311,6 +327,7 @@ export const synchronizeCanonicalAssets = async ({
revision,
properties: current.document.properties,
excerpt: current.document.excerpt,
metadataError: current.document.metadataError,
});
if (
assetEntries.length === 1 &&
@@ -209,4 +209,48 @@ describe("previewAssetResourceQuery", () => {
length: 8,
});
});
test("applies static candidate selection before the candidate limit", async () => {
hasProjectPermit.mockResolvedValue(true);
loadCanonicalAssetFileEntries.mockResolvedValue(
Array.from({ length: 1001 }, (_, index) => {
const selected = index === 1000;
const assetId = `asset-${index}`;
return {
projectId,
assetId,
revision,
document: {
_id: assetId,
_type: "asset.file" as const,
name: `${index}.md`,
path: selected ? "blog/selected.md" : `other/${index}.md`,
key: `${index}`,
extension: "md",
mimeType: "text/markdown",
size: 1,
revision,
contentRef: `assets/${assetId}`,
properties: {},
},
fieldContributions: [],
};
})
);
const result = await previewAssetResourceQuery({
projectId,
request: {
query: '*[path match "blog/**"]{_id}',
parameters: {},
resultLimit: 1,
content: { mode: "none" },
},
context,
assetClient: { readFile: vi.fn() },
dependencies,
});
expect(result.result).toEqual([{ _id: "asset-1000" }]);
});
});
+10 -5
View File
@@ -1,6 +1,6 @@
import {
buildAssetResourceIndex,
computeAssetResourceQueryHash,
computeCanonicalAssetRevision,
executeAndHydrateAssetResourceQuery,
} from "@webstudio-is/asset-resource";
import type { AssetResourceQueryRequest } from "@webstudio-is/sdk";
@@ -32,13 +32,18 @@ export const previewAssetResourceQuery = async ({
"You don't have access to preview this project's asset resources",
dependencies,
});
const assetRevision = await computeCanonicalAssetRevision(entries);
const index = await buildAssetResourceIndex({
projectId,
resourceId: "preview",
query: request.query,
entries,
});
return await executeAndHydrateAssetResourceQuery({
request,
documents: entries.map(({ document }) => document),
documents: index.documents,
queryHash: await computeAssetResourceQueryHash(request.query),
indexRevision: `metadata:${assetRevision}`,
assetRevision,
indexRevision: `metadata:${index.integrity.checksum}`,
assetRevision: index.assetRevision,
read: assetClient.readFile,
});
};
@@ -105,7 +105,7 @@ describe("resource index garbage collection", () => {
).resolves.toEqual({ claimed: 1, deleted: 0, missing: 1 });
});
test("drains every available batch", async () => {
test("processes at most one batch", async () => {
let claims = 0;
server.use(
db.post("rpc/claim_asset_resource_index_garbage", () => {
@@ -133,7 +133,7 @@ describe("resource index garbage collection", () => {
limit: 2,
});
expect(claims).toBe(2);
expect(claims).toBe(1);
expect(remove).toHaveBeenCalledTimes(2);
});
});
@@ -109,16 +109,7 @@ export const collectAssetResourceIndexGarbageBestEffort = async ({
limit?: number;
}) => {
try {
while (true) {
const result = await collectAssetResourceIndexGarbage({
client,
store,
limit,
});
if (result.claimed < limit) {
break;
}
}
await collectAssetResourceIndexGarbage({ client, store, limit });
} catch (error) {
console.error("Asset resource index cleanup failed", error);
}
@@ -59,7 +59,7 @@ describe("asset resource query lifecycle", () => {
).toBeUndefined();
});
test("rebuilds all current queries from one canonical load", async () => {
test("rebuilds only created and changed queries from one canonical load", async () => {
const previous = [
resource("changed", "*[false]"),
resource("unchanged", "*[]"),
@@ -83,11 +83,11 @@ describe("asset resource query lifecycle", () => {
})
).resolves.toEqual({
deletedResourceIds: [],
updatedResourceIds: ["changed", "created", "unchanged"],
updatedResourceIds: ["changed", "created"],
});
expect(loadCanonicalAssetFileSnapshot).toHaveBeenCalledOnce();
expect(synchronizeCanonicalAssets).toHaveBeenCalledOnce();
expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(3);
expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(2);
expect(collectAssetResourceIndexGarbageBestEffort).toHaveBeenCalledOnce();
});
@@ -62,7 +62,10 @@ export const synchronizeAssetResourceIndexQueries = async ({
.map(([resourceId]) => resourceId)
.sort();
const changed = [...current]
.filter(([, query]) => query !== undefined)
.filter(
([resourceId, query]) =>
query !== undefined && previous.get(resourceId) !== query
)
.map(([resourceId, query]) => ({ resourceId, query: query as string }))
.sort((left, right) => left.resourceId.localeCompare(right.resourceId));
@@ -211,6 +211,7 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({
metadataSnapshot,
read,
referenceId,
currentResourceIds = [],
}: {
client: Client;
store: ImmutableAssetResourceIndexStore;
@@ -225,6 +226,7 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({
metadataSnapshot: CanonicalAssetMetadataSnapshot;
read: (name: string) => Promise<{ data: AsyncIterable<Uint8Array> }>;
referenceId: string;
currentResourceIds?: readonly string[];
}): Promise<AssetResourceIndexSnapshot[]> => {
if (resources.length === 0) {
return [];
@@ -241,12 +243,14 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({
const states = new Map(
(statesResult.data ?? []).map((state) => [state.resourceId, state])
);
const explicitlyCurrent = new Set(currentResourceIds);
const currentResources = resources.filter((resource) => {
const state = states.get(resource.resourceId);
return (
state !== undefined &&
state.deletedAt === null &&
state.queryHash === resource.queryHash
explicitlyCurrent.has(resource.resourceId) ||
(state !== undefined &&
state.deletedAt === null &&
state.queryHash === resource.queryHash)
);
});
await reconcileAssetResourceIndexesForPublication({
+24 -2
View File
@@ -12,13 +12,14 @@ import {
type Asset,
} from "@webstudio-is/sdk";
import type { Database } from "@webstudio-is/postgrest/index.server";
import type { AssetUploadClient } from "./client";
import type { AssetClient, AssetUploadClient } from "./client";
import type { AssetDataOverride } from "./utils/get-asset-data";
import { createUniqueAssetFilename } from "./utils/get-unique-filename";
import { sanitizeS3Key } from "./utils/sanitize-s3-key";
import { formatAsset } from "./utils/format-asset";
import { assertPostgrestSuccess } from "./patch-utils";
import type { UploadTicket } from "./types";
import { synchronizeAssetResourceStateAfterAssetChange } from "./resource-index-maintenance";
type UploadData = {
projectId: string;
@@ -514,12 +515,33 @@ export const uploadFile = async (
if (typeof projectId !== "string") {
throw Error("File uploader project is missing");
}
return await getUploadedAsset({
const asset = await getUploadedAsset({
name,
projectId,
context,
file,
});
const readableClient = client as Partial<AssetClient>;
if (
readableClient.readFile !== undefined &&
readableClient.resourceIndexStore !== undefined
) {
try {
await synchronizeAssetResourceStateAfterAssetChange({
client: context.postgrest.client,
assetClient: readableClient as AssetClient & {
resourceIndexStore: NonNullable<AssetClient["resourceIndexStore"]>;
},
projectId,
assetId: asset.id,
});
} catch (error) {
// The upload has committed. Derived metadata and indexes are repaired
// by the next preview or publication if this best-effort update fails.
console.error("Asset upload resource synchronization failed", error);
}
}
return asset;
} catch (error) {
await cleanupUploadError(name, context, onUploadError);
throw error;
@@ -247,6 +247,17 @@ export const apiCommandMetadata = publicApiOperations.map((operation) => ({
export const highLevelCliCommands = [
{ command: "audit", operation: "audit" },
{ command: "permissions", operation: "permissions" },
{ command: "get-assets-resource", operation: "get-assets-resource" },
{ command: "create-assets-resource", operation: "create-assets-resource" },
{ command: "update-assets-resource", operation: "update-assets-resource" },
{
command: "get-asset-resource-index-status",
operation: "get-asset-resource-index-status",
},
{
command: "rebuild-asset-resource-index",
operation: "rebuild-asset-resource-index",
},
{
command: "get-marketplace-product",
operation: "get-marketplace-product",
+39
View File
@@ -2877,6 +2877,45 @@ const apiCommandHandlers: Partial<Record<ApiCommandName, ApiCommandHandler>> = {
dependencies
);
},
"get-assets-resource": async (options, connection, dependencies) =>
runProjectSessionCommand(
"get-assets-resource",
{ resourceId: requireOption(options.resource, "--resource") },
connection,
dependencies
),
"create-assets-resource": async (options, connection, dependencies) =>
runProjectSessionCommand(
"create-assets-resource",
await readInputObject(dependencies, options),
connection,
dependencies
),
"update-assets-resource": async (options, connection, dependencies) =>
runProjectSessionCommand(
"update-assets-resource",
await readInputObject(dependencies, options),
connection,
dependencies
),
"get-asset-resource-index-status": async (
options,
connection,
dependencies
) =>
runProjectSessionCommand(
"get-asset-resource-index-status",
{ resourceId: requireOption(options.resource, "--resource") },
connection,
dependencies
),
"rebuild-asset-resource-index": async (options, connection, dependencies) =>
runProjectSessionCommand(
"rebuild-asset-resource-index",
{ resourceId: requireOption(options.resource, "--resource") },
connection,
dependencies
),
"create-resource": async (options, connection, dependencies) => {
const input = {
resource: (await getResourceFields(
+3
View File
@@ -472,6 +472,9 @@ const apiCommandsByArea = {
projectQuality: cliCommandMetadata
.filter(({ cliCommand }) => cliCommand === "audit")
.map(({ cliCommand }) => cliCommand),
assetsAndIndexes: cliCommandMetadata
.filter(({ cliCommand }) => cliCommand.includes("asset"))
.map(({ cliCommand }) => cliCommand),
publishAndDomains: cliCommandMetadata
.filter(
({ cliCommand }) =>
+31 -10
View File
@@ -1,6 +1,6 @@
import { join } from "node:path";
import { readFile } from "node:fs/promises";
import { isPathnamePattern } from "@webstudio-is/sdk";
import { isPathnamePattern, matchPathnameParams } from "@webstudio-is/sdk";
import {
baseComponentImportSource,
createFrameworkComponentRegistry,
@@ -16,7 +16,16 @@ const generateVikeRoute = (pagePath: string) => {
if (pagePath === "/") {
return "index";
}
return pagePath;
let route = pagePath;
const matches = [...matchPathnameParams(pagePath)].reverse();
for (const match of matches) {
const name = match.groups?.name;
if (name === undefined || match.index === undefined) {
continue;
}
route = `${route.slice(0, match.index)}@${name}${route.slice(match.index + match[0].length)}`;
}
return route;
};
export const createFramework = async (
@@ -49,25 +58,37 @@ export const createFramework = async (
select: `${baseComponentImportSource}:Select`,
a: `${baseComponentImportSource}:Link`,
},
html: ({ pagePath }: { pagePath: string }) => {
// ignore dynamic pages in static export
if (isPathnamePattern(pagePath)) {
return [];
html: ({ pagePath, prerenderPaths = [] }) => {
const dynamic = isPathnamePattern(pagePath);
if (dynamic && prerenderPaths.length === 0) {
throw new Error(
`Dynamic SSG page ${JSON.stringify(pagePath)} has no enumerable Assets query paths`
);
}
return [
const route = generateVikeRoute(pagePath);
const entries = [
{
file: join("pages", generateVikeRoute(pagePath), "+Page.tsx"),
file: join("pages", route, "+Page.tsx"),
template: htmlPageTemplate,
},
{
file: join("pages", generateVikeRoute(pagePath), "+Head.tsx"),
file: join("pages", route, "+Head.tsx"),
template: htmlHeadTemplate,
},
{
file: join("pages", generateVikeRoute(pagePath), "+data.ts"),
file: join("pages", route, "+data.ts"),
template: htmlDataTemplate,
},
];
if (dynamic) {
entries.push({
file: join("pages", route, "+onBeforePrerenderStart.ts"),
template: `export const onBeforePrerenderStart = () => ${JSON.stringify(
prerenderPaths
)};\n`,
});
}
return entries;
},
xml: () => [],
text: () => [],
+4 -1
View File
@@ -25,7 +25,10 @@ export type Framework = {
components: Record<string, string>;
// instance.tag: "importSource:importSpecifier"
tags: Record<string, string>;
html: (params: { pagePath: string }) => FrameworkTemplateEntry[];
html: (params: {
pagePath: string;
prerenderPaths?: readonly string[];
}) => FrameworkTemplateEntry[];
xml: (params: { pagePath: string }) => FrameworkTemplateEntry[];
text: (params: { pagePath: string }) => FrameworkTemplateEntry[];
defaultSitemap: () => FrameworkTemplateEntry[];
+123 -17
View File
@@ -430,7 +430,9 @@ test("materializes immutable resource indexes as public JSON with a reference-on
"app/__generated__/$resources.asset-query-runtime.ts",
"utf8"
);
expect(runtimeModule).toContain('from "@webstudio-is/asset-resource"');
expect(runtimeModule).toContain(
'from "@webstudio-is/asset-resource/runtime"'
);
expect(runtimeModule).not.toContain('"documents"');
expect(runtimeModule).not.toContain('"properties"');
});
@@ -1016,7 +1018,21 @@ describe("prebuild", () => {
path.includes("$slug")
)
).toHaveLength(1);
});
await symlink(join(originalCwd, "node_modules"), "node_modules", "dir");
await runGeneratedCommand("react-router", ["build"]);
const serverBundle = (
await Promise.all(
(
await getFilePaths("build/server")
)
.filter((path) => path.endsWith(".js"))
.map((path) => readFile(path, "utf8"))
)
).join("\n");
expect(serverBundle).not.toContain("Published post");
expect(serverBundle).not.toContain("draft secret");
expect(serverBundle).not.toContain("post-revision");
}, 30_000);
test("uses pass-through images in the base react-router template", async () => {
await prebuild({ assets: false, template: ["react-router"] });
@@ -1087,7 +1103,7 @@ describe("prebuild", () => {
);
});
test("selects ssg templates and skips dynamic routes", async () => {
test("selects ssg templates", async () => {
await writeSiteData(
createSiteData({
pages: [
@@ -1099,14 +1115,6 @@ describe("prebuild", () => {
rootInstanceId: "root",
meta: {},
},
{
id: "post",
name: "Post",
title: "Post",
path: "/blog/:slug",
rootInstanceId: "root",
meta: {},
},
],
})
);
@@ -1122,12 +1130,6 @@ describe("prebuild", () => {
await expect(readFile("pages/index/+data.ts", "utf8")).resolves.toContain(
"../app/__generated__/_index.server"
);
await expect(
readFile("pages/blog/:slug/+Page.tsx", "utf8")
).rejects.toThrow("ENOENT");
await expect(
readFile("app/__generated__/[blog].$slug._index.tsx", "utf8")
).resolves.toContain("export { Page }");
await expect(readFile("pages/index/+data.ts", "utf8")).resolves.toContain(
"createSsgAssetResourceFetch"
);
@@ -1137,6 +1139,110 @@ describe("prebuild", () => {
);
});
test("prerenders dynamic SSG paths from parameterized Assets resources", async () => {
const query =
'*[properties.slug == $slug][0]{_id, "title": properties.title}';
const index = await createAssetResourceIndex({
format: "webstudio-resource-index",
version: 1,
resourceId: "post",
query,
assetRevision: `sha256:${"a".repeat(64)}`,
queryMode: "parameterized",
parameterNames: ["slug"],
documents: [
{
_id: "post-1",
_type: "asset.file",
name: "post.md",
path: "blog/post.md",
key: "post",
extension: "md",
mimeType: "text/markdown",
size: 1,
revision: "post-revision",
contentRef: "post.md",
properties: { slug: "hello-world", title: "Hello" },
},
],
});
const siteData = {
...createSiteData({
pages: [
{
id: "home",
name: "Home",
title: "Home",
path: "",
rootInstanceId: "home-root",
meta: {},
},
{
id: "post",
name: "Post",
title: "Post",
path: "/blog/:slug",
rootInstanceId: "post-root",
meta: {},
},
],
instances: [
["home-root", { id: "home-root", component: "Box", children: [] }],
["post-root", { id: "post-root", component: "Box", children: [] }],
],
}),
assetResourceIndexes: [
{
resourceId: index.resourceId,
revision: index.integrity.checksum,
index,
},
],
};
siteData.build.resources = [
[
"post",
{
id: "post",
name: "Post",
control: "system",
method: "post",
url: '"/$resources/assets/query"',
headers: [],
body: createAssetQueryResourceBody({
query,
parameters: [{ name: "slug", value: "system.params.slug" }],
resultLimit: 1,
}),
},
],
] as never;
siteData.build.dataSources = [
[
"post-data",
{
id: "post-data",
type: "resource",
name: "post",
resourceId: "post",
scopeInstanceId: "post-root",
},
],
] as never;
await writeSiteData(siteData);
await prebuild({ assets: false, template: ["ssg"] });
await expect(
readFile("pages/blog/@slug/+onBeforePrerenderStart.ts", "utf8")
).resolves.toContain("/blog/hello-world");
await symlink(join(originalCwd, "node_modules"), "node_modules", "dir");
await runGeneratedCommand("vite", ["build"]);
await runGeneratedCommand("vike", ["prerender"]);
await expect(
readFile("dist/client/blog/hello-world/index.html", "utf8")
).resolves.toContain("<!DOCTYPE html>");
}, 30_000);
test("prerenders SSG pages with asset query data", async () => {
const query = '*[]{"title": properties.title}';
const index = await createAssetResourceIndex({
+149 -2
View File
@@ -36,6 +36,7 @@ import {
createScope,
findTreeInstanceIds,
getAllPages,
getAssetResourceQuery,
getPagePath,
getPublishablePages,
generateResources,
@@ -50,8 +51,11 @@ import {
elementComponent,
toRuntimeAsset,
assetResourceContentOptions,
assetResourceLimits,
matchPathnameParams,
parseAssetQueryResourceBody,
} from "@webstudio-is/sdk";
import { getAssetResourceParameterFieldPaths } from "@webstudio-is/asset-resource";
import { migratePages } from "@webstudio-is/project-migrations/pages";
import { collectFontFamiliesFromStyleDecls } from "@webstudio-is/project-build/runtime";
import {
@@ -99,6 +103,141 @@ type SiteDataByPage = {
};
};
const getExpressionMemberPath = (node: unknown): string[] | undefined => {
if (typeof node !== "object" || node === null) {
return;
}
if (
Reflect.get(node, "type") === "Identifier" &&
typeof Reflect.get(node, "name") === "string"
) {
return [Reflect.get(node, "name") as string];
}
if (Reflect.get(node, "type") !== "MemberExpression") {
return;
}
const base = getExpressionMemberPath(Reflect.get(node, "object"));
if (base === undefined) {
return;
}
const computed = Reflect.get(node, "computed") === true;
const property = Reflect.get(node, "property");
const name = computed
? Reflect.get(property, "value")
: Reflect.get(property, "name");
return typeof name === "string" ? [...base, name] : undefined;
};
const getBoundSystemRouteParameter = (expression: string) => {
try {
const program = parse(`(${expression})`, { ecmaVersion: "latest" });
const statement = program.body[0];
const node =
statement?.type === "ExpressionStatement"
? statement.expression
: undefined;
const path = getExpressionMemberPath(node);
return path?.length === 3 && path[0] === "system" && path[1] === "params"
? path[2]
: undefined;
} catch {
return;
}
};
const getDocumentPathValue = (document: unknown, path: readonly string[]) => {
let value = document;
for (const segment of path) {
if (typeof value !== "object" || value === null) {
return;
}
value = Reflect.get(value, segment);
}
return value;
};
export const getAssetResourcePrerenderPaths = ({
pagePath,
resources,
indexes,
}: {
pagePath: string;
resources: readonly [string, Resource][];
indexes: PublishedProjectBundle["assetResourceIndexes"];
}) => {
const pathParameters = [...matchPathnameParams(pagePath)];
if (
pathParameters.length === 0 ||
pathParameters.some(
(match) =>
match.groups?.name === undefined || (match.groups.modifier ?? "") !== ""
)
) {
return [];
}
const routeParameterNames = new Set(
pathParameters.map((match) => match.groups?.name as string)
);
const indexByResourceId = new Map(
(indexes ?? []).map((snapshot) => [snapshot.resourceId, snapshot.index])
);
const paths = new Set<string>();
for (const [, resource] of resources) {
const index = indexByResourceId.get(resource.id);
if (index === undefined) {
continue;
}
const query = getAssetResourceQuery(resource);
if (query === undefined) {
continue;
}
const parameterFields = getAssetResourceParameterFieldPaths(query);
const routeFields = new Map<string, string[]>();
for (const binding of parseAssetQueryResourceBody(resource.body)
.parameters) {
const routeParameter = getBoundSystemRouteParameter(binding.value);
const fieldPath = parameterFields.get(binding.name);
if (
routeParameter !== undefined &&
routeParameterNames.has(routeParameter) &&
fieldPath !== undefined
) {
routeFields.set(routeParameter, fieldPath);
}
}
if (routeFields.size !== routeParameterNames.size) {
continue;
}
for (const document of index.documents) {
const values = new Map<string, string>();
for (const [name, fieldPath] of routeFields) {
const value = getDocumentPathValue(document, fieldPath);
if (
(typeof value === "string" && value.length > 0) ||
(typeof value === "number" && Number.isFinite(value)) ||
typeof value === "boolean"
) {
values.set(name, String(value));
}
}
if (values.size !== routeParameterNames.size) {
continue;
}
let path = pagePath;
for (const match of [...pathParameters].reverse()) {
const name = match.groups?.name as string;
const value = values.get(name) as string;
path = `${path.slice(0, match.index)}${encodeURIComponent(value)}${path.slice((match.index ?? 0) + match[0].length)}`;
}
paths.add(path);
if (paths.size > assetResourceLimits.candidateDocuments) {
throw new Error("Dynamic SSG path count exceeds the Assets limit");
}
}
}
return [...paths].sort();
};
const mergeJsonInto = async (sourcePath: string, destinationPath: string) => {
const sourceJson = await readFile(sourcePath, "utf8");
const destinationJson = await readFile(destinationPath, "utf8").catch(
@@ -215,7 +354,7 @@ export const generateAssetQueryRuntimeModule = ({
if (manifest.length === 0) {
return `export const createGeneratedAssetResourceFetch = async ({ fallback }: ${inputType}): Promise<typeof fetch> => fallback;\n`;
}
return `import { createGeneratedAssetResourceFetch as createRuntimeFetch } from "@webstudio-is/asset-resource";
return `import { createGeneratedAssetResourceFetch as createRuntimeFetch } from "@webstudio-is/asset-resource/runtime";
const deploymentId = ${JSON.stringify(deploymentId)};
const manifest = ${JSON.stringify(manifest, null, 2)};
@@ -970,7 +1109,15 @@ export const prebuild = async (options: {
await writeGeneratedFile(serverFile, serverExports);
const getTemplates = framework[documentType];
for (const { file, template } of getTemplates({ pagePath })) {
const prerenderPaths = getAssetResourcePrerenderPaths({
pagePath,
resources: pageData.build.resources,
indexes: siteData.assetResourceIndexes,
});
for (const { file, template } of getTemplates({
pagePath,
prerenderPaths,
})) {
const content = template
.replaceAll("__CONSTANTS__", importFrom("./app/constants.mjs", file))
.replaceAll(
+2 -2
View File
@@ -1,5 +1,5 @@
declare module "__ASSET_QUERY_MANIFEST__" {
import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource";
import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource/runtime";
export const assetQueryDeploymentId: string;
export const assetQueryManifest: PublishedAssetResourceManifestEntry[];
@@ -14,7 +14,7 @@ declare module "__ASSET_QUERY_RUNTIME__" {
}
declare module "__ASSET_RESOURCE_FETCH__" {
import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource";
import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource/runtime";
export const createSsgAssetResourceFetch: (options: {
deploymentId: string;
@@ -132,7 +132,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
const resources = await loadResources(
generatedFetch,
getResources({ system }).data,
url
url,
{ signal: arg.request.signal }
);
const pageMeta = getPageMeta({ system, resources });
@@ -91,7 +91,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
const resources = await loadResources(
generatedFetch,
getResources({ system }).data,
url
url,
{ signal: arg.request.signal }
);
const pageMeta = getPageMeta({ system, resources });
@@ -99,7 +99,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
const resources = await loadResources(
generatedFetch,
getResources({ system }).data,
url
url,
{ signal: arg.request.signal }
);
const pageMeta = getPageMeta({ system, resources });
@@ -131,7 +131,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
const resources = await loadResources(
generatedFetch,
getResources({ system }).data,
url
url,
{ signal: arg.request.signal }
);
const pageMeta = getPageMeta({ system, resources });
@@ -91,7 +91,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
const resources = await loadResources(
generatedFetch,
getResources({ system }).data,
url
url,
{ signal: arg.request.signal }
);
const pageMeta = getPageMeta({ system, resources });
@@ -99,7 +99,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
const resources = await loadResources(
generatedFetch,
getResources({ system }).data,
url
url,
{ signal: arg.request.signal }
);
const pageMeta = getPageMeta({ system, resources });
@@ -1,6 +1,6 @@
import { open } from "node:fs/promises";
import { resolve, sep } from "node:path";
import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource";
import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource/runtime";
const resolvePublicAsset = (path: string) => {
const publicDirectory = resolve("public");
+4
View File
@@ -1647,6 +1647,7 @@ test("imports project bundle through staged upload", async () => {
projectId: "project-id",
data: {
largeContent: "x".repeat(3 * 1024 * 1024 + 1),
assetResourceIndexes: [{ marker: "derived-index-marker" }],
} as unknown as PublishedProjectBundle,
})
).resolves.toEqual({ version: 2 });
@@ -1657,6 +1658,9 @@ test("imports project bundle through staged upload", async () => {
}
expect(uploadChunks).toHaveLength(2);
expect(Buffer.concat(uploadChunks).toString("utf8")).not.toContain(
"derived-index-marker"
);
expect(JSON.stringify(trpcBody)).toContain('"uploadId":"upload-id"');
expect(JSON.stringify(trpcBody)).not.toContain("largeContent");
});
+5 -1
View File
@@ -1811,7 +1811,11 @@ const uploadProjectBundleData = async (
) => {
const { sourceOrigin } = parseBuilderUrl(params.origin);
const endpoint = new URL(stagedUploadPath, sourceOrigin);
const data = JSON.stringify(params.data);
// Resource indexes are derived deployment artifacts. Import rebuilds them
// from the destination project's assets and must not transport stale blobs.
const { assetResourceIndexes: _assetResourceIndexes, ...portableData } =
params.data;
const data = JSON.stringify(portableData);
if (new TextEncoder().encode(data).byteLength > maxProjectBundleSize) {
throw new Error(
`Project bundle is too large to import. Maximum size is ${formatMebibytes(maxProjectBundleSize)}.`
@@ -160,13 +160,30 @@ VALUES ('resource-index-test.md', 'file', 10, 'UPLOADED');
INSERT INTO "Asset" (id, "projectId", name)
VALUES ('resource-index-test-asset', 'resource-index-test-project', 'resource-index-test.md');
SELECT throws_ok(
$$
INSERT INTO "AssetFileMetadata" (
"projectId", "assetId", revision, document, "fieldContributions"
) VALUES (
'resource-index-test-project',
'resource-index-test-asset',
'file:resource-index-test.md:invalid',
'{}'::JSONB,
'[]'::JSONB
)
$$,
'23514',
NULL,
'Canonical metadata requires explicit matching identity fields'
);
INSERT INTO "AssetFileMetadata" (
"projectId", "assetId", revision, document, "fieldContributions"
) VALUES (
'resource-index-test-project',
'resource-index-test-asset',
'file:resource-index-test.md:1',
'{}'::JSONB,
'{"_id":"resource-index-test-asset","revision":"file:resource-index-test.md:1"}'::JSONB,
'[]'::JSONB
);
@@ -545,10 +562,89 @@ SELECT is(
'Repeated query deletion is idempotent'
);
INSERT INTO "AssetResourceIndexState" (
"projectId", "resourceId", query, "queryHash", "assetRevision",
"buildAttemptId", "buildStatus"
) VALUES (
'resource-index-test-project',
'project-delete-test',
'*[]',
'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
'project-delete-attempt',
'BUILDING'
);
INSERT INTO "AssetResourceIndexRevision" (
"projectId", "resourceId", revision, "queryHash", "assetRevision",
checksum, "objectKey"
) VALUES (
'resource-index-test-project',
'project-delete-test',
'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc',
'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc',
'projects/test/resources/project-delete-test.json'
);
UPDATE "AssetResourceIndexState"
SET "activeRevision" =
'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc',
"buildStatus" = 'ACTIVE'
WHERE "projectId" = 'resource-index-test-project'
AND "resourceId" = 'project-delete-test';
UPDATE "Project"
SET "isDeleted" = TRUE
WHERE id = 'resource-index-test-project';
SELECT ok(
(
SELECT "activeRevision" IS NULL AND "deletedAt" IS NOT NULL
FROM "AssetResourceIndexState"
WHERE "projectId" = 'resource-index-test-project'
AND "resourceId" = 'project-delete-test'
),
'Soft project deletion retires active resource indexes'
);
SELECT ok(
(
SELECT "unreferencedAt" IS NOT NULL
FROM "AssetResourceIndexRevision"
WHERE "projectId" = 'resource-index-test-project'
AND "resourceId" = 'project-delete-test'
),
'Soft project deletion makes immutable indexes collectible'
);
DELETE FROM "Build"
WHERE "id" = 'resource-index-test-build'
AND "projectId" = 'resource-index-test-project';
SELECT throws_ok(
$$ DELETE FROM "Project" WHERE id = 'resource-index-test-project' $$,
'23503',
NULL,
'Hard project deletion cannot orphan immutable resource indexes'
);
CREATE TEMP TABLE project_delete_garbage AS
SELECT * FROM claim_asset_resource_index_garbage('2100-01-01', 10)
WHERE "resourceId" = 'project-delete-test';
SELECT is(
finalize_asset_resource_index_garbage(
(SELECT "projectId" FROM project_delete_garbage),
(SELECT "resourceId" FROM project_delete_garbage),
(SELECT revision FROM project_delete_garbage),
(SELECT "gcClaimId" FROM project_delete_garbage)
),
TRUE,
'Project deletion cleanup finalizes its immutable resource index'
);
DELETE FROM "Project" WHERE "id" = 'resource-index-test-project';
SELECT is(
@@ -15,11 +15,13 @@ CREATE TABLE "AssetFileMetadata" (
CONSTRAINT "AssetFileMetadata_pkey"
PRIMARY KEY ("projectId", "assetId", "revision"),
CONSTRAINT "AssetFileMetadata_document_identity_check"
CHECK (
CHECK ((
jsonb_typeof("document") = 'object'
AND jsonb_typeof("document"->'_id') = 'string'
AND "document"->>'_id' = "assetId"
AND jsonb_typeof("document"->'revision') = 'string'
AND "document"->>'revision' = "revision"
),
) IS TRUE),
CONSTRAINT "AssetFileMetadata_field_contributions_check"
CHECK (jsonb_typeof("fieldContributions") = 'array'),
CONSTRAINT "AssetFileMetadata_assetId_projectId_fkey"
@@ -27,10 +29,6 @@ CREATE TABLE "AssetFileMetadata" (
ON DELETE CASCADE ON UPDATE CASCADE
);
CREATE INDEX "AssetFileMetadata_projectId_assetId_idx"
ON "AssetFileMetadata"("projectId", "assetId");
-- Canonical metadata lifecycle
CREATE FUNCTION replace_asset_file_metadata(
@@ -244,6 +242,15 @@ CREATE INDEX "AssetResourceIndexRevision_lookup_idx"
"assetRevision"
);
CREATE INDEX "AssetResourceIndexRevision_garbage_idx"
ON "AssetResourceIndexRevision"(
"unreferencedAt",
"projectId",
"resourceId",
"revision"
)
WHERE "unreferencedAt" IS NOT NULL;
ALTER TABLE "AssetResourceIndexState"
ADD CONSTRAINT "AssetResourceIndexState_activeRevision_fkey"
FOREIGN KEY ("projectId", "resourceId", "activeRevision")
@@ -254,6 +261,58 @@ ALTER TABLE "AssetResourceIndexState"
)
ON DELETE NO ACTION ON UPDATE CASCADE;
-- Derived index objects must enter garbage collection before project rows can
-- disappear. Normal project deletion is soft; the guard prevents a later hard
-- delete from losing the only durable object keys.
CREATE FUNCTION retire_asset_resource_indexes_on_project_delete()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
BEGIN
IF OLD."isDeleted" IS DISTINCT FROM TRUE AND NEW."isDeleted" = TRUE THEN
UPDATE public."AssetResourceIndexState"
SET "activeRevision" = NULL,
"buildStatus" = 'STALE',
"buildError" = NULL,
"deletedAt" = COALESCE("deletedAt", CURRENT_TIMESTAMP),
"updatedAt" = CURRENT_TIMESTAMP
WHERE "projectId" = NEW.id;
UPDATE public."AssetResourceIndexRevision"
SET "unreferencedAt" = COALESCE("unreferencedAt", CURRENT_TIMESTAMP)
WHERE "projectId" = NEW.id;
END IF;
RETURN NEW;
END;
$$;
CREATE TRIGGER "Project_retire_asset_resource_indexes"
AFTER UPDATE OF "isDeleted" ON "Project"
FOR EACH ROW
EXECUTE FUNCTION retire_asset_resource_indexes_on_project_delete();
CREATE FUNCTION protect_asset_resource_objects_before_project_delete()
RETURNS TRIGGER
LANGUAGE plpgsql
AS $$
BEGIN
IF EXISTS (
SELECT 1 FROM public."AssetResourceIndexRevision"
WHERE "projectId" = OLD.id
) THEN
RAISE EXCEPTION
'Project asset resource indexes must be collected before hard deletion'
USING ERRCODE = '23503';
END IF;
RETURN OLD;
END;
$$;
CREATE TRIGGER "Project_protect_asset_resource_objects"
BEFORE DELETE ON "Project"
FOR EACH ROW
EXECUTE FUNCTION protect_asset_resource_objects_before_project_delete();
-- Resource index build and activation lifecycle
@@ -975,7 +1034,16 @@ CREATE FUNCTION finalize_asset_resource_index_garbage(
RETURNS BOOLEAN
LANGUAGE plpgsql
AS $$
DECLARE
removed BOOLEAN;
BEGIN
PERFORM pg_catalog.pg_advisory_xact_lock(
pg_catalog.hashtextextended(
pg_catalog.jsonb_build_array(p_project_id, p_resource_id)::TEXT,
0
)
);
DELETE FROM public."AssetResourceIndexRevision" AS candidate
WHERE candidate."projectId" = p_project_id
AND candidate."resourceId" = p_resource_id
@@ -997,7 +1065,22 @@ BEGIN
AND state."resourceId" = candidate."resourceId"
AND state."activeRevision" = candidate.revision
);
RETURN FOUND;
removed := FOUND;
IF removed THEN
DELETE FROM public."AssetResourceIndexState" AS state
WHERE state."projectId" = p_project_id
AND state."resourceId" = p_resource_id
AND state."deletedAt" IS NOT NULL
AND state."activeRevision" IS NULL
AND NOT EXISTS (
SELECT 1 FROM public."AssetResourceIndexRevision" AS revision
WHERE revision."projectId" = state."projectId"
AND revision."resourceId" = state."resourceId"
);
END IF;
RETURN removed;
END;
$$;
+1
View File
@@ -67,6 +67,7 @@
"dependencies": {
"@emotion/hash": "^0.9.2",
"@modelcontextprotocol/sdk": "^1.10.0",
"@webstudio-is/asset-resource": "workspace:*",
"@webstudio-is/authorization-token": "workspace:*",
"@webstudio-is/css-data": "workspace:*",
"@webstudio-is/css-engine": "workspace:*",
@@ -0,0 +1,58 @@
import { describe, expect, test } from "vitest";
import {
assetsResourceCreateInput,
assetsResourceUpdateInput,
} from "./asset-resources";
const baseQuery = {
resultLimit: 20,
content: { mode: "none" as const },
parameters: [],
};
describe("Assets resource mutation input", () => {
test("rejects invalid GROQ on create", () => {
const result = assetsResourceCreateInput.safeParse({
name: "Posts",
scopeInstanceId: "root",
query: { ...baseQuery, groq: "*[" },
});
expect(result.success).toBe(false);
});
test("rejects missing parameter bindings on create and update", () => {
const query = {
...baseQuery,
groq: "*[properties.slug == $slug]",
};
expect(
assetsResourceCreateInput.safeParse({
name: "Posts",
scopeInstanceId: "root",
query,
}).success
).toBe(false);
expect(
assetsResourceUpdateInput.safeParse({
resourceId: "posts",
values: { query },
}).success
).toBe(false);
});
test("accepts a binding for every GROQ parameter", () => {
expect(
assetsResourceCreateInput.safeParse({
name: "Post",
scopeInstanceId: "root",
query: {
...baseQuery,
groq: "*[properties.slug == $slug]",
parameters: [{ name: "slug", value: "system.params.slug" }],
},
}).success
).toBe(true);
});
});
@@ -16,6 +16,7 @@ import {
assetsQueryResourceUrl,
assetsResourceUrl,
} from "@webstudio-is/sdk/runtime";
import { validateAssetResourceQuery } from "@webstudio-is/asset-resource";
import { z } from "zod";
import type { BuilderState } from "../state/builder-state";
import type { BuilderRuntimeContext } from "./context";
@@ -67,16 +68,58 @@ const assetResourceConfigurationFields = {
content: assetResourceContentOptions.default({ mode: "none" }),
};
const storedAssetQueryConfigurationInput = z.object(
assetResourceConfigurationFields
);
const refineAssetQueryConfiguration = (
configuration: {
query: string;
parameters: readonly { name: string }[];
},
context: z.RefinementCtx
) => {
let parameterNames: readonly string[];
try {
parameterNames = validateAssetResourceQuery(
configuration.query
).parameterNames;
} catch (error) {
context.addIssue({
code: "custom",
path: ["query"],
message:
error instanceof Error ? error.message : "Asset query is invalid.",
});
return;
}
const bindings = new Set(
configuration.parameters.map((parameter) => parameter.name)
);
for (const parameterName of parameterNames) {
if (bindings.has(parameterName) === false) {
context.addIssue({
code: "custom",
path: ["parameters"],
message: `Asset query parameter $${parameterName} requires a binding.`,
});
}
}
};
export const assetsQueryConfigurationInput = z.object({
groq: assetResourceConfigurationFields.query,
parameters: assetResourceConfigurationFields.parameters,
resultLimit: assetResourceConfigurationFields.resultLimit,
content: assetResourceConfigurationFields.content,
});
const storedAssetQueryConfigurationInput = z
.object(assetResourceConfigurationFields)
.superRefine(refineAssetQueryConfiguration);
export const assetsQueryConfigurationInput = z
.object({
groq: assetResourceConfigurationFields.query,
parameters: assetResourceConfigurationFields.parameters,
resultLimit: assetResourceConfigurationFields.resultLimit,
content: assetResourceConfigurationFields.content,
})
.superRefine((configuration, context) =>
refineAssetQueryConfiguration(
{ query: configuration.groq, parameters: configuration.parameters },
context
)
);
export const assetsResourceListInput = z.object({
scopeInstanceId: z.string().optional(),
+2 -1
View File
@@ -1,5 +1,6 @@
import { generateObjectExpression, parseObjectExpression } from "./expression";
import { assetsQueryResourceUrl, assetsResourceUrl } from "./resource-loader";
import { assetResourceLimits } from "./schema/asset-resource";
import type { Resource } from "./schema/resources";
export type AssetQueryParameterBinding = {
@@ -34,7 +35,7 @@ export const parseAssetQueryResourceBody = (body: string | undefined) => {
export const createAssetQueryResourceBody = ({
query,
parameters,
resultLimit = 100,
resultLimit = assetResourceLimits.defaultResultCount,
contentExpression = '{ "mode": "none" }',
}: {
query: string;
@@ -37,6 +37,12 @@ export const assetFileDocument = z.strictObject({
contentRef: z.string().min(1),
properties: z.record(z.string(), z.json()),
excerpt: z.string().optional(),
metadataError: z
.strictObject({
code: z.string().min(1),
message: z.string().min(1),
})
.optional(),
});
export type AssetFileDocument = z.infer<typeof assetFileDocument>;
@@ -138,6 +144,9 @@ export const assetResourceLimits = {
resultBytes: 1024 * 1024,
candidateDocuments: 1000,
indexBytes: 16 * 1024 * 1024,
publishedResourceCount: 32,
publishedIndexBytes: 32 * 1024 * 1024,
runtimeCachedIndexes: 4,
frontmatterBytes: 64 * 1024,
frontmatterDepth: 8,
frontmatterFields: 256,
+3
View File
@@ -2047,6 +2047,9 @@ importers:
'@modelcontextprotocol/sdk':
specifier: ^1.10.0
version: 1.10.0
'@webstudio-is/asset-resource':
specifier: workspace:*
version: link:../asset-resource
'@webstudio-is/authorization-token':
specifier: workspace:*
version: link:../authorization-token