mirror of
https://github.com/webstudio-is/webstudio.git
synced 2026-11-03 02:27:57 +09:00
fix: harden asset resource publication
This commit is contained in:
1 parent
eaad08bb13
commit
cc10a4e893
52 files changed
+1367
-313
No files matched your search
@@ -146,5 +146,13 @@ describe("asset query resource body", () => {
|
||||
content: { mode: "none" },
|
||||
})
|
||||
).toContain("$slug");
|
||||
expect(
|
||||
getAssetQueryConfigurationError({
|
||||
query: "*[properties.slug == $slug]",
|
||||
parameters: [{ name: "slug", value: "system.params." }],
|
||||
resultLimit: 1,
|
||||
content: { mode: "none" },
|
||||
})
|
||||
).toContain("Webstudio expression");
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
type AssetResourceIndexStatus,
|
||||
} from "@webstudio-is/sdk";
|
||||
import { validateAssetResourceQuery } from "@webstudio-is/asset-resource";
|
||||
import { getExpressionErrorMessages } from "@webstudio-is/project-build/runtime";
|
||||
export {
|
||||
createAssetQueryResourceBody,
|
||||
parseAssetQueryResourceBody,
|
||||
@@ -58,7 +59,7 @@ export const getAssetQueryConfigurationError = ({
|
||||
content,
|
||||
}: {
|
||||
query: string;
|
||||
parameters: readonly { name: string }[];
|
||||
parameters: readonly { name: string; value?: string }[];
|
||||
resultLimit: number;
|
||||
content: AssetResourceContentOptions;
|
||||
}) => {
|
||||
@@ -84,6 +85,15 @@ export const getAssetQueryConfigurationError = ({
|
||||
if (new Set(names).size !== names.length) {
|
||||
return "Runtime parameter names must be unique.";
|
||||
}
|
||||
if (
|
||||
parameters.some(
|
||||
({ value }) =>
|
||||
value !== undefined &&
|
||||
getExpressionErrorMessages({ expression: value }).length > 0
|
||||
)
|
||||
) {
|
||||
return "Enter a valid Webstudio expression for every runtime parameter.";
|
||||
}
|
||||
const configuredNames = new Set(names);
|
||||
const missingName = referencedParameterNames.find(
|
||||
(name) => configuredNames.has(name) === false
|
||||
|
||||
@@ -306,6 +306,13 @@ const AssetQueryPreview = ({
|
||||
}
|
||||
let ignore = false;
|
||||
let pollTimeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let pollAttempts = 0;
|
||||
const schedulePoll = () => {
|
||||
pollAttempts += 1;
|
||||
if (pollAttempts < 10) {
|
||||
pollTimeout = setTimeout(load, 1000);
|
||||
}
|
||||
};
|
||||
const load = async () => {
|
||||
try {
|
||||
const response = await loadBuilderAssetIndexStatus(resourceId);
|
||||
@@ -316,12 +323,13 @@ const AssetQueryPreview = ({
|
||||
const parsed = assetResourceIndexStatus.safeParse(data.status);
|
||||
const status = parsed.success ? parsed.data : undefined;
|
||||
onIndexStatusChange(status);
|
||||
if (status?.state === "indexing") {
|
||||
pollTimeout = setTimeout(load, 1000);
|
||||
if (status?.state === "indexing" || status === undefined) {
|
||||
schedulePoll();
|
||||
}
|
||||
} catch {
|
||||
if (ignore === false) {
|
||||
onIndexStatusChange(undefined);
|
||||
schedulePoll();
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -448,7 +456,9 @@ export const AssetQueryForm = ({
|
||||
{}
|
||||
);
|
||||
const [resultLimit, setResultLimit] = useState(
|
||||
typeof parsedResultLimit === "number" ? parsedResultLimit : 100
|
||||
typeof parsedResultLimit === "number"
|
||||
? parsedResultLimit
|
||||
: assetResourceLimits.defaultResultCount
|
||||
);
|
||||
const parsedContent = assetResourceContentOptions.safeParse(
|
||||
evaluateExpressionWithinScope(parsedBody.contentExpression ?? "", {})
|
||||
|
||||
@@ -20,12 +20,15 @@ import {
|
||||
findPageByIdOrPath,
|
||||
getAllPages,
|
||||
getStyleDeclKey,
|
||||
assetResourceLimits,
|
||||
isStoredAssetQueryResource,
|
||||
type Asset,
|
||||
type AssetFolder,
|
||||
} from "@webstudio-is/sdk";
|
||||
import {
|
||||
computeAssetResourceQueryHash,
|
||||
computeCanonicalAssetRevision,
|
||||
serializeAssetResourceIndex,
|
||||
} from "@webstudio-is/asset-resource";
|
||||
import { serializePages } from "@webstudio-is/project-migrations/pages";
|
||||
import { loadById } from "@webstudio-is/project/index.server";
|
||||
@@ -164,25 +167,42 @@ const addProjectMetadata = async (
|
||||
? undefined
|
||||
: await getUserById(context, project.userId);
|
||||
|
||||
const reachableResourceIds = new Set<string>();
|
||||
for (const [, prop] of data.build.props) {
|
||||
if (prop.type === "resource") {
|
||||
reachableResourceIds.add(prop.value);
|
||||
}
|
||||
}
|
||||
for (const [, dataSource] of data.build.dataSources) {
|
||||
if (dataSource.type === "resource") {
|
||||
reachableResourceIds.add(dataSource.resourceId);
|
||||
}
|
||||
}
|
||||
const assetQueryResources = data.build.resources
|
||||
.map(([, resource]) => resource)
|
||||
.flatMap((resource) => {
|
||||
if (reachableResourceIds.has(resource.id) === false) {
|
||||
return [];
|
||||
}
|
||||
const query = getAssetResourceQuery(resource);
|
||||
return query === undefined ? [] : [{ resourceId: resource.id, query }];
|
||||
if (query !== undefined) {
|
||||
return [{ resourceId: resource.id, query }];
|
||||
}
|
||||
if (isStoredAssetQueryResource(resource)) {
|
||||
throw new Error(
|
||||
`Assets resource ${JSON.stringify(resource.id)} has an invalid query configuration`
|
||||
);
|
||||
}
|
||||
return [];
|
||||
});
|
||||
if (assetQueryResources.length > assetResourceLimits.publishedResourceCount) {
|
||||
throw new Error("Published Assets resource count exceeds the limit");
|
||||
}
|
||||
let assetResourceIndexes: PublishedProjectBundle["assetResourceIndexes"];
|
||||
let publishedAssets = data.assets;
|
||||
let publishedAssetFolders = data.assetFolders;
|
||||
if (assetQueryResources.length > 0) {
|
||||
const assetClient = createAssetClientWithResourceIndexStore();
|
||||
await synchronizeCanonicalAssets({
|
||||
client: context.postgrest.client,
|
||||
projectId: project.id,
|
||||
assetClient,
|
||||
});
|
||||
const { entries: canonicalEntries, metadataSnapshot } =
|
||||
await loadCanonicalAssetFileSnapshot({
|
||||
client: context.postgrest.client,
|
||||
projectId: project.id,
|
||||
});
|
||||
const indexedResources = await Promise.all(
|
||||
assetQueryResources.map(async ({ resourceId, query }) => ({
|
||||
resourceId,
|
||||
@@ -190,24 +210,97 @@ const addProjectMetadata = async (
|
||||
queryHash: await computeAssetResourceQueryHash(query),
|
||||
}))
|
||||
);
|
||||
const expectedAssetRevision =
|
||||
await computeCanonicalAssetRevision(canonicalEntries);
|
||||
assetResourceIndexes =
|
||||
await prepareAssetResourceIndexSnapshotsForPublication({
|
||||
const currentBuild = await loadDevBuildByProjectId(context, project.id);
|
||||
const currentQueries = new Map(
|
||||
(currentBuild?.resources ?? []).map((resource) => [
|
||||
resource.id,
|
||||
getAssetResourceQuery(resource),
|
||||
])
|
||||
);
|
||||
const currentResourceIds = indexedResources
|
||||
.filter(
|
||||
({ resourceId, query }) => currentQueries.get(resourceId) === query
|
||||
)
|
||||
.map(({ resourceId }) => resourceId);
|
||||
const retainedFontIds = new Set(
|
||||
data.assets
|
||||
.filter((asset) => asset.type === "font")
|
||||
.map((asset) => asset.id)
|
||||
);
|
||||
for (let attempt = 0; attempt < 2; attempt += 1) {
|
||||
const assetDataBefore = await loadAssetDataByProject(project.id, context);
|
||||
await synchronizeCanonicalAssets({
|
||||
client: context.postgrest.client,
|
||||
store: assetClient.resourceIndexStore,
|
||||
projectId: project.id,
|
||||
resources: indexedResources,
|
||||
entries: canonicalEntries,
|
||||
read: assetClient.resourceIndexStore.read,
|
||||
referenceId: data.build.id,
|
||||
assetRevision: expectedAssetRevision,
|
||||
metadataSnapshot,
|
||||
assetClient,
|
||||
});
|
||||
const { entries: canonicalEntries, metadataSnapshot } =
|
||||
await loadCanonicalAssetFileSnapshot({
|
||||
client: context.postgrest.client,
|
||||
projectId: project.id,
|
||||
});
|
||||
const expectedAssetRevision =
|
||||
await computeCanonicalAssetRevision(canonicalEntries);
|
||||
const preparedIndexes =
|
||||
await prepareAssetResourceIndexSnapshotsForPublication({
|
||||
client: context.postgrest.client,
|
||||
store: assetClient.resourceIndexStore,
|
||||
projectId: project.id,
|
||||
resources: indexedResources,
|
||||
entries: canonicalEntries,
|
||||
read: assetClient.resourceIndexStore.read,
|
||||
referenceId: data.build.id,
|
||||
assetRevision: expectedAssetRevision,
|
||||
metadataSnapshot,
|
||||
currentResourceIds,
|
||||
});
|
||||
const [assetDataAfter, latestCanonical] = await Promise.all([
|
||||
loadAssetDataByProject(project.id, context),
|
||||
loadCanonicalAssetFileSnapshot({
|
||||
client: context.postgrest.client,
|
||||
projectId: project.id,
|
||||
}),
|
||||
]);
|
||||
const latestAssetRevision = await computeCanonicalAssetRevision(
|
||||
latestCanonical.entries
|
||||
);
|
||||
const assetsStayedStable =
|
||||
JSON.stringify(assetDataBefore) === JSON.stringify(assetDataAfter) &&
|
||||
expectedAssetRevision === latestAssetRevision;
|
||||
if (assetsStayedStable === false) {
|
||||
if (attempt === 0) {
|
||||
continue;
|
||||
}
|
||||
throw new Error("Assets changed while preparing publication; retry");
|
||||
}
|
||||
assetResourceIndexes = preparedIndexes;
|
||||
publishedAssets = assetDataAfter.assets.filter(
|
||||
(asset) => asset.type !== "font" || retainedFontIds.has(asset.id)
|
||||
);
|
||||
publishedAssetFolders = assetDataAfter.assetFolders;
|
||||
break;
|
||||
}
|
||||
if (assetResourceIndexes === undefined) {
|
||||
throw new Error("Asset resource indexes were not prepared");
|
||||
}
|
||||
const totalIndexBytes = assetResourceIndexes.reduce(
|
||||
(total, snapshot) =>
|
||||
total +
|
||||
new TextEncoder().encode(serializeAssetResourceIndex(snapshot.index))
|
||||
.byteLength,
|
||||
0
|
||||
);
|
||||
if (totalIndexBytes > assetResourceLimits.publishedIndexBytes) {
|
||||
throw new Error(
|
||||
"Published Assets resource indexes exceed the byte limit"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...data,
|
||||
assets: publishedAssets,
|
||||
assetFolders: publishedAssetFolders,
|
||||
bundleVersion,
|
||||
user: user ? { email: user.email } : undefined,
|
||||
projectDomain: project.domain,
|
||||
|
||||
@@ -286,6 +286,10 @@ claims are 15-minute leases. A later
|
||||
cleanup worker rotates and resumes an expired claim, so a crash before or after
|
||||
object deletion cannot permanently strand the revision row.
|
||||
|
||||
Each inline cleanup invocation processes at most one bounded batch. Production
|
||||
must also invoke the same collector from scheduled background work so cleanup
|
||||
does not depend on later Builder or publication traffic.
|
||||
|
||||
## Schemaless frontmatter and publication
|
||||
|
||||
Frontmatter fields have no built-in publication meaning. Names such as `draft`,
|
||||
@@ -313,6 +317,9 @@ single source for Builder, indexer, generated runtime, and test limits. V1 uses:
|
||||
| Serialized result | 1 MiB |
|
||||
| Candidate documents | 1,000 |
|
||||
| Serialized index | 16 MiB |
|
||||
| Published query resources | 32 |
|
||||
| Published indexes combined | 32 MiB |
|
||||
| Parsed indexes per isolate | 4, least-recently used |
|
||||
| Frontmatter per file | 64 KiB |
|
||||
| Frontmatter structure | Depth 8, 256 fields |
|
||||
| One frontmatter string | 16 KiB UTF-8 |
|
||||
@@ -648,6 +655,16 @@ server-only adapter reads immutable indexes and ranged Markdown bytes from the
|
||||
generated `public` directory, so prerendering requires neither a live HTTP
|
||||
endpoint nor bundling content into JavaScript. Range hydration uses positioned
|
||||
file reads rather than loading the complete source asset before slicing it.
|
||||
Dynamic SSG pages are enumerated when every required `:parameter` is bound
|
||||
directly from `system.params.<name>` and its GROQ parameter is compared directly
|
||||
with one canonical document field, for example
|
||||
`properties.slug == $slug`. Prebuild fails explicitly when a dynamic route has
|
||||
no safely enumerable Assets resource instead of silently omitting its pages.
|
||||
|
||||
Redeploying a historical build preserves that build's stored query and
|
||||
application structure but evaluates it against the project's latest assets.
|
||||
This matches the existing non-versioned asset contract; it is not an exact
|
||||
historical content rollback.
|
||||
|
||||
Public index construction applies only the configured GROQ candidate selection.
|
||||
Prebuild materializes Markdown through the same static asset pipeline as other
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
"benchmark:groq": "node scripts/benchmark-groq.mjs",
|
||||
"benchmark:index": "NODE_OPTIONS=--conditions=webstudio pnpm tsx scripts/benchmark-index.mjs",
|
||||
"benchmark:scale": "NODE_OPTIONS=--conditions=webstudio pnpm tsx scripts/benchmark-scale.mjs",
|
||||
"build": "rm -rf lib && esbuild src/index.ts --outdir=lib --bundle --format=esm --packages=external",
|
||||
"build": "rm -rf lib && esbuild src/index.ts src/published-runtime.ts src/query.ts src/markdown.ts src/indexing.ts --outdir=lib --bundle --format=esm --packages=external",
|
||||
"dts": "tsc --project tsconfig.dts.json",
|
||||
"test": "vitest run",
|
||||
"typecheck": "tsc --noEmit"
|
||||
@@ -35,10 +35,34 @@
|
||||
"types": "./lib/types/index.d.ts",
|
||||
"import": "./lib/index.js",
|
||||
"default": "./src/index.ts"
|
||||
},
|
||||
"./runtime": {
|
||||
"webstudio": "./src/published-runtime.ts",
|
||||
"types": "./lib/types/published-runtime.d.ts",
|
||||
"import": "./lib/published-runtime.js",
|
||||
"default": "./src/published-runtime.ts"
|
||||
},
|
||||
"./query": {
|
||||
"webstudio": "./src/query.ts",
|
||||
"types": "./lib/types/query.d.ts",
|
||||
"import": "./lib/query.js",
|
||||
"default": "./src/query.ts"
|
||||
},
|
||||
"./markdown": {
|
||||
"webstudio": "./src/markdown.ts",
|
||||
"types": "./lib/types/markdown.d.ts",
|
||||
"import": "./lib/markdown.js",
|
||||
"default": "./src/markdown.ts"
|
||||
},
|
||||
"./indexing": {
|
||||
"webstudio": "./src/indexing.ts",
|
||||
"types": "./lib/types/indexing.d.ts",
|
||||
"import": "./lib/indexing.js",
|
||||
"default": "./src/indexing.ts"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"lib/*",
|
||||
"lib/**/*",
|
||||
"!*.test.*"
|
||||
]
|
||||
}
|
||||
@@ -63,17 +63,28 @@ describe("normalizeAssetFileDocument", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test.each(["", ".", "..", "nested/name", "nested\\name"])(
|
||||
"rejects invalid path segment %j",
|
||||
(name) => {
|
||||
expect(() =>
|
||||
normalizeAssetFileDocument({
|
||||
asset: { ...asset, name },
|
||||
properties: {},
|
||||
})
|
||||
).toThrow("one normalized path segment");
|
||||
}
|
||||
);
|
||||
test.each([
|
||||
[".", "%2E"],
|
||||
["..", "%2E%2E"],
|
||||
["nested/name", "nested%2Fname"],
|
||||
["nested\\name", "nested%5Cname"],
|
||||
])("encodes user-defined path segment %j", (name, path) => {
|
||||
expect(
|
||||
normalizeAssetFileDocument({
|
||||
asset: { ...asset, name, folderId: undefined, folderNames: undefined },
|
||||
properties: {},
|
||||
}).path
|
||||
).toBe(path);
|
||||
});
|
||||
|
||||
test("rejects an empty asset name", () => {
|
||||
expect(() =>
|
||||
normalizeAssetFileDocument({
|
||||
asset: { ...asset, name: "" },
|
||||
properties: {},
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
test("rejects JSON-incompatible property values", () => {
|
||||
expect(() =>
|
||||
|
||||
@@ -107,16 +107,18 @@ export const createCanonicalAssetFileEntry = ({
|
||||
};
|
||||
};
|
||||
|
||||
const assertPathSegment = (segment: string, label: string) => {
|
||||
if (
|
||||
segment.length === 0 ||
|
||||
segment === "." ||
|
||||
segment === ".." ||
|
||||
segment.includes("/") ||
|
||||
segment.includes("\\")
|
||||
) {
|
||||
throw new Error(`${label} must be one normalized path segment`);
|
||||
// Asset and folder names are user data, not filesystem paths. Encode each name
|
||||
// independently so slashes and dot segments cannot change path structure while
|
||||
// preserving the original name in the document for display and exact matching.
|
||||
const encodeAssetPathSegment = (segment: string) => {
|
||||
const encoded = encodeURIComponent(segment);
|
||||
if (encoded === ".") {
|
||||
return "%2E";
|
||||
}
|
||||
if (encoded === "..") {
|
||||
return "%2E%2E";
|
||||
}
|
||||
return encoded;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -127,22 +129,19 @@ export const normalizeAssetFileDocument = ({
|
||||
asset,
|
||||
properties,
|
||||
excerpt,
|
||||
metadataError,
|
||||
}: {
|
||||
asset: AssetFileMetadataInput;
|
||||
properties: Record<string, unknown>;
|
||||
excerpt?: string;
|
||||
metadataError?: AssetFileDocument["metadataError"];
|
||||
}): AssetFileDocument => {
|
||||
assertPathSegment(asset.name, "Asset name");
|
||||
const folderNames = asset.folderNames ?? [];
|
||||
if ((asset.folderId === undefined) !== (folderNames.length === 0)) {
|
||||
throw new Error(
|
||||
"Asset folder ID and complete folder-name path must be provided together"
|
||||
);
|
||||
}
|
||||
for (const folderName of folderNames) {
|
||||
assertPathSegment(folderName, "Asset folder name");
|
||||
}
|
||||
|
||||
const extensionSeparator = asset.name.lastIndexOf(".");
|
||||
const extension =
|
||||
asset.extension ??
|
||||
@@ -153,7 +152,9 @@ export const normalizeAssetFileDocument = ({
|
||||
extensionSeparator <= 0
|
||||
? asset.name
|
||||
: asset.name.slice(0, extensionSeparator);
|
||||
const path = [...folderNames, asset.name].join("/");
|
||||
const path = [...folderNames, asset.name]
|
||||
.map(encodeAssetPathSegment)
|
||||
.join("/");
|
||||
|
||||
return assetFileDocument.parse({
|
||||
_id: asset.id,
|
||||
@@ -169,5 +170,6 @@ export const normalizeAssetFileDocument = ({
|
||||
contentRef: asset.contentRef,
|
||||
properties,
|
||||
...(excerpt === undefined ? {} : { excerpt }),
|
||||
...(metadataError === undefined ? {} : { metadataError }),
|
||||
});
|
||||
};
|
||||
@@ -79,6 +79,13 @@ const getStandardFieldContributions = (
|
||||
...(document.excerpt === undefined
|
||||
? []
|
||||
: [{ path: "excerpt", type: "string" } as const]),
|
||||
...(document.metadataError === undefined
|
||||
? []
|
||||
: [
|
||||
{ path: "metadataError", type: "object" } as const,
|
||||
{ path: "metadataError.code", type: "string" } as const,
|
||||
{ path: "metadataError.message", type: "string" } as const,
|
||||
]),
|
||||
];
|
||||
|
||||
type MutableField = {
|
||||
|
||||
@@ -1,19 +1,9 @@
|
||||
import { evaluate, type ExprNode } from "groq-js/1";
|
||||
import {
|
||||
assetResourceLimits,
|
||||
assetResourceQueryRequest,
|
||||
assetResourceQuerySuccess,
|
||||
type AssetFileDocument,
|
||||
type AssetResourceQueryInput,
|
||||
type AssetResourceQueryRequest,
|
||||
type ResourceRequest,
|
||||
} from "@webstudio-is/sdk";
|
||||
import { assetsQueryResourceUrl } from "@webstudio-is/sdk/runtime";
|
||||
import { validateAssetResourceQuery } from "./query-validation";
|
||||
import {
|
||||
hydrateAssetResourceResult,
|
||||
type AssetResourceContentReader,
|
||||
} from "./hydration";
|
||||
|
||||
export * from "./markdown";
|
||||
export * from "./canonical";
|
||||
@@ -23,6 +13,7 @@ export * from "./candidate-selection";
|
||||
export * from "./index-storage";
|
||||
export * from "./query-validation";
|
||||
export * from "./hydration";
|
||||
export * from "./query-execution";
|
||||
export * from "./published-runtime";
|
||||
|
||||
export const createAssetResourceRequest = (
|
||||
@@ -36,154 +27,3 @@ export const createAssetResourceRequest = (
|
||||
headers: [{ name: "content-type", value: "application/json" }],
|
||||
body: assetResourceQueryRequest.parse(request),
|
||||
});
|
||||
|
||||
const evaluateAssetResourceQuery = async ({
|
||||
tree,
|
||||
documents,
|
||||
parameters = {},
|
||||
}: {
|
||||
tree: ExprNode;
|
||||
documents: readonly AssetFileDocument[];
|
||||
parameters?: Record<string, unknown>;
|
||||
}) => {
|
||||
const value = await evaluate(tree, {
|
||||
dataset: documents,
|
||||
params: parameters,
|
||||
});
|
||||
return await value.get();
|
||||
};
|
||||
|
||||
export class AssetResourceQueryExecutionError extends Error {
|
||||
readonly code:
|
||||
| "MISSING_PARAMETER"
|
||||
| "RESULT_LIMIT_EXCEEDED"
|
||||
| "RESULT_SIZE_EXCEEDED";
|
||||
readonly details?: Record<string, number | string>;
|
||||
|
||||
constructor({
|
||||
code,
|
||||
message,
|
||||
details,
|
||||
}: {
|
||||
code: AssetResourceQueryExecutionError["code"];
|
||||
message: string;
|
||||
details?: AssetResourceQueryExecutionError["details"];
|
||||
}) {
|
||||
super(message);
|
||||
this.name = "AssetResourceQueryExecutionError";
|
||||
this.code = code;
|
||||
this.details = details;
|
||||
}
|
||||
}
|
||||
|
||||
const getResultCount = (result: unknown) => {
|
||||
if (Array.isArray(result)) {
|
||||
return result.length;
|
||||
}
|
||||
return result === null || result === undefined ? 0 : 1;
|
||||
};
|
||||
|
||||
export const executeAssetResourceQuery = async ({
|
||||
request,
|
||||
documents,
|
||||
queryHash,
|
||||
indexRevision,
|
||||
assetRevision,
|
||||
}: {
|
||||
request: AssetResourceQueryRequest;
|
||||
documents: readonly AssetFileDocument[];
|
||||
queryHash: string;
|
||||
indexRevision: string;
|
||||
assetRevision: string;
|
||||
}) => {
|
||||
const validated = validateAssetResourceQuery(request.query);
|
||||
for (const parameterName of validated.parameterNames) {
|
||||
if (Object.hasOwn(request.parameters, parameterName) === false) {
|
||||
throw new AssetResourceQueryExecutionError({
|
||||
code: "MISSING_PARAMETER",
|
||||
message: `Asset resource parameter $${parameterName} is required`,
|
||||
details: { parameter: parameterName },
|
||||
});
|
||||
}
|
||||
}
|
||||
if (documents.length > assetResourceLimits.candidateDocuments) {
|
||||
throw new AssetResourceQueryExecutionError({
|
||||
code: "RESULT_LIMIT_EXCEEDED",
|
||||
message: "Asset resource candidate document limit was exceeded",
|
||||
details: {
|
||||
candidateCount: documents.length,
|
||||
candidateLimit: assetResourceLimits.candidateDocuments,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const rawResult = await evaluateAssetResourceQuery({
|
||||
tree: validated.tree,
|
||||
documents,
|
||||
parameters: request.parameters,
|
||||
});
|
||||
const result = rawResult ?? null;
|
||||
const resultCount = getResultCount(result);
|
||||
if (resultCount > request.resultLimit) {
|
||||
throw new AssetResourceQueryExecutionError({
|
||||
code: "RESULT_LIMIT_EXCEEDED",
|
||||
message: "Asset resource query returned too many results",
|
||||
details: { resultCount, resultLimit: request.resultLimit },
|
||||
});
|
||||
}
|
||||
const resultBytes = new TextEncoder().encode(JSON.stringify(result)).length;
|
||||
if (resultBytes > assetResourceLimits.resultBytes) {
|
||||
throw new AssetResourceQueryExecutionError({
|
||||
code: "RESULT_SIZE_EXCEEDED",
|
||||
message: "Asset resource query result exceeds the byte limit",
|
||||
details: {
|
||||
resultBytes,
|
||||
resultByteLimit: assetResourceLimits.resultBytes,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return assetResourceQuerySuccess.parse({
|
||||
ok: true,
|
||||
result,
|
||||
content: {},
|
||||
meta: {
|
||||
queryHash,
|
||||
indexRevision,
|
||||
assetRevision,
|
||||
resultCount,
|
||||
hydratedFileCount: 0,
|
||||
hydratedBytes: 0,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
export const executeAndHydrateAssetResourceQuery = async ({
|
||||
read,
|
||||
...input
|
||||
}: Parameters<typeof executeAssetResourceQuery>[0] & {
|
||||
read?: AssetResourceContentReader;
|
||||
}) => {
|
||||
const response = await executeAssetResourceQuery(input);
|
||||
if (input.request.content.mode === "none") {
|
||||
return response;
|
||||
}
|
||||
if (read === undefined) {
|
||||
throw new Error("Asset content reader is required for hydration");
|
||||
}
|
||||
const hydration = await hydrateAssetResourceResult({
|
||||
result: response.result,
|
||||
documents: input.documents,
|
||||
options: input.request.content,
|
||||
read,
|
||||
});
|
||||
return {
|
||||
...response,
|
||||
content: hydration.content,
|
||||
meta: {
|
||||
...response.meta,
|
||||
hydratedFileCount: hydration.hydratedFileCount,
|
||||
hydratedBytes: hydration.hydratedBytes,
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,5 @@
|
||||
export * from "./canonical";
|
||||
export * from "./field-catalog";
|
||||
export * from "./resource-index";
|
||||
export * from "./candidate-selection";
|
||||
export * from "./index-storage";
|
||||
@@ -132,6 +132,22 @@ describe("published asset resource runtime", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("bounds parsed indexes retained by one isolate", async () => {
|
||||
const { manifest, fetchAsset } = await createRuntime();
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
const runtimeFetch = createPublishedAssetResourceFetch({
|
||||
baseUrl: "https://site.example",
|
||||
deploymentId: `deployment-${index}`,
|
||||
manifest,
|
||||
fetchAsset,
|
||||
});
|
||||
const response = await runtimeFetch(createQueryRequest());
|
||||
expect(response?.status).toBe(200);
|
||||
}
|
||||
|
||||
expect(__testing.getParsedIndexCacheSize()).toBe(4);
|
||||
});
|
||||
|
||||
test("hydrates exactly the selected complete Markdown file", async () => {
|
||||
const { runtimeFetch, fetchAsset } = await createRuntime();
|
||||
const response = await runtimeFetch(createQueryRequest({ mode: "full" }));
|
||||
@@ -142,6 +158,7 @@ describe("published asset resource runtime", () => {
|
||||
});
|
||||
expect(fetchAsset).toHaveBeenCalledWith("/assets/post.md", {
|
||||
headers: expect.any(Headers),
|
||||
signal: expect.any(AbortSignal),
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
assetResourceLimits,
|
||||
assetResourceQueryFailure,
|
||||
assetResourceQueryRequest,
|
||||
type AssetResourceQueryFailure,
|
||||
@@ -9,9 +10,9 @@ import {
|
||||
} from "@webstudio-is/sdk/runtime";
|
||||
import {
|
||||
AssetResourceQueryExecutionError,
|
||||
computeAssetResourceQueryHash,
|
||||
executeAndHydrateAssetResourceQuery,
|
||||
} from "./index";
|
||||
} from "./query-execution";
|
||||
import { computeAssetResourceQueryHash } from "./resource-index";
|
||||
import { AssetResourceHydrationError } from "./hydration";
|
||||
import { verifyAssetResourceIndex } from "./resource-index";
|
||||
import { sha256Hex } from "./sha256";
|
||||
@@ -58,7 +59,11 @@ const loadIndex = ({
|
||||
}) => {
|
||||
const cacheKey = getParsedIndexCacheKey({ deploymentId, entry });
|
||||
let pending = parsedIndexCache.get(cacheKey);
|
||||
if (pending === undefined) {
|
||||
if (pending !== undefined) {
|
||||
// Map insertion order doubles as the least-recently-used order.
|
||||
parsedIndexCache.delete(cacheKey);
|
||||
parsedIndexCache.set(cacheKey, pending);
|
||||
} else {
|
||||
pending = (async () => {
|
||||
const response = await fetchAsset(entry.indexPath);
|
||||
if (response.ok === false) {
|
||||
@@ -77,6 +82,13 @@ const loadIndex = ({
|
||||
})();
|
||||
parsedIndexCache.set(cacheKey, pending);
|
||||
pending.catch(() => parsedIndexCache.delete(cacheKey));
|
||||
while (parsedIndexCache.size > assetResourceLimits.runtimeCachedIndexes) {
|
||||
const oldestKey = parsedIndexCache.keys().next().value;
|
||||
if (oldestKey === undefined) {
|
||||
break;
|
||||
}
|
||||
parsedIndexCache.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
return pending;
|
||||
};
|
||||
@@ -235,7 +247,21 @@ export const createPublishedAssetResourceFetch = ({
|
||||
}
|
||||
|
||||
try {
|
||||
if (request.signal.aborted) {
|
||||
return failure({
|
||||
code: "REQUEST_CANCELLED",
|
||||
message: "Published asset resource query was cancelled",
|
||||
status: 499,
|
||||
});
|
||||
}
|
||||
const index = await loadIndex({ deploymentId, entry, fetchAsset });
|
||||
if (request.signal.aborted) {
|
||||
return failure({
|
||||
code: "REQUEST_CANCELLED",
|
||||
message: "Published asset resource query was cancelled",
|
||||
status: 499,
|
||||
});
|
||||
}
|
||||
const result = await executeAndHydrateAssetResourceQuery({
|
||||
request: parsedRequest,
|
||||
documents: index.documents,
|
||||
@@ -252,7 +278,7 @@ export const createPublishedAssetResourceFetch = ({
|
||||
}
|
||||
const assetResponse = await fetchAsset(
|
||||
`/assets/${encodeURIComponent(contentRef)}`,
|
||||
{ headers }
|
||||
{ headers, signal: request.signal }
|
||||
);
|
||||
if (assetResponse.ok === false || assetResponse.body === null) {
|
||||
throw new Error("Selected published asset content was not found");
|
||||
@@ -266,7 +292,7 @@ export const createPublishedAssetResourceFetch = ({
|
||||
},
|
||||
});
|
||||
const resultResponse = jsonResponse(result);
|
||||
if (cacheKey !== undefined) {
|
||||
if (cacheKey !== undefined && request.signal.aborted === false) {
|
||||
resultResponse.headers.set(
|
||||
"cache-control",
|
||||
request.headers.get("cache-control") as string
|
||||
@@ -277,6 +303,13 @@ export const createPublishedAssetResourceFetch = ({
|
||||
}
|
||||
return resultResponse;
|
||||
} catch (error) {
|
||||
if (request.signal.aborted) {
|
||||
return failure({
|
||||
code: "REQUEST_CANCELLED",
|
||||
message: "Published asset resource query was cancelled",
|
||||
status: 499,
|
||||
});
|
||||
}
|
||||
if (
|
||||
error instanceof AssetResourceQueryExecutionError ||
|
||||
error instanceof AssetResourceHydrationError
|
||||
@@ -353,4 +386,5 @@ export const createGeneratedAssetResourceFetch = async ({
|
||||
|
||||
export const __testing = {
|
||||
clearParsedIndexCache: () => parsedIndexCache.clear(),
|
||||
getParsedIndexCacheSize: () => parsedIndexCache.size,
|
||||
};
|
||||
@@ -0,0 +1,163 @@
|
||||
import { evaluate, type ExprNode } from "groq-js/1";
|
||||
import {
|
||||
assetResourceLimits,
|
||||
assetResourceQuerySuccess,
|
||||
type AssetFileDocument,
|
||||
type AssetResourceQueryRequest,
|
||||
} from "@webstudio-is/sdk";
|
||||
import { validateAssetResourceQuery } from "./query-validation";
|
||||
import {
|
||||
hydrateAssetResourceResult,
|
||||
type AssetResourceContentReader,
|
||||
} from "./hydration";
|
||||
|
||||
const evaluateAssetResourceQuery = async ({
|
||||
tree,
|
||||
documents,
|
||||
parameters = {},
|
||||
}: {
|
||||
tree: ExprNode;
|
||||
documents: readonly AssetFileDocument[];
|
||||
parameters?: Record<string, unknown>;
|
||||
}) => {
|
||||
const value = await evaluate(tree, {
|
||||
dataset: documents,
|
||||
params: parameters,
|
||||
});
|
||||
return await value.get();
|
||||
};
|
||||
|
||||
export class AssetResourceQueryExecutionError extends Error {
|
||||
readonly code:
|
||||
| "MISSING_PARAMETER"
|
||||
| "RESULT_LIMIT_EXCEEDED"
|
||||
| "RESULT_SIZE_EXCEEDED";
|
||||
readonly details?: Record<string, number | string>;
|
||||
|
||||
constructor({
|
||||
code,
|
||||
message,
|
||||
details,
|
||||
}: {
|
||||
code: AssetResourceQueryExecutionError["code"];
|
||||
message: string;
|
||||
details?: AssetResourceQueryExecutionError["details"];
|
||||
}) {
|
||||
super(message);
|
||||
this.name = "AssetResourceQueryExecutionError";
|
||||
this.code = code;
|
||||
this.details = details;
|
||||
}
|
||||
}
|
||||
|
||||
const getResultCount = (result: unknown) => {
|
||||
if (Array.isArray(result)) {
|
||||
return result.length;
|
||||
}
|
||||
return result === null || result === undefined ? 0 : 1;
|
||||
};
|
||||
|
||||
export const executeAssetResourceQuery = async ({
|
||||
request,
|
||||
documents,
|
||||
queryHash,
|
||||
indexRevision,
|
||||
assetRevision,
|
||||
}: {
|
||||
request: AssetResourceQueryRequest;
|
||||
documents: readonly AssetFileDocument[];
|
||||
queryHash: string;
|
||||
indexRevision: string;
|
||||
assetRevision: string;
|
||||
}) => {
|
||||
const validated = validateAssetResourceQuery(request.query);
|
||||
for (const parameterName of validated.parameterNames) {
|
||||
if (Object.hasOwn(request.parameters, parameterName) === false) {
|
||||
throw new AssetResourceQueryExecutionError({
|
||||
code: "MISSING_PARAMETER",
|
||||
message: `Asset resource parameter $${parameterName} is required`,
|
||||
details: { parameter: parameterName },
|
||||
});
|
||||
}
|
||||
}
|
||||
if (documents.length > assetResourceLimits.candidateDocuments) {
|
||||
throw new AssetResourceQueryExecutionError({
|
||||
code: "RESULT_LIMIT_EXCEEDED",
|
||||
message: "Asset resource candidate document limit was exceeded",
|
||||
details: {
|
||||
candidateCount: documents.length,
|
||||
candidateLimit: assetResourceLimits.candidateDocuments,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const rawResult = await evaluateAssetResourceQuery({
|
||||
tree: validated.tree,
|
||||
documents,
|
||||
parameters: request.parameters,
|
||||
});
|
||||
const result = rawResult ?? null;
|
||||
const resultCount = getResultCount(result);
|
||||
if (resultCount > request.resultLimit) {
|
||||
throw new AssetResourceQueryExecutionError({
|
||||
code: "RESULT_LIMIT_EXCEEDED",
|
||||
message: "Asset resource query returned too many results",
|
||||
details: { resultCount, resultLimit: request.resultLimit },
|
||||
});
|
||||
}
|
||||
const resultBytes = new TextEncoder().encode(JSON.stringify(result)).length;
|
||||
if (resultBytes > assetResourceLimits.resultBytes) {
|
||||
throw new AssetResourceQueryExecutionError({
|
||||
code: "RESULT_SIZE_EXCEEDED",
|
||||
message: "Asset resource query result exceeds the byte limit",
|
||||
details: {
|
||||
resultBytes,
|
||||
resultByteLimit: assetResourceLimits.resultBytes,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return assetResourceQuerySuccess.parse({
|
||||
ok: true,
|
||||
result,
|
||||
content: {},
|
||||
meta: {
|
||||
queryHash,
|
||||
indexRevision,
|
||||
assetRevision,
|
||||
resultCount,
|
||||
hydratedFileCount: 0,
|
||||
hydratedBytes: 0,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
export const executeAndHydrateAssetResourceQuery = async ({
|
||||
read,
|
||||
...input
|
||||
}: Parameters<typeof executeAssetResourceQuery>[0] & {
|
||||
read?: AssetResourceContentReader;
|
||||
}) => {
|
||||
const response = await executeAssetResourceQuery(input);
|
||||
if (input.request.content.mode === "none") {
|
||||
return response;
|
||||
}
|
||||
if (read === undefined) {
|
||||
throw new Error("Asset content reader is required for hydration");
|
||||
}
|
||||
const hydration = await hydrateAssetResourceResult({
|
||||
result: response.result,
|
||||
documents: input.documents,
|
||||
options: input.request.content,
|
||||
read,
|
||||
});
|
||||
return {
|
||||
...response,
|
||||
content: hydration.content,
|
||||
meta: {
|
||||
...response.meta,
|
||||
hydratedFileCount: hydration.hydratedFileCount,
|
||||
hydratedBytes: hydration.hydratedBytes,
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { assetResourceLimits } from "@webstudio-is/sdk";
|
||||
import {
|
||||
getAssetResourceParameterFieldPaths,
|
||||
AssetResourceQueryValidationError,
|
||||
getAssetResourceReferencedFieldPaths,
|
||||
getAssetResourceReferencedFieldPathsFromTree,
|
||||
@@ -21,6 +22,18 @@ const expectValidationCode = (
|
||||
};
|
||||
|
||||
describe("asset resource query validation", () => {
|
||||
test("finds unambiguous fields compared with route parameters", () => {
|
||||
expect(
|
||||
getAssetResourceParameterFieldPaths(
|
||||
"*[properties.slug == $slug && $locale == properties.locale]"
|
||||
)
|
||||
).toEqual(
|
||||
new Map([
|
||||
["locale", ["properties", "locale"]],
|
||||
["slug", ["properties", "slug"]],
|
||||
])
|
||||
);
|
||||
});
|
||||
test("returns one parsed analysis for a valid parameterized query", () => {
|
||||
expect(
|
||||
validateAssetResourceQuery(
|
||||
|
||||
@@ -147,6 +147,71 @@ const getAttributePath = (node: unknown): string | undefined => {
|
||||
: appendAssetFieldPath(basePath, attribute.name);
|
||||
};
|
||||
|
||||
const getAttributePathSegments = (node: unknown): string[] | undefined => {
|
||||
if (
|
||||
typeof node !== "object" ||
|
||||
node === null ||
|
||||
Reflect.get(node, "type") !== "AccessAttribute" ||
|
||||
typeof Reflect.get(node, "name") !== "string"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const base = Reflect.get(node, "base");
|
||||
const baseSegments = base === undefined ? [] : getAttributePathSegments(base);
|
||||
if (baseSegments === undefined) {
|
||||
return;
|
||||
}
|
||||
return [...baseSegments, Reflect.get(node, "name") as string];
|
||||
};
|
||||
|
||||
const getParameterName = (node: unknown) =>
|
||||
typeof node === "object" &&
|
||||
node !== null &&
|
||||
Reflect.get(node, "type") === "Parameter" &&
|
||||
typeof Reflect.get(node, "name") === "string"
|
||||
? (Reflect.get(node, "name") as string)
|
||||
: undefined;
|
||||
|
||||
/**
|
||||
* Returns the document field directly compared with each GROQ parameter.
|
||||
* Ambiguous parameters are omitted instead of guessing a prerender identity.
|
||||
*/
|
||||
export const getAssetResourceParameterFieldPaths = (query: string) => {
|
||||
const paths = new Map<string, string[] | undefined>();
|
||||
const add = (parameterName: string, path: string[]) => {
|
||||
const previous = paths.get(parameterName);
|
||||
if (previous === undefined && paths.has(parameterName) === false) {
|
||||
paths.set(parameterName, path);
|
||||
return;
|
||||
}
|
||||
if (JSON.stringify(previous) !== JSON.stringify(path)) {
|
||||
paths.set(parameterName, undefined);
|
||||
}
|
||||
};
|
||||
visitGroqAst(validateAssetResourceQuery(query).tree, (node) => {
|
||||
if (node.type !== "OpCall" || Reflect.get(node, "op") !== "==") {
|
||||
return;
|
||||
}
|
||||
const left = Reflect.get(node, "left");
|
||||
const right = Reflect.get(node, "right");
|
||||
const leftParameter = getParameterName(left);
|
||||
const rightParameter = getParameterName(right);
|
||||
const leftPath = getAttributePathSegments(left);
|
||||
const rightPath = getAttributePathSegments(right);
|
||||
if (leftParameter !== undefined && rightPath !== undefined) {
|
||||
add(leftParameter, rightPath);
|
||||
}
|
||||
if (rightParameter !== undefined && leftPath !== undefined) {
|
||||
add(rightParameter, leftPath);
|
||||
}
|
||||
});
|
||||
return new Map(
|
||||
[...paths]
|
||||
.filter((entry): entry is [string, string[]] => entry[1] !== undefined)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
);
|
||||
};
|
||||
|
||||
export const getAssetResourceReferencedFieldPathsFromTree = (
|
||||
tree: ExprNode
|
||||
) => {
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from "./query-validation";
|
||||
export * from "./query-execution";
|
||||
@@ -342,7 +342,7 @@ describe("canonical asset metadata synchronization", () => {
|
||||
).resolves.toEqual({ status: "deleted" });
|
||||
});
|
||||
|
||||
test("keeps existing canonical metadata when changed Markdown has invalid YAML", async () => {
|
||||
test("persists standard metadata and a parse error for invalid YAML", async () => {
|
||||
const source = "---\ntitle: [broken\n---\nBody";
|
||||
const bytes = encoder.encode(source);
|
||||
const readFile = vi.fn<AssetClient["readFile"]>(async () => ({
|
||||
@@ -369,7 +369,16 @@ describe("canonical asset metadata synchronization", () => {
|
||||
},
|
||||
])
|
||||
),
|
||||
db.get("AssetFolder", () => json([]))
|
||||
db.get("AssetFolder", () => json([])),
|
||||
db.post("rpc/replace_asset_file_metadata", async ({ request }) => {
|
||||
const input = (await request.json()) as ReplaceMetadataRpcArgs;
|
||||
expect(input.p_document).toMatchObject({
|
||||
_id: "asset-1",
|
||||
properties: {},
|
||||
metadataError: { code: "FRONTMATTER_INVALID" },
|
||||
});
|
||||
return json(true);
|
||||
})
|
||||
);
|
||||
|
||||
await expect(
|
||||
@@ -379,7 +388,7 @@ describe("canonical asset metadata synchronization", () => {
|
||||
client: testContext.postgrest.client,
|
||||
assetClient: { uploadFile: vi.fn(), readFile },
|
||||
})
|
||||
).rejects.toMatchObject({ code: "FRONTMATTER_INVALID" });
|
||||
).resolves.toEqual(expect.anything());
|
||||
expect(readFile).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
createCanonicalAssetFileEntry,
|
||||
extractMarkdownBodyAndExcerpt,
|
||||
extractMarkdownFrontmatter,
|
||||
MarkdownMetadataError,
|
||||
normalizeAssetFileDocument,
|
||||
} from "@webstudio-is/asset-resource";
|
||||
import {
|
||||
@@ -113,12 +114,14 @@ const createCanonicalDocument = ({
|
||||
revision,
|
||||
properties,
|
||||
excerpt,
|
||||
metadataError,
|
||||
}: {
|
||||
asset: UploadedAssetRow;
|
||||
hierarchy: AssetFolderHierarchy;
|
||||
revision: string;
|
||||
properties: Record<string, unknown>;
|
||||
excerpt?: string;
|
||||
metadataError?: AssetFileDocument["metadataError"];
|
||||
}) => {
|
||||
const name = formatAssetName({
|
||||
name: asset.file.name,
|
||||
@@ -140,6 +143,7 @@ const createCanonicalDocument = ({
|
||||
},
|
||||
properties,
|
||||
...(excerpt === undefined || excerpt.length === 0 ? {} : { excerpt }),
|
||||
...(metadataError === undefined ? {} : { metadataError }),
|
||||
});
|
||||
};
|
||||
|
||||
@@ -171,6 +175,7 @@ const indexCanonicalAsset = async ({
|
||||
}) => {
|
||||
let properties: Record<string, unknown> = {};
|
||||
let excerpt: string | undefined;
|
||||
let metadataError: AssetFileDocument["metadataError"];
|
||||
if (markdownExtension.test(asset.file.name)) {
|
||||
const prefixLength = Math.min(
|
||||
asset.file.size,
|
||||
@@ -182,12 +187,22 @@ const indexCanonicalAsset = async ({
|
||||
: await assetClient
|
||||
.readFile(asset.file.name, { offset: 0, length: prefixLength })
|
||||
.then((stored) => readPrefix(stored.data, prefixLength));
|
||||
const markdown = await Promise.all([
|
||||
extractMarkdownFrontmatter(bytes),
|
||||
extractMarkdownBodyAndExcerpt(bytes),
|
||||
]);
|
||||
properties = markdown[0].properties;
|
||||
excerpt = markdown[1].excerpt;
|
||||
try {
|
||||
properties = (await extractMarkdownFrontmatter(bytes)).properties;
|
||||
} catch (error) {
|
||||
if (error instanceof MarkdownMetadataError === false) {
|
||||
throw error;
|
||||
}
|
||||
metadataError = { code: error.code, message: error.message };
|
||||
}
|
||||
try {
|
||||
excerpt = (await extractMarkdownBodyAndExcerpt(bytes)).excerpt;
|
||||
} catch (error) {
|
||||
if (error instanceof MarkdownMetadataError === false) {
|
||||
throw error;
|
||||
}
|
||||
metadataError ??= { code: error.code, message: error.message };
|
||||
}
|
||||
}
|
||||
const revision = createAssetContentRevision({
|
||||
storageName: asset.file.name,
|
||||
@@ -200,6 +215,7 @@ const indexCanonicalAsset = async ({
|
||||
revision,
|
||||
properties,
|
||||
excerpt,
|
||||
metadataError,
|
||||
});
|
||||
await replaceCanonicalAssetFileEntry({
|
||||
client,
|
||||
@@ -311,6 +327,7 @@ export const synchronizeCanonicalAssets = async ({
|
||||
revision,
|
||||
properties: current.document.properties,
|
||||
excerpt: current.document.excerpt,
|
||||
metadataError: current.document.metadataError,
|
||||
});
|
||||
if (
|
||||
assetEntries.length === 1 &&
|
||||
|
||||
@@ -209,4 +209,48 @@ describe("previewAssetResourceQuery", () => {
|
||||
length: 8,
|
||||
});
|
||||
});
|
||||
|
||||
test("applies static candidate selection before the candidate limit", async () => {
|
||||
hasProjectPermit.mockResolvedValue(true);
|
||||
loadCanonicalAssetFileEntries.mockResolvedValue(
|
||||
Array.from({ length: 1001 }, (_, index) => {
|
||||
const selected = index === 1000;
|
||||
const assetId = `asset-${index}`;
|
||||
return {
|
||||
projectId,
|
||||
assetId,
|
||||
revision,
|
||||
document: {
|
||||
_id: assetId,
|
||||
_type: "asset.file" as const,
|
||||
name: `${index}.md`,
|
||||
path: selected ? "blog/selected.md" : `other/${index}.md`,
|
||||
key: `${index}`,
|
||||
extension: "md",
|
||||
mimeType: "text/markdown",
|
||||
size: 1,
|
||||
revision,
|
||||
contentRef: `assets/${assetId}`,
|
||||
properties: {},
|
||||
},
|
||||
fieldContributions: [],
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
const result = await previewAssetResourceQuery({
|
||||
projectId,
|
||||
request: {
|
||||
query: '*[path match "blog/**"]{_id}',
|
||||
parameters: {},
|
||||
resultLimit: 1,
|
||||
content: { mode: "none" },
|
||||
},
|
||||
context,
|
||||
assetClient: { readFile: vi.fn() },
|
||||
dependencies,
|
||||
});
|
||||
|
||||
expect(result.result).toEqual([{ _id: "asset-1000" }]);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import {
|
||||
buildAssetResourceIndex,
|
||||
computeAssetResourceQueryHash,
|
||||
computeCanonicalAssetRevision,
|
||||
executeAndHydrateAssetResourceQuery,
|
||||
} from "@webstudio-is/asset-resource";
|
||||
import type { AssetResourceQueryRequest } from "@webstudio-is/sdk";
|
||||
@@ -32,13 +32,18 @@ export const previewAssetResourceQuery = async ({
|
||||
"You don't have access to preview this project's asset resources",
|
||||
dependencies,
|
||||
});
|
||||
const assetRevision = await computeCanonicalAssetRevision(entries);
|
||||
const index = await buildAssetResourceIndex({
|
||||
projectId,
|
||||
resourceId: "preview",
|
||||
query: request.query,
|
||||
entries,
|
||||
});
|
||||
return await executeAndHydrateAssetResourceQuery({
|
||||
request,
|
||||
documents: entries.map(({ document }) => document),
|
||||
documents: index.documents,
|
||||
queryHash: await computeAssetResourceQueryHash(request.query),
|
||||
indexRevision: `metadata:${assetRevision}`,
|
||||
assetRevision,
|
||||
indexRevision: `metadata:${index.integrity.checksum}`,
|
||||
assetRevision: index.assetRevision,
|
||||
read: assetClient.readFile,
|
||||
});
|
||||
};
|
||||
@@ -105,7 +105,7 @@ describe("resource index garbage collection", () => {
|
||||
).resolves.toEqual({ claimed: 1, deleted: 0, missing: 1 });
|
||||
});
|
||||
|
||||
test("drains every available batch", async () => {
|
||||
test("processes at most one batch", async () => {
|
||||
let claims = 0;
|
||||
server.use(
|
||||
db.post("rpc/claim_asset_resource_index_garbage", () => {
|
||||
@@ -133,7 +133,7 @@ describe("resource index garbage collection", () => {
|
||||
limit: 2,
|
||||
});
|
||||
|
||||
expect(claims).toBe(2);
|
||||
expect(claims).toBe(1);
|
||||
expect(remove).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -109,16 +109,7 @@ export const collectAssetResourceIndexGarbageBestEffort = async ({
|
||||
limit?: number;
|
||||
}) => {
|
||||
try {
|
||||
while (true) {
|
||||
const result = await collectAssetResourceIndexGarbage({
|
||||
client,
|
||||
store,
|
||||
limit,
|
||||
});
|
||||
if (result.claimed < limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
await collectAssetResourceIndexGarbage({ client, store, limit });
|
||||
} catch (error) {
|
||||
console.error("Asset resource index cleanup failed", error);
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ describe("asset resource query lifecycle", () => {
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
test("rebuilds all current queries from one canonical load", async () => {
|
||||
test("rebuilds only created and changed queries from one canonical load", async () => {
|
||||
const previous = [
|
||||
resource("changed", "*[false]"),
|
||||
resource("unchanged", "*[]"),
|
||||
@@ -83,11 +83,11 @@ describe("asset resource query lifecycle", () => {
|
||||
})
|
||||
).resolves.toEqual({
|
||||
deletedResourceIds: [],
|
||||
updatedResourceIds: ["changed", "created", "unchanged"],
|
||||
updatedResourceIds: ["changed", "created"],
|
||||
});
|
||||
expect(loadCanonicalAssetFileSnapshot).toHaveBeenCalledOnce();
|
||||
expect(synchronizeCanonicalAssets).toHaveBeenCalledOnce();
|
||||
expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(3);
|
||||
expect(buildPersistAndActivateAssetResourceIndex).toHaveBeenCalledTimes(2);
|
||||
expect(collectAssetResourceIndexGarbageBestEffort).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
|
||||
@@ -62,7 +62,10 @@ export const synchronizeAssetResourceIndexQueries = async ({
|
||||
.map(([resourceId]) => resourceId)
|
||||
.sort();
|
||||
const changed = [...current]
|
||||
.filter(([, query]) => query !== undefined)
|
||||
.filter(
|
||||
([resourceId, query]) =>
|
||||
query !== undefined && previous.get(resourceId) !== query
|
||||
)
|
||||
.map(([resourceId, query]) => ({ resourceId, query: query as string }))
|
||||
.sort((left, right) => left.resourceId.localeCompare(right.resourceId));
|
||||
|
||||
|
||||
@@ -211,6 +211,7 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({
|
||||
metadataSnapshot,
|
||||
read,
|
||||
referenceId,
|
||||
currentResourceIds = [],
|
||||
}: {
|
||||
client: Client;
|
||||
store: ImmutableAssetResourceIndexStore;
|
||||
@@ -225,6 +226,7 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({
|
||||
metadataSnapshot: CanonicalAssetMetadataSnapshot;
|
||||
read: (name: string) => Promise<{ data: AsyncIterable<Uint8Array> }>;
|
||||
referenceId: string;
|
||||
currentResourceIds?: readonly string[];
|
||||
}): Promise<AssetResourceIndexSnapshot[]> => {
|
||||
if (resources.length === 0) {
|
||||
return [];
|
||||
@@ -241,12 +243,14 @@ export const prepareAssetResourceIndexSnapshotsForPublication = async ({
|
||||
const states = new Map(
|
||||
(statesResult.data ?? []).map((state) => [state.resourceId, state])
|
||||
);
|
||||
const explicitlyCurrent = new Set(currentResourceIds);
|
||||
const currentResources = resources.filter((resource) => {
|
||||
const state = states.get(resource.resourceId);
|
||||
return (
|
||||
state !== undefined &&
|
||||
state.deletedAt === null &&
|
||||
state.queryHash === resource.queryHash
|
||||
explicitlyCurrent.has(resource.resourceId) ||
|
||||
(state !== undefined &&
|
||||
state.deletedAt === null &&
|
||||
state.queryHash === resource.queryHash)
|
||||
);
|
||||
});
|
||||
await reconcileAssetResourceIndexesForPublication({
|
||||
|
||||
@@ -12,13 +12,14 @@ import {
|
||||
type Asset,
|
||||
} from "@webstudio-is/sdk";
|
||||
import type { Database } from "@webstudio-is/postgrest/index.server";
|
||||
import type { AssetUploadClient } from "./client";
|
||||
import type { AssetClient, AssetUploadClient } from "./client";
|
||||
import type { AssetDataOverride } from "./utils/get-asset-data";
|
||||
import { createUniqueAssetFilename } from "./utils/get-unique-filename";
|
||||
import { sanitizeS3Key } from "./utils/sanitize-s3-key";
|
||||
import { formatAsset } from "./utils/format-asset";
|
||||
import { assertPostgrestSuccess } from "./patch-utils";
|
||||
import type { UploadTicket } from "./types";
|
||||
import { synchronizeAssetResourceStateAfterAssetChange } from "./resource-index-maintenance";
|
||||
|
||||
type UploadData = {
|
||||
projectId: string;
|
||||
@@ -514,12 +515,33 @@ export const uploadFile = async (
|
||||
if (typeof projectId !== "string") {
|
||||
throw Error("File uploader project is missing");
|
||||
}
|
||||
return await getUploadedAsset({
|
||||
const asset = await getUploadedAsset({
|
||||
name,
|
||||
projectId,
|
||||
context,
|
||||
file,
|
||||
});
|
||||
const readableClient = client as Partial<AssetClient>;
|
||||
if (
|
||||
readableClient.readFile !== undefined &&
|
||||
readableClient.resourceIndexStore !== undefined
|
||||
) {
|
||||
try {
|
||||
await synchronizeAssetResourceStateAfterAssetChange({
|
||||
client: context.postgrest.client,
|
||||
assetClient: readableClient as AssetClient & {
|
||||
resourceIndexStore: NonNullable<AssetClient["resourceIndexStore"]>;
|
||||
},
|
||||
projectId,
|
||||
assetId: asset.id,
|
||||
});
|
||||
} catch (error) {
|
||||
// The upload has committed. Derived metadata and indexes are repaired
|
||||
// by the next preview or publication if this best-effort update fails.
|
||||
console.error("Asset upload resource synchronization failed", error);
|
||||
}
|
||||
}
|
||||
return asset;
|
||||
} catch (error) {
|
||||
await cleanupUploadError(name, context, onUploadError);
|
||||
throw error;
|
||||
|
||||
@@ -247,6 +247,17 @@ export const apiCommandMetadata = publicApiOperations.map((operation) => ({
|
||||
export const highLevelCliCommands = [
|
||||
{ command: "audit", operation: "audit" },
|
||||
{ command: "permissions", operation: "permissions" },
|
||||
{ command: "get-assets-resource", operation: "get-assets-resource" },
|
||||
{ command: "create-assets-resource", operation: "create-assets-resource" },
|
||||
{ command: "update-assets-resource", operation: "update-assets-resource" },
|
||||
{
|
||||
command: "get-asset-resource-index-status",
|
||||
operation: "get-asset-resource-index-status",
|
||||
},
|
||||
{
|
||||
command: "rebuild-asset-resource-index",
|
||||
operation: "rebuild-asset-resource-index",
|
||||
},
|
||||
{
|
||||
command: "get-marketplace-product",
|
||||
operation: "get-marketplace-product",
|
||||
|
||||
@@ -2877,6 +2877,45 @@ const apiCommandHandlers: Partial<Record<ApiCommandName, ApiCommandHandler>> = {
|
||||
dependencies
|
||||
);
|
||||
},
|
||||
"get-assets-resource": async (options, connection, dependencies) =>
|
||||
runProjectSessionCommand(
|
||||
"get-assets-resource",
|
||||
{ resourceId: requireOption(options.resource, "--resource") },
|
||||
connection,
|
||||
dependencies
|
||||
),
|
||||
"create-assets-resource": async (options, connection, dependencies) =>
|
||||
runProjectSessionCommand(
|
||||
"create-assets-resource",
|
||||
await readInputObject(dependencies, options),
|
||||
connection,
|
||||
dependencies
|
||||
),
|
||||
"update-assets-resource": async (options, connection, dependencies) =>
|
||||
runProjectSessionCommand(
|
||||
"update-assets-resource",
|
||||
await readInputObject(dependencies, options),
|
||||
connection,
|
||||
dependencies
|
||||
),
|
||||
"get-asset-resource-index-status": async (
|
||||
options,
|
||||
connection,
|
||||
dependencies
|
||||
) =>
|
||||
runProjectSessionCommand(
|
||||
"get-asset-resource-index-status",
|
||||
{ resourceId: requireOption(options.resource, "--resource") },
|
||||
connection,
|
||||
dependencies
|
||||
),
|
||||
"rebuild-asset-resource-index": async (options, connection, dependencies) =>
|
||||
runProjectSessionCommand(
|
||||
"rebuild-asset-resource-index",
|
||||
{ resourceId: requireOption(options.resource, "--resource") },
|
||||
connection,
|
||||
dependencies
|
||||
),
|
||||
"create-resource": async (options, connection, dependencies) => {
|
||||
const input = {
|
||||
resource: (await getResourceFields(
|
||||
|
||||
@@ -472,6 +472,9 @@ const apiCommandsByArea = {
|
||||
projectQuality: cliCommandMetadata
|
||||
.filter(({ cliCommand }) => cliCommand === "audit")
|
||||
.map(({ cliCommand }) => cliCommand),
|
||||
assetsAndIndexes: cliCommandMetadata
|
||||
.filter(({ cliCommand }) => cliCommand.includes("asset"))
|
||||
.map(({ cliCommand }) => cliCommand),
|
||||
publishAndDomains: cliCommandMetadata
|
||||
.filter(
|
||||
({ cliCommand }) =>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { join } from "node:path";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { isPathnamePattern } from "@webstudio-is/sdk";
|
||||
import { isPathnamePattern, matchPathnameParams } from "@webstudio-is/sdk";
|
||||
import {
|
||||
baseComponentImportSource,
|
||||
createFrameworkComponentRegistry,
|
||||
@@ -16,7 +16,16 @@ const generateVikeRoute = (pagePath: string) => {
|
||||
if (pagePath === "/") {
|
||||
return "index";
|
||||
}
|
||||
return pagePath;
|
||||
let route = pagePath;
|
||||
const matches = [...matchPathnameParams(pagePath)].reverse();
|
||||
for (const match of matches) {
|
||||
const name = match.groups?.name;
|
||||
if (name === undefined || match.index === undefined) {
|
||||
continue;
|
||||
}
|
||||
route = `${route.slice(0, match.index)}@${name}${route.slice(match.index + match[0].length)}`;
|
||||
}
|
||||
return route;
|
||||
};
|
||||
|
||||
export const createFramework = async (
|
||||
@@ -49,25 +58,37 @@ export const createFramework = async (
|
||||
select: `${baseComponentImportSource}:Select`,
|
||||
a: `${baseComponentImportSource}:Link`,
|
||||
},
|
||||
html: ({ pagePath }: { pagePath: string }) => {
|
||||
// ignore dynamic pages in static export
|
||||
if (isPathnamePattern(pagePath)) {
|
||||
return [];
|
||||
html: ({ pagePath, prerenderPaths = [] }) => {
|
||||
const dynamic = isPathnamePattern(pagePath);
|
||||
if (dynamic && prerenderPaths.length === 0) {
|
||||
throw new Error(
|
||||
`Dynamic SSG page ${JSON.stringify(pagePath)} has no enumerable Assets query paths`
|
||||
);
|
||||
}
|
||||
return [
|
||||
const route = generateVikeRoute(pagePath);
|
||||
const entries = [
|
||||
{
|
||||
file: join("pages", generateVikeRoute(pagePath), "+Page.tsx"),
|
||||
file: join("pages", route, "+Page.tsx"),
|
||||
template: htmlPageTemplate,
|
||||
},
|
||||
{
|
||||
file: join("pages", generateVikeRoute(pagePath), "+Head.tsx"),
|
||||
file: join("pages", route, "+Head.tsx"),
|
||||
template: htmlHeadTemplate,
|
||||
},
|
||||
{
|
||||
file: join("pages", generateVikeRoute(pagePath), "+data.ts"),
|
||||
file: join("pages", route, "+data.ts"),
|
||||
template: htmlDataTemplate,
|
||||
},
|
||||
];
|
||||
if (dynamic) {
|
||||
entries.push({
|
||||
file: join("pages", route, "+onBeforePrerenderStart.ts"),
|
||||
template: `export const onBeforePrerenderStart = () => ${JSON.stringify(
|
||||
prerenderPaths
|
||||
)};\n`,
|
||||
});
|
||||
}
|
||||
return entries;
|
||||
},
|
||||
xml: () => [],
|
||||
text: () => [],
|
||||
|
||||
@@ -25,7 +25,10 @@ export type Framework = {
|
||||
components: Record<string, string>;
|
||||
// instance.tag: "importSource:importSpecifier"
|
||||
tags: Record<string, string>;
|
||||
html: (params: { pagePath: string }) => FrameworkTemplateEntry[];
|
||||
html: (params: {
|
||||
pagePath: string;
|
||||
prerenderPaths?: readonly string[];
|
||||
}) => FrameworkTemplateEntry[];
|
||||
xml: (params: { pagePath: string }) => FrameworkTemplateEntry[];
|
||||
text: (params: { pagePath: string }) => FrameworkTemplateEntry[];
|
||||
defaultSitemap: () => FrameworkTemplateEntry[];
|
||||
|
||||
@@ -430,7 +430,9 @@ test("materializes immutable resource indexes as public JSON with a reference-on
|
||||
"app/__generated__/$resources.asset-query-runtime.ts",
|
||||
"utf8"
|
||||
);
|
||||
expect(runtimeModule).toContain('from "@webstudio-is/asset-resource"');
|
||||
expect(runtimeModule).toContain(
|
||||
'from "@webstudio-is/asset-resource/runtime"'
|
||||
);
|
||||
expect(runtimeModule).not.toContain('"documents"');
|
||||
expect(runtimeModule).not.toContain('"properties"');
|
||||
});
|
||||
@@ -1016,7 +1018,21 @@ describe("prebuild", () => {
|
||||
path.includes("$slug")
|
||||
)
|
||||
).toHaveLength(1);
|
||||
});
|
||||
await symlink(join(originalCwd, "node_modules"), "node_modules", "dir");
|
||||
await runGeneratedCommand("react-router", ["build"]);
|
||||
const serverBundle = (
|
||||
await Promise.all(
|
||||
(
|
||||
await getFilePaths("build/server")
|
||||
)
|
||||
.filter((path) => path.endsWith(".js"))
|
||||
.map((path) => readFile(path, "utf8"))
|
||||
)
|
||||
).join("\n");
|
||||
expect(serverBundle).not.toContain("Published post");
|
||||
expect(serverBundle).not.toContain("draft secret");
|
||||
expect(serverBundle).not.toContain("post-revision");
|
||||
}, 30_000);
|
||||
|
||||
test("uses pass-through images in the base react-router template", async () => {
|
||||
await prebuild({ assets: false, template: ["react-router"] });
|
||||
@@ -1087,7 +1103,7 @@ describe("prebuild", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("selects ssg templates and skips dynamic routes", async () => {
|
||||
test("selects ssg templates", async () => {
|
||||
await writeSiteData(
|
||||
createSiteData({
|
||||
pages: [
|
||||
@@ -1099,14 +1115,6 @@ describe("prebuild", () => {
|
||||
rootInstanceId: "root",
|
||||
meta: {},
|
||||
},
|
||||
{
|
||||
id: "post",
|
||||
name: "Post",
|
||||
title: "Post",
|
||||
path: "/blog/:slug",
|
||||
rootInstanceId: "root",
|
||||
meta: {},
|
||||
},
|
||||
],
|
||||
})
|
||||
);
|
||||
@@ -1122,12 +1130,6 @@ describe("prebuild", () => {
|
||||
await expect(readFile("pages/index/+data.ts", "utf8")).resolves.toContain(
|
||||
"../app/__generated__/_index.server"
|
||||
);
|
||||
await expect(
|
||||
readFile("pages/blog/:slug/+Page.tsx", "utf8")
|
||||
).rejects.toThrow("ENOENT");
|
||||
await expect(
|
||||
readFile("app/__generated__/[blog].$slug._index.tsx", "utf8")
|
||||
).resolves.toContain("export { Page }");
|
||||
await expect(readFile("pages/index/+data.ts", "utf8")).resolves.toContain(
|
||||
"createSsgAssetResourceFetch"
|
||||
);
|
||||
@@ -1137,6 +1139,110 @@ describe("prebuild", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("prerenders dynamic SSG paths from parameterized Assets resources", async () => {
|
||||
const query =
|
||||
'*[properties.slug == $slug][0]{_id, "title": properties.title}';
|
||||
const index = await createAssetResourceIndex({
|
||||
format: "webstudio-resource-index",
|
||||
version: 1,
|
||||
resourceId: "post",
|
||||
query,
|
||||
assetRevision: `sha256:${"a".repeat(64)}`,
|
||||
queryMode: "parameterized",
|
||||
parameterNames: ["slug"],
|
||||
documents: [
|
||||
{
|
||||
_id: "post-1",
|
||||
_type: "asset.file",
|
||||
name: "post.md",
|
||||
path: "blog/post.md",
|
||||
key: "post",
|
||||
extension: "md",
|
||||
mimeType: "text/markdown",
|
||||
size: 1,
|
||||
revision: "post-revision",
|
||||
contentRef: "post.md",
|
||||
properties: { slug: "hello-world", title: "Hello" },
|
||||
},
|
||||
],
|
||||
});
|
||||
const siteData = {
|
||||
...createSiteData({
|
||||
pages: [
|
||||
{
|
||||
id: "home",
|
||||
name: "Home",
|
||||
title: "Home",
|
||||
path: "",
|
||||
rootInstanceId: "home-root",
|
||||
meta: {},
|
||||
},
|
||||
{
|
||||
id: "post",
|
||||
name: "Post",
|
||||
title: "Post",
|
||||
path: "/blog/:slug",
|
||||
rootInstanceId: "post-root",
|
||||
meta: {},
|
||||
},
|
||||
],
|
||||
instances: [
|
||||
["home-root", { id: "home-root", component: "Box", children: [] }],
|
||||
["post-root", { id: "post-root", component: "Box", children: [] }],
|
||||
],
|
||||
}),
|
||||
assetResourceIndexes: [
|
||||
{
|
||||
resourceId: index.resourceId,
|
||||
revision: index.integrity.checksum,
|
||||
index,
|
||||
},
|
||||
],
|
||||
};
|
||||
siteData.build.resources = [
|
||||
[
|
||||
"post",
|
||||
{
|
||||
id: "post",
|
||||
name: "Post",
|
||||
control: "system",
|
||||
method: "post",
|
||||
url: '"/$resources/assets/query"',
|
||||
headers: [],
|
||||
body: createAssetQueryResourceBody({
|
||||
query,
|
||||
parameters: [{ name: "slug", value: "system.params.slug" }],
|
||||
resultLimit: 1,
|
||||
}),
|
||||
},
|
||||
],
|
||||
] as never;
|
||||
siteData.build.dataSources = [
|
||||
[
|
||||
"post-data",
|
||||
{
|
||||
id: "post-data",
|
||||
type: "resource",
|
||||
name: "post",
|
||||
resourceId: "post",
|
||||
scopeInstanceId: "post-root",
|
||||
},
|
||||
],
|
||||
] as never;
|
||||
await writeSiteData(siteData);
|
||||
|
||||
await prebuild({ assets: false, template: ["ssg"] });
|
||||
await expect(
|
||||
readFile("pages/blog/@slug/+onBeforePrerenderStart.ts", "utf8")
|
||||
).resolves.toContain("/blog/hello-world");
|
||||
await symlink(join(originalCwd, "node_modules"), "node_modules", "dir");
|
||||
await runGeneratedCommand("vite", ["build"]);
|
||||
await runGeneratedCommand("vike", ["prerender"]);
|
||||
await expect(
|
||||
readFile("dist/client/blog/hello-world/index.html", "utf8")
|
||||
).resolves.toContain("<!DOCTYPE html>");
|
||||
}, 30_000);
|
||||
|
||||
test("prerenders SSG pages with asset query data", async () => {
|
||||
const query = '*[]{"title": properties.title}';
|
||||
const index = await createAssetResourceIndex({
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
createScope,
|
||||
findTreeInstanceIds,
|
||||
getAllPages,
|
||||
getAssetResourceQuery,
|
||||
getPagePath,
|
||||
getPublishablePages,
|
||||
generateResources,
|
||||
@@ -50,8 +51,11 @@ import {
|
||||
elementComponent,
|
||||
toRuntimeAsset,
|
||||
assetResourceContentOptions,
|
||||
assetResourceLimits,
|
||||
matchPathnameParams,
|
||||
parseAssetQueryResourceBody,
|
||||
} from "@webstudio-is/sdk";
|
||||
import { getAssetResourceParameterFieldPaths } from "@webstudio-is/asset-resource";
|
||||
import { migratePages } from "@webstudio-is/project-migrations/pages";
|
||||
import { collectFontFamiliesFromStyleDecls } from "@webstudio-is/project-build/runtime";
|
||||
import {
|
||||
@@ -99,6 +103,141 @@ type SiteDataByPage = {
|
||||
};
|
||||
};
|
||||
|
||||
const getExpressionMemberPath = (node: unknown): string[] | undefined => {
|
||||
if (typeof node !== "object" || node === null) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
Reflect.get(node, "type") === "Identifier" &&
|
||||
typeof Reflect.get(node, "name") === "string"
|
||||
) {
|
||||
return [Reflect.get(node, "name") as string];
|
||||
}
|
||||
if (Reflect.get(node, "type") !== "MemberExpression") {
|
||||
return;
|
||||
}
|
||||
const base = getExpressionMemberPath(Reflect.get(node, "object"));
|
||||
if (base === undefined) {
|
||||
return;
|
||||
}
|
||||
const computed = Reflect.get(node, "computed") === true;
|
||||
const property = Reflect.get(node, "property");
|
||||
const name = computed
|
||||
? Reflect.get(property, "value")
|
||||
: Reflect.get(property, "name");
|
||||
return typeof name === "string" ? [...base, name] : undefined;
|
||||
};
|
||||
|
||||
const getBoundSystemRouteParameter = (expression: string) => {
|
||||
try {
|
||||
const program = parse(`(${expression})`, { ecmaVersion: "latest" });
|
||||
const statement = program.body[0];
|
||||
const node =
|
||||
statement?.type === "ExpressionStatement"
|
||||
? statement.expression
|
||||
: undefined;
|
||||
const path = getExpressionMemberPath(node);
|
||||
return path?.length === 3 && path[0] === "system" && path[1] === "params"
|
||||
? path[2]
|
||||
: undefined;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
const getDocumentPathValue = (document: unknown, path: readonly string[]) => {
|
||||
let value = document;
|
||||
for (const segment of path) {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return;
|
||||
}
|
||||
value = Reflect.get(value, segment);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
export const getAssetResourcePrerenderPaths = ({
|
||||
pagePath,
|
||||
resources,
|
||||
indexes,
|
||||
}: {
|
||||
pagePath: string;
|
||||
resources: readonly [string, Resource][];
|
||||
indexes: PublishedProjectBundle["assetResourceIndexes"];
|
||||
}) => {
|
||||
const pathParameters = [...matchPathnameParams(pagePath)];
|
||||
if (
|
||||
pathParameters.length === 0 ||
|
||||
pathParameters.some(
|
||||
(match) =>
|
||||
match.groups?.name === undefined || (match.groups.modifier ?? "") !== ""
|
||||
)
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
const routeParameterNames = new Set(
|
||||
pathParameters.map((match) => match.groups?.name as string)
|
||||
);
|
||||
const indexByResourceId = new Map(
|
||||
(indexes ?? []).map((snapshot) => [snapshot.resourceId, snapshot.index])
|
||||
);
|
||||
const paths = new Set<string>();
|
||||
for (const [, resource] of resources) {
|
||||
const index = indexByResourceId.get(resource.id);
|
||||
if (index === undefined) {
|
||||
continue;
|
||||
}
|
||||
const query = getAssetResourceQuery(resource);
|
||||
if (query === undefined) {
|
||||
continue;
|
||||
}
|
||||
const parameterFields = getAssetResourceParameterFieldPaths(query);
|
||||
const routeFields = new Map<string, string[]>();
|
||||
for (const binding of parseAssetQueryResourceBody(resource.body)
|
||||
.parameters) {
|
||||
const routeParameter = getBoundSystemRouteParameter(binding.value);
|
||||
const fieldPath = parameterFields.get(binding.name);
|
||||
if (
|
||||
routeParameter !== undefined &&
|
||||
routeParameterNames.has(routeParameter) &&
|
||||
fieldPath !== undefined
|
||||
) {
|
||||
routeFields.set(routeParameter, fieldPath);
|
||||
}
|
||||
}
|
||||
if (routeFields.size !== routeParameterNames.size) {
|
||||
continue;
|
||||
}
|
||||
for (const document of index.documents) {
|
||||
const values = new Map<string, string>();
|
||||
for (const [name, fieldPath] of routeFields) {
|
||||
const value = getDocumentPathValue(document, fieldPath);
|
||||
if (
|
||||
(typeof value === "string" && value.length > 0) ||
|
||||
(typeof value === "number" && Number.isFinite(value)) ||
|
||||
typeof value === "boolean"
|
||||
) {
|
||||
values.set(name, String(value));
|
||||
}
|
||||
}
|
||||
if (values.size !== routeParameterNames.size) {
|
||||
continue;
|
||||
}
|
||||
let path = pagePath;
|
||||
for (const match of [...pathParameters].reverse()) {
|
||||
const name = match.groups?.name as string;
|
||||
const value = values.get(name) as string;
|
||||
path = `${path.slice(0, match.index)}${encodeURIComponent(value)}${path.slice((match.index ?? 0) + match[0].length)}`;
|
||||
}
|
||||
paths.add(path);
|
||||
if (paths.size > assetResourceLimits.candidateDocuments) {
|
||||
throw new Error("Dynamic SSG path count exceeds the Assets limit");
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...paths].sort();
|
||||
};
|
||||
|
||||
const mergeJsonInto = async (sourcePath: string, destinationPath: string) => {
|
||||
const sourceJson = await readFile(sourcePath, "utf8");
|
||||
const destinationJson = await readFile(destinationPath, "utf8").catch(
|
||||
@@ -215,7 +354,7 @@ export const generateAssetQueryRuntimeModule = ({
|
||||
if (manifest.length === 0) {
|
||||
return `export const createGeneratedAssetResourceFetch = async ({ fallback }: ${inputType}): Promise<typeof fetch> => fallback;\n`;
|
||||
}
|
||||
return `import { createGeneratedAssetResourceFetch as createRuntimeFetch } from "@webstudio-is/asset-resource";
|
||||
return `import { createGeneratedAssetResourceFetch as createRuntimeFetch } from "@webstudio-is/asset-resource/runtime";
|
||||
|
||||
const deploymentId = ${JSON.stringify(deploymentId)};
|
||||
const manifest = ${JSON.stringify(manifest, null, 2)};
|
||||
@@ -970,7 +1109,15 @@ export const prebuild = async (options: {
|
||||
await writeGeneratedFile(serverFile, serverExports);
|
||||
|
||||
const getTemplates = framework[documentType];
|
||||
for (const { file, template } of getTemplates({ pagePath })) {
|
||||
const prerenderPaths = getAssetResourcePrerenderPaths({
|
||||
pagePath,
|
||||
resources: pageData.build.resources,
|
||||
indexes: siteData.assetResourceIndexes,
|
||||
});
|
||||
for (const { file, template } of getTemplates({
|
||||
pagePath,
|
||||
prerenderPaths,
|
||||
})) {
|
||||
const content = template
|
||||
.replaceAll("__CONSTANTS__", importFrom("./app/constants.mjs", file))
|
||||
.replaceAll(
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
declare module "__ASSET_QUERY_MANIFEST__" {
|
||||
import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource";
|
||||
import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource/runtime";
|
||||
|
||||
export const assetQueryDeploymentId: string;
|
||||
export const assetQueryManifest: PublishedAssetResourceManifestEntry[];
|
||||
@@ -14,7 +14,7 @@ declare module "__ASSET_QUERY_RUNTIME__" {
|
||||
}
|
||||
|
||||
declare module "__ASSET_RESOURCE_FETCH__" {
|
||||
import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource";
|
||||
import type { PublishedAssetResourceManifestEntry } from "@webstudio-is/asset-resource/runtime";
|
||||
|
||||
export const createSsgAssetResourceFetch: (options: {
|
||||
deploymentId: string;
|
||||
|
||||
@@ -132,7 +132,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
|
||||
const resources = await loadResources(
|
||||
generatedFetch,
|
||||
getResources({ system }).data,
|
||||
url
|
||||
url,
|
||||
{ signal: arg.request.signal }
|
||||
);
|
||||
const pageMeta = getPageMeta({ system, resources });
|
||||
|
||||
|
||||
@@ -91,7 +91,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
|
||||
const resources = await loadResources(
|
||||
generatedFetch,
|
||||
getResources({ system }).data,
|
||||
url
|
||||
url,
|
||||
{ signal: arg.request.signal }
|
||||
);
|
||||
const pageMeta = getPageMeta({ system, resources });
|
||||
|
||||
|
||||
@@ -99,7 +99,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
|
||||
const resources = await loadResources(
|
||||
generatedFetch,
|
||||
getResources({ system }).data,
|
||||
url
|
||||
url,
|
||||
{ signal: arg.request.signal }
|
||||
);
|
||||
const pageMeta = getPageMeta({ system, resources });
|
||||
|
||||
|
||||
@@ -131,7 +131,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
|
||||
const resources = await loadResources(
|
||||
generatedFetch,
|
||||
getResources({ system }).data,
|
||||
url
|
||||
url,
|
||||
{ signal: arg.request.signal }
|
||||
);
|
||||
const pageMeta = getPageMeta({ system, resources });
|
||||
|
||||
|
||||
@@ -91,7 +91,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
|
||||
const resources = await loadResources(
|
||||
generatedFetch,
|
||||
getResources({ system }).data,
|
||||
url
|
||||
url,
|
||||
{ signal: arg.request.signal }
|
||||
);
|
||||
const pageMeta = getPageMeta({ system, resources });
|
||||
|
||||
|
||||
@@ -99,7 +99,8 @@ export const loader = async (arg: LoaderFunctionArgs) => {
|
||||
const resources = await loadResources(
|
||||
generatedFetch,
|
||||
getResources({ system }).data,
|
||||
url
|
||||
url,
|
||||
{ signal: arg.request.signal }
|
||||
);
|
||||
const pageMeta = getPageMeta({ system, resources });
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { open } from "node:fs/promises";
|
||||
import { resolve, sep } from "node:path";
|
||||
import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource";
|
||||
import { createPublishedAssetResourceFetch } from "@webstudio-is/asset-resource/runtime";
|
||||
|
||||
const resolvePublicAsset = (path: string) => {
|
||||
const publicDirectory = resolve("public");
|
||||
|
||||
@@ -1647,6 +1647,7 @@ test("imports project bundle through staged upload", async () => {
|
||||
projectId: "project-id",
|
||||
data: {
|
||||
largeContent: "x".repeat(3 * 1024 * 1024 + 1),
|
||||
assetResourceIndexes: [{ marker: "derived-index-marker" }],
|
||||
} as unknown as PublishedProjectBundle,
|
||||
})
|
||||
).resolves.toEqual({ version: 2 });
|
||||
@@ -1657,6 +1658,9 @@ test("imports project bundle through staged upload", async () => {
|
||||
}
|
||||
|
||||
expect(uploadChunks).toHaveLength(2);
|
||||
expect(Buffer.concat(uploadChunks).toString("utf8")).not.toContain(
|
||||
"derived-index-marker"
|
||||
);
|
||||
expect(JSON.stringify(trpcBody)).toContain('"uploadId":"upload-id"');
|
||||
expect(JSON.stringify(trpcBody)).not.toContain("largeContent");
|
||||
});
|
||||
|
||||
@@ -1811,7 +1811,11 @@ const uploadProjectBundleData = async (
|
||||
) => {
|
||||
const { sourceOrigin } = parseBuilderUrl(params.origin);
|
||||
const endpoint = new URL(stagedUploadPath, sourceOrigin);
|
||||
const data = JSON.stringify(params.data);
|
||||
// Resource indexes are derived deployment artifacts. Import rebuilds them
|
||||
// from the destination project's assets and must not transport stale blobs.
|
||||
const { assetResourceIndexes: _assetResourceIndexes, ...portableData } =
|
||||
params.data;
|
||||
const data = JSON.stringify(portableData);
|
||||
if (new TextEncoder().encode(data).byteLength > maxProjectBundleSize) {
|
||||
throw new Error(
|
||||
`Project bundle is too large to import. Maximum size is ${formatMebibytes(maxProjectBundleSize)}.`
|
||||
|
||||
@@ -160,13 +160,30 @@ VALUES ('resource-index-test.md', 'file', 10, 'UPLOADED');
|
||||
INSERT INTO "Asset" (id, "projectId", name)
|
||||
VALUES ('resource-index-test-asset', 'resource-index-test-project', 'resource-index-test.md');
|
||||
|
||||
SELECT throws_ok(
|
||||
$$
|
||||
INSERT INTO "AssetFileMetadata" (
|
||||
"projectId", "assetId", revision, document, "fieldContributions"
|
||||
) VALUES (
|
||||
'resource-index-test-project',
|
||||
'resource-index-test-asset',
|
||||
'file:resource-index-test.md:invalid',
|
||||
'{}'::JSONB,
|
||||
'[]'::JSONB
|
||||
)
|
||||
$$,
|
||||
'23514',
|
||||
NULL,
|
||||
'Canonical metadata requires explicit matching identity fields'
|
||||
);
|
||||
|
||||
INSERT INTO "AssetFileMetadata" (
|
||||
"projectId", "assetId", revision, document, "fieldContributions"
|
||||
) VALUES (
|
||||
'resource-index-test-project',
|
||||
'resource-index-test-asset',
|
||||
'file:resource-index-test.md:1',
|
||||
'{}'::JSONB,
|
||||
'{"_id":"resource-index-test-asset","revision":"file:resource-index-test.md:1"}'::JSONB,
|
||||
'[]'::JSONB
|
||||
);
|
||||
|
||||
@@ -545,10 +562,89 @@ SELECT is(
|
||||
'Repeated query deletion is idempotent'
|
||||
);
|
||||
|
||||
INSERT INTO "AssetResourceIndexState" (
|
||||
"projectId", "resourceId", query, "queryHash", "assetRevision",
|
||||
"buildAttemptId", "buildStatus"
|
||||
) VALUES (
|
||||
'resource-index-test-project',
|
||||
'project-delete-test',
|
||||
'*[]',
|
||||
'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
|
||||
'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
|
||||
'project-delete-attempt',
|
||||
'BUILDING'
|
||||
);
|
||||
|
||||
INSERT INTO "AssetResourceIndexRevision" (
|
||||
"projectId", "resourceId", revision, "queryHash", "assetRevision",
|
||||
checksum, "objectKey"
|
||||
) VALUES (
|
||||
'resource-index-test-project',
|
||||
'project-delete-test',
|
||||
'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc',
|
||||
'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
|
||||
'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
|
||||
'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc',
|
||||
'projects/test/resources/project-delete-test.json'
|
||||
);
|
||||
|
||||
UPDATE "AssetResourceIndexState"
|
||||
SET "activeRevision" =
|
||||
'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc',
|
||||
"buildStatus" = 'ACTIVE'
|
||||
WHERE "projectId" = 'resource-index-test-project'
|
||||
AND "resourceId" = 'project-delete-test';
|
||||
|
||||
UPDATE "Project"
|
||||
SET "isDeleted" = TRUE
|
||||
WHERE id = 'resource-index-test-project';
|
||||
|
||||
SELECT ok(
|
||||
(
|
||||
SELECT "activeRevision" IS NULL AND "deletedAt" IS NOT NULL
|
||||
FROM "AssetResourceIndexState"
|
||||
WHERE "projectId" = 'resource-index-test-project'
|
||||
AND "resourceId" = 'project-delete-test'
|
||||
),
|
||||
'Soft project deletion retires active resource indexes'
|
||||
);
|
||||
|
||||
SELECT ok(
|
||||
(
|
||||
SELECT "unreferencedAt" IS NOT NULL
|
||||
FROM "AssetResourceIndexRevision"
|
||||
WHERE "projectId" = 'resource-index-test-project'
|
||||
AND "resourceId" = 'project-delete-test'
|
||||
),
|
||||
'Soft project deletion makes immutable indexes collectible'
|
||||
);
|
||||
|
||||
DELETE FROM "Build"
|
||||
WHERE "id" = 'resource-index-test-build'
|
||||
AND "projectId" = 'resource-index-test-project';
|
||||
|
||||
SELECT throws_ok(
|
||||
$$ DELETE FROM "Project" WHERE id = 'resource-index-test-project' $$,
|
||||
'23503',
|
||||
NULL,
|
||||
'Hard project deletion cannot orphan immutable resource indexes'
|
||||
);
|
||||
|
||||
CREATE TEMP TABLE project_delete_garbage AS
|
||||
SELECT * FROM claim_asset_resource_index_garbage('2100-01-01', 10)
|
||||
WHERE "resourceId" = 'project-delete-test';
|
||||
|
||||
SELECT is(
|
||||
finalize_asset_resource_index_garbage(
|
||||
(SELECT "projectId" FROM project_delete_garbage),
|
||||
(SELECT "resourceId" FROM project_delete_garbage),
|
||||
(SELECT revision FROM project_delete_garbage),
|
||||
(SELECT "gcClaimId" FROM project_delete_garbage)
|
||||
),
|
||||
TRUE,
|
||||
'Project deletion cleanup finalizes its immutable resource index'
|
||||
);
|
||||
|
||||
DELETE FROM "Project" WHERE "id" = 'resource-index-test-project';
|
||||
|
||||
SELECT is(
|
||||
|
||||
+90
-7
@@ -15,11 +15,13 @@ CREATE TABLE "AssetFileMetadata" (
|
||||
CONSTRAINT "AssetFileMetadata_pkey"
|
||||
PRIMARY KEY ("projectId", "assetId", "revision"),
|
||||
CONSTRAINT "AssetFileMetadata_document_identity_check"
|
||||
CHECK (
|
||||
CHECK ((
|
||||
jsonb_typeof("document") = 'object'
|
||||
AND jsonb_typeof("document"->'_id') = 'string'
|
||||
AND "document"->>'_id' = "assetId"
|
||||
AND jsonb_typeof("document"->'revision') = 'string'
|
||||
AND "document"->>'revision' = "revision"
|
||||
),
|
||||
) IS TRUE),
|
||||
CONSTRAINT "AssetFileMetadata_field_contributions_check"
|
||||
CHECK (jsonb_typeof("fieldContributions") = 'array'),
|
||||
CONSTRAINT "AssetFileMetadata_assetId_projectId_fkey"
|
||||
@@ -27,10 +29,6 @@ CREATE TABLE "AssetFileMetadata" (
|
||||
ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX "AssetFileMetadata_projectId_assetId_idx"
|
||||
ON "AssetFileMetadata"("projectId", "assetId");
|
||||
|
||||
|
||||
-- Canonical metadata lifecycle
|
||||
|
||||
CREATE FUNCTION replace_asset_file_metadata(
|
||||
@@ -244,6 +242,15 @@ CREATE INDEX "AssetResourceIndexRevision_lookup_idx"
|
||||
"assetRevision"
|
||||
);
|
||||
|
||||
CREATE INDEX "AssetResourceIndexRevision_garbage_idx"
|
||||
ON "AssetResourceIndexRevision"(
|
||||
"unreferencedAt",
|
||||
"projectId",
|
||||
"resourceId",
|
||||
"revision"
|
||||
)
|
||||
WHERE "unreferencedAt" IS NOT NULL;
|
||||
|
||||
ALTER TABLE "AssetResourceIndexState"
|
||||
ADD CONSTRAINT "AssetResourceIndexState_activeRevision_fkey"
|
||||
FOREIGN KEY ("projectId", "resourceId", "activeRevision")
|
||||
@@ -254,6 +261,58 @@ ALTER TABLE "AssetResourceIndexState"
|
||||
)
|
||||
ON DELETE NO ACTION ON UPDATE CASCADE;
|
||||
|
||||
-- Derived index objects must enter garbage collection before project rows can
|
||||
-- disappear. Normal project deletion is soft; the guard prevents a later hard
|
||||
-- delete from losing the only durable object keys.
|
||||
CREATE FUNCTION retire_asset_resource_indexes_on_project_delete()
|
||||
RETURNS TRIGGER
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
IF OLD."isDeleted" IS DISTINCT FROM TRUE AND NEW."isDeleted" = TRUE THEN
|
||||
UPDATE public."AssetResourceIndexState"
|
||||
SET "activeRevision" = NULL,
|
||||
"buildStatus" = 'STALE',
|
||||
"buildError" = NULL,
|
||||
"deletedAt" = COALESCE("deletedAt", CURRENT_TIMESTAMP),
|
||||
"updatedAt" = CURRENT_TIMESTAMP
|
||||
WHERE "projectId" = NEW.id;
|
||||
|
||||
UPDATE public."AssetResourceIndexRevision"
|
||||
SET "unreferencedAt" = COALESCE("unreferencedAt", CURRENT_TIMESTAMP)
|
||||
WHERE "projectId" = NEW.id;
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE TRIGGER "Project_retire_asset_resource_indexes"
|
||||
AFTER UPDATE OF "isDeleted" ON "Project"
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION retire_asset_resource_indexes_on_project_delete();
|
||||
|
||||
CREATE FUNCTION protect_asset_resource_objects_before_project_delete()
|
||||
RETURNS TRIGGER
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM public."AssetResourceIndexRevision"
|
||||
WHERE "projectId" = OLD.id
|
||||
) THEN
|
||||
RAISE EXCEPTION
|
||||
'Project asset resource indexes must be collected before hard deletion'
|
||||
USING ERRCODE = '23503';
|
||||
END IF;
|
||||
RETURN OLD;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE TRIGGER "Project_protect_asset_resource_objects"
|
||||
BEFORE DELETE ON "Project"
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION protect_asset_resource_objects_before_project_delete();
|
||||
|
||||
|
||||
-- Resource index build and activation lifecycle
|
||||
|
||||
@@ -975,7 +1034,16 @@ CREATE FUNCTION finalize_asset_resource_index_garbage(
|
||||
RETURNS BOOLEAN
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
removed BOOLEAN;
|
||||
BEGIN
|
||||
PERFORM pg_catalog.pg_advisory_xact_lock(
|
||||
pg_catalog.hashtextextended(
|
||||
pg_catalog.jsonb_build_array(p_project_id, p_resource_id)::TEXT,
|
||||
0
|
||||
)
|
||||
);
|
||||
|
||||
DELETE FROM public."AssetResourceIndexRevision" AS candidate
|
||||
WHERE candidate."projectId" = p_project_id
|
||||
AND candidate."resourceId" = p_resource_id
|
||||
@@ -997,7 +1065,22 @@ BEGIN
|
||||
AND state."resourceId" = candidate."resourceId"
|
||||
AND state."activeRevision" = candidate.revision
|
||||
);
|
||||
RETURN FOUND;
|
||||
removed := FOUND;
|
||||
|
||||
IF removed THEN
|
||||
DELETE FROM public."AssetResourceIndexState" AS state
|
||||
WHERE state."projectId" = p_project_id
|
||||
AND state."resourceId" = p_resource_id
|
||||
AND state."deletedAt" IS NOT NULL
|
||||
AND state."activeRevision" IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM public."AssetResourceIndexRevision" AS revision
|
||||
WHERE revision."projectId" = state."projectId"
|
||||
AND revision."resourceId" = state."resourceId"
|
||||
);
|
||||
END IF;
|
||||
|
||||
RETURN removed;
|
||||
END;
|
||||
$$;
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@
|
||||
"dependencies": {
|
||||
"@emotion/hash": "^0.9.2",
|
||||
"@modelcontextprotocol/sdk": "^1.10.0",
|
||||
"@webstudio-is/asset-resource": "workspace:*",
|
||||
"@webstudio-is/authorization-token": "workspace:*",
|
||||
"@webstudio-is/css-data": "workspace:*",
|
||||
"@webstudio-is/css-engine": "workspace:*",
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import {
|
||||
assetsResourceCreateInput,
|
||||
assetsResourceUpdateInput,
|
||||
} from "./asset-resources";
|
||||
|
||||
const baseQuery = {
|
||||
resultLimit: 20,
|
||||
content: { mode: "none" as const },
|
||||
parameters: [],
|
||||
};
|
||||
|
||||
describe("Assets resource mutation input", () => {
|
||||
test("rejects invalid GROQ on create", () => {
|
||||
const result = assetsResourceCreateInput.safeParse({
|
||||
name: "Posts",
|
||||
scopeInstanceId: "root",
|
||||
query: { ...baseQuery, groq: "*[" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
test("rejects missing parameter bindings on create and update", () => {
|
||||
const query = {
|
||||
...baseQuery,
|
||||
groq: "*[properties.slug == $slug]",
|
||||
};
|
||||
|
||||
expect(
|
||||
assetsResourceCreateInput.safeParse({
|
||||
name: "Posts",
|
||||
scopeInstanceId: "root",
|
||||
query,
|
||||
}).success
|
||||
).toBe(false);
|
||||
expect(
|
||||
assetsResourceUpdateInput.safeParse({
|
||||
resourceId: "posts",
|
||||
values: { query },
|
||||
}).success
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
test("accepts a binding for every GROQ parameter", () => {
|
||||
expect(
|
||||
assetsResourceCreateInput.safeParse({
|
||||
name: "Post",
|
||||
scopeInstanceId: "root",
|
||||
query: {
|
||||
...baseQuery,
|
||||
groq: "*[properties.slug == $slug]",
|
||||
parameters: [{ name: "slug", value: "system.params.slug" }],
|
||||
},
|
||||
}).success
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
assetsQueryResourceUrl,
|
||||
assetsResourceUrl,
|
||||
} from "@webstudio-is/sdk/runtime";
|
||||
import { validateAssetResourceQuery } from "@webstudio-is/asset-resource";
|
||||
import { z } from "zod";
|
||||
import type { BuilderState } from "../state/builder-state";
|
||||
import type { BuilderRuntimeContext } from "./context";
|
||||
@@ -67,16 +68,58 @@ const assetResourceConfigurationFields = {
|
||||
content: assetResourceContentOptions.default({ mode: "none" }),
|
||||
};
|
||||
|
||||
const storedAssetQueryConfigurationInput = z.object(
|
||||
assetResourceConfigurationFields
|
||||
);
|
||||
const refineAssetQueryConfiguration = (
|
||||
configuration: {
|
||||
query: string;
|
||||
parameters: readonly { name: string }[];
|
||||
},
|
||||
context: z.RefinementCtx
|
||||
) => {
|
||||
let parameterNames: readonly string[];
|
||||
try {
|
||||
parameterNames = validateAssetResourceQuery(
|
||||
configuration.query
|
||||
).parameterNames;
|
||||
} catch (error) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["query"],
|
||||
message:
|
||||
error instanceof Error ? error.message : "Asset query is invalid.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const bindings = new Set(
|
||||
configuration.parameters.map((parameter) => parameter.name)
|
||||
);
|
||||
for (const parameterName of parameterNames) {
|
||||
if (bindings.has(parameterName) === false) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
path: ["parameters"],
|
||||
message: `Asset query parameter $${parameterName} requires a binding.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const assetsQueryConfigurationInput = z.object({
|
||||
groq: assetResourceConfigurationFields.query,
|
||||
parameters: assetResourceConfigurationFields.parameters,
|
||||
resultLimit: assetResourceConfigurationFields.resultLimit,
|
||||
content: assetResourceConfigurationFields.content,
|
||||
});
|
||||
const storedAssetQueryConfigurationInput = z
|
||||
.object(assetResourceConfigurationFields)
|
||||
.superRefine(refineAssetQueryConfiguration);
|
||||
|
||||
export const assetsQueryConfigurationInput = z
|
||||
.object({
|
||||
groq: assetResourceConfigurationFields.query,
|
||||
parameters: assetResourceConfigurationFields.parameters,
|
||||
resultLimit: assetResourceConfigurationFields.resultLimit,
|
||||
content: assetResourceConfigurationFields.content,
|
||||
})
|
||||
.superRefine((configuration, context) =>
|
||||
refineAssetQueryConfiguration(
|
||||
{ query: configuration.groq, parameters: configuration.parameters },
|
||||
context
|
||||
)
|
||||
);
|
||||
|
||||
export const assetsResourceListInput = z.object({
|
||||
scopeInstanceId: z.string().optional(),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { generateObjectExpression, parseObjectExpression } from "./expression";
|
||||
import { assetsQueryResourceUrl, assetsResourceUrl } from "./resource-loader";
|
||||
import { assetResourceLimits } from "./schema/asset-resource";
|
||||
import type { Resource } from "./schema/resources";
|
||||
|
||||
export type AssetQueryParameterBinding = {
|
||||
@@ -34,7 +35,7 @@ export const parseAssetQueryResourceBody = (body: string | undefined) => {
|
||||
export const createAssetQueryResourceBody = ({
|
||||
query,
|
||||
parameters,
|
||||
resultLimit = 100,
|
||||
resultLimit = assetResourceLimits.defaultResultCount,
|
||||
contentExpression = '{ "mode": "none" }',
|
||||
}: {
|
||||
query: string;
|
||||
|
||||
@@ -37,6 +37,12 @@ export const assetFileDocument = z.strictObject({
|
||||
contentRef: z.string().min(1),
|
||||
properties: z.record(z.string(), z.json()),
|
||||
excerpt: z.string().optional(),
|
||||
metadataError: z
|
||||
.strictObject({
|
||||
code: z.string().min(1),
|
||||
message: z.string().min(1),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
export type AssetFileDocument = z.infer<typeof assetFileDocument>;
|
||||
@@ -138,6 +144,9 @@ export const assetResourceLimits = {
|
||||
resultBytes: 1024 * 1024,
|
||||
candidateDocuments: 1000,
|
||||
indexBytes: 16 * 1024 * 1024,
|
||||
publishedResourceCount: 32,
|
||||
publishedIndexBytes: 32 * 1024 * 1024,
|
||||
runtimeCachedIndexes: 4,
|
||||
frontmatterBytes: 64 * 1024,
|
||||
frontmatterDepth: 8,
|
||||
frontmatterFields: 256,
|
||||
|
||||
Generated
+3
@@ -2047,6 +2047,9 @@ importers:
|
||||
'@modelcontextprotocol/sdk':
|
||||
specifier: ^1.10.0
|
||||
version: 1.10.0
|
||||
'@webstudio-is/asset-resource':
|
||||
specifier: workspace:*
|
||||
version: link:../asset-resource
|
||||
'@webstudio-is/authorization-token':
|
||||
specifier: workspace:*
|
||||
version: link:../authorization-token
|
||||
|
||||
Reference in new issue
Block a user