cleanup assets uploader

packages/asset-uploader/package.json

@@ -15,6 +15,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.131.0",
     "@aws-sdk/lib-storage": "^3.131.0",
+    "@types/node": "^18.7.16",
     "@webstudio-is/prisma-client": "*",
     "app-root-path": "^3.0.0",
     "sharp": "^0.30.7"

packages/asset-uploader/src/db/delete.ts

@@ -2,7 +2,7 @@ import { prisma } from "@webstudio-is/prisma-client";
 import { Asset } from "../types";
 import { formatAsset } from "../utils/format-asset";
 
-export const deleteAssetInDb = async (assetId: Asset["id"]) => {
+export const deleteFromDb = async (assetId: Asset["id"]) => {
   if (typeof assetId !== "string") {
     throw new Error("Asset ID required");
   }

packages/asset-uploader/src/delete.ts

@@ -1,12 +1,8 @@
-import { DeleteObjectCommand } from "@aws-sdk/client-s3";
 import { prisma } from "@webstudio-is/prisma-client";
-import { unlink } from "fs/promises";
-import path from "path";
-import { deleteAssetInDb } from "./db";
-import { FILE_DIRECTORY } from "./targets/disk/file-path";
-import { S3EnvVariables } from "./schema";
-import { getS3Client } from "./targets/s3/client";
+import { deleteFromDb } from "./db";
 import { Asset } from "./types";
+import { deleteFromFs } from "./targets/fs/delete";
+import { deleteFromS3 } from "./targets/s3/delete";
 
 export const deleteAsset = async ({
   id,
@@ -21,18 +17,10 @@ export const deleteAsset = async ({
   if (!currentAsset) throw new Error("Asset does not exist");
 
   if (currentAsset.location === "REMOTE") {
-    const s3Envs = S3EnvVariables.parse(process.env);
-    await getS3Client().send(
-      new DeleteObjectCommand({
-        Bucket: s3Envs.S3_BUCKET,
-        Key: name,
-      })
-    );
-
-    return await deleteAssetInDb(id);
+    await deleteFromS3(name);
   } else {
-    await unlink(path.join(FILE_DIRECTORY, name));
-
-    return await deleteAssetInDb(id);
+    await deleteFromFs(name);
   }
+
+  return await deleteFromDb(id);
 };

packages/asset-uploader/src/schema.ts

@@ -2,17 +2,12 @@ import { z } from "zod";
 import { NodeOnDiskFile } from "@remix-run/node";
 import { DEFAULT_UPLOAD_PATH } from "./constants";
 
-const SingleImageInUpload = z.instanceof(NodeOnDiskFile);
+export const AssetsFromFs = z.array(z.instanceof(NodeOnDiskFile));
+export const AssetsFromRemote = z.array(z.string());
 
-export const ImagesUpload = z.array(SingleImageInUpload);
-
-export type ImagesUpload = z.infer<typeof ImagesUpload>;
-export type SingleImageInUpload = z.infer<typeof SingleImageInUpload>;
-
-export const ImagesUploadedSuccess = z.object({
+export const AssetsUploadedSuccess = z.object({
   Location: z.string(),
 });
-export type ImagesUploadedSuccess = z.infer<typeof ImagesUploadedSuccess>;
 
 export const S3EnvVariables = z.object({
   S3_ENDPOINT: z.string(),

packages/asset-uploader/src/targets/disk/handler.ts

@@ -1 +0,0 @@
-export { unstable_createFileUploadHandler as diskUploadHandler } from "@remix-run/node";

packages/asset-uploader/src/targets/disk/upload.ts

@@ -1,33 +0,0 @@
-import { Location } from "@webstudio-is/prisma-client";
-import sharp from "sharp";
-import { create } from "../../";
-import { ImagesUpload } from "../../schema";
-
-export const uploadToDisk = async ({
-  formData,
-  projectId,
-}: {
-  formData: FormData;
-  projectId: string;
-}) => {
-  const imagesInfo = ImagesUpload.parse(formData.getAll("image"));
-  const allInfo = imagesInfo.map(async (image) => {
-    const arrayBuffer = await image.arrayBuffer();
-    const sharpImage = sharp(arrayBuffer as Uint8Array);
-
-    const metadata = await sharpImage.metadata();
-
-    const data = {
-      name: image.name,
-      size: image.size,
-      metadata,
-      location: Location.FS,
-    };
-
-    const newAsset = await create(projectId, data);
-
-    return newAsset;
-  });
-
-  return await Promise.all(allInfo);
-};

packages/asset-uploader/src/targets/fs/delete.ts

@@ -0,0 +1,18 @@
+import { unlink } from "fs/promises";
+import path from "path";
+import { FILE_DIRECTORY } from "./file-path";
+
+// @todo we need a better place for this to reuse it
+const isNodeError = (error: unknown): error is NodeJS.ErrnoException =>
+  error instanceof Error && "code" in error;
+
+export const deleteFromFs = async (name: string) => {
+  try {
+    await unlink(path.join(FILE_DIRECTORY, name));
+  } catch (error) {
+    // When file was deleted from local file system, we don't need to throw an error.
+    if (isNodeError(error) && error.code !== "ENOENT") {
+      throw error;
+    }
+  }
+};

packages/asset-uploader/src/targets/fs/upload.ts

@@ -0,0 +1,57 @@
+import {
+  unstable_parseMultipartFormData,
+  unstable_createFileUploadHandler,
+} from "@remix-run/node";
+import { Location } from "@webstudio-is/prisma-client";
+import { AssetsFromFs } from "../../schema";
+import { getAssetData } from "../../utils/get-asset-data";
+import { create } from "../../db";
+import { FILE_DIRECTORY } from "./file-path";
+import { Asset } from "../../types";
+import { getUniqueFilename } from "../../utils/get-unique-filename";
+
+const location = Location.FS;
+
+export const uploadToFs = async ({
+  request,
+  projectId,
+  maxSize,
+}: {
+  request: Request;
+  projectId: string;
+  maxSize: number;
+}): Promise<Array<Asset>> => {
+  const uploadHandler = await unstable_createFileUploadHandler({
+    maxPartSize: maxSize,
+    directory: FILE_DIRECTORY,
+    file: ({ filename }) => getUniqueFilename(filename),
+  });
+
+  const formData = await unstable_parseMultipartFormData(
+    request,
+    uploadHandler
+  );
+  const formDataImages = AssetsFromFs.parse(formData.getAll("image"));
+
+  // @todo fonts
+  const assets = formDataImages.map(async (asset) =>
+    getAssetData({
+      type: "image" as const,
+      name: asset.name,
+      size: asset.size,
+      buffer: (await asset.arrayBuffer()) as Uint8Array,
+    })
+  );
+
+  const assetsData = await Promise.all(assets);
+
+  // @todo this could be one aggregated query for perf.
+  const savedAssetsData = assetsData.map((data) => {
+    return create(projectId, {
+      ...data,
+      location,
+    });
+  });
+
+  return await Promise.all(savedAssetsData);
+};

packages/asset-uploader/src/targets/s3/delete.ts

@@ -0,0 +1,13 @@
+import { DeleteObjectCommand } from "@aws-sdk/client-s3";
+import { S3EnvVariables } from "../../schema";
+import { getS3Client } from "../../targets/s3/client";
+
+export const deleteFromS3 = async (name: string) => {
+  const s3Envs = S3EnvVariables.parse(process.env);
+  await getS3Client().send(
+    new DeleteObjectCommand({
+      Bucket: s3Envs.S3_BUCKET,
+      Key: name,
+    })
+  );
+};

packages/asset-uploader/src/targets/s3/handler.ts

@@ -1,73 +0,0 @@
-import { UploadHandlerPart } from "@remix-run/node";
-import { PutObjectCommandInput } from "@aws-sdk/client-s3";
-import { Upload } from "@aws-sdk/lib-storage";
-import ObjectID from "bson-objectid";
-
-import sharp from "sharp";
-import {
-  AssetEnvVariables,
-  ImagesUploadedSuccess,
-  S3EnvVariables,
-} from "../../schema";
-import {
-  getArrayBufferFromIterable,
-  getFilenameAndExtension,
-} from "../../utils/array-buffer-helpers";
-import { Location } from "@webstudio-is/prisma-client";
-import { getS3Client } from "./client";
-
-type S3UploadHandler = ({
-  file,
-  maxPartSize,
-}: {
-  file: UploadHandlerPart;
-  maxPartSize: number;
-}) => Promise<string>;
-
-export const s3UploadHandler: S3UploadHandler = async ({
-  file: { data, filename: baseFileName, contentType },
-  maxPartSize,
-}) => {
-  const s3Envs = S3EnvVariables.parse(process.env);
-  const { MAX_UPLOAD_SIZE } = AssetEnvVariables.parse(process.env);
-  if (!data) {
-    throw new Error("Your asset seems to be empty");
-  }
-  const filename = baseFileName ?? ObjectID().toString();
-
-  const uint8Array = await getArrayBufferFromIterable(data);
-
-  if (uint8Array.byteLength > maxPartSize) {
-    throw new Error(`Asset cannot be bigger than ${MAX_UPLOAD_SIZE}MB`);
-  }
-
-  const [name, extension] = getFilenameAndExtension(filename);
-  const key = encodeURI(`${name}_${Date.now()}.${extension}`);
-
-  // if there is no ACL passed we do not default since some providers do not support it
-  const ACL = s3Envs.S3_ACL ? { ACL: s3Envs.S3_ACL } : {};
-
-  const params: PutObjectCommandInput = {
-    Bucket: s3Envs.S3_BUCKET,
-    ...ACL,
-    Key: key,
-    Body: uint8Array,
-    ContentType: contentType,
-    Metadata: {
-      filename,
-    },
-  };
-
-  const upload = new Upload({ client: getS3Client(), params });
-
-  ImagesUploadedSuccess.parse(await upload.done());
-  const image = sharp(uint8Array);
-  const metadata = await image.metadata();
-
-  // these upload handlers only allow strings to be returned
-  return JSON.stringify({
-    metadata,
-    name: key,
-    location: Location.REMOTE,
-  });
-};

packages/asset-uploader/src/targets/s3/upload.ts

@@ -1,17 +1,104 @@
-import { create } from "../..";
+import {
+  type UploadHandlerPart,
+  unstable_parseMultipartFormData,
+} from "@remix-run/node";
+import { PutObjectCommandInput } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { Location } from "@webstudio-is/prisma-client";
+import { AssetsUploadedSuccess, S3EnvVariables } from "../../schema";
+import { toBuffer } from "../../utils/to-buffer";
+import { getAssetData } from "../../utils/get-asset-data";
+import { create } from "../../db";
+import { Asset } from "../../types";
+import { getUniqueFilename } from "../../utils/get-unique-filename";
+import { getS3Client } from "./client";
+
+const location = Location.REMOTE;
 
 export const uploadToS3 = async ({
-  formData,
+  request,
   projectId,
+  maxSize,
 }: {
-  formData: FormData;
+  request: Request;
   projectId: string;
-}) => {
-  const imagesInfo = formData.getAll("image") as Array<string>;
-  const allInfo = imagesInfo.map(async (image) => {
-    const uploadedImage = JSON.parse(image);
-    const newAsset = await create(projectId, uploadedImage);
-    return newAsset;
+  maxSize: number;
+}): Promise<Array<Asset>> => {
+  const formData = await unstable_parseMultipartFormData(
+    request,
+    (file: UploadHandlerPart) =>
+      uploadHandler({
+        file,
+        maxSize,
+      })
+  );
+
+  const imagesFormData = formData.getAll("image") as Array<string>;
+  // @todo this could be one aggregated query for perf.
+  const assetsData = imagesFormData.map((dataString) => {
+    // @todo validate with zod
+    const data = JSON.parse(dataString);
+    return create(projectId, {
+      ...data,
+      location,
+    });
   });
-  return await Promise.all(allInfo);
+
+  return await Promise.all(assetsData);
+};
+
+const uploadHandler = async ({
+  file,
+  maxSize,
+}: {
+  file: UploadHandlerPart;
+  maxSize: number;
+}): Promise<string> => {
+  if (!file.data) {
+    throw new Error("Your asset seems to be empty");
+  }
+
+  // @todo this is going to put the entire file in memory
+  // this has to be a stream that goes directly to s3
+  // Size check has to happen as you stream and interrupted when size is too big
+  // Also check if S3 client has an option to check the size limit
+  const buffer = await toBuffer(file.data);
+
+  if (buffer.byteLength > maxSize) {
+    throw new Error(`Asset cannot be bigger than ${maxSize}MB`);
+  }
+
+  const uniqueFilename = getUniqueFilename(file.filename);
+
+  const s3Envs = S3EnvVariables.parse(process.env);
+
+  // if there is no ACL passed we do not default since some providers do not support it
+  const ACL = s3Envs.S3_ACL ? { ACL: s3Envs.S3_ACL } : {};
+
+  const params: PutObjectCommandInput = {
+    ...ACL,
+    Bucket: s3Envs.S3_BUCKET,
+    Key: uniqueFilename,
+    Body: buffer,
+    ContentType: file.contentType,
+    Metadata: {
+      filename: file.filename || "unnamed",
+    },
+  };
+
+  const upload = new Upload({ client: getS3Client(), params });
+
+  AssetsUploadedSuccess.parse(await upload.done());
+
+  // @todo fonts
+  const assetOptions = {
+    type: "image" as const,
+    name: uniqueFilename,
+    size: buffer.byteLength,
+    buffer,
+  };
+
+  const assetData = await getAssetData(assetOptions);
+
+  return JSON.stringify(assetData);
 };

packages/asset-uploader/src/targets/s3/uploader.ts

@@ -1,17 +0,0 @@
-import { create } from "../../";
-
-export const uploadToS3 = async ({
-  formData,
-  projectId,
-}: {
-  formData: FormData;
-  projectId: string;
-}) => {
-  const imagesInfo = formData.getAll("image") as Array<string>;
-  const allInfo = imagesInfo.map(async (image) => {
-    const uploadedImage = JSON.parse(image);
-    const newAsset = await create(projectId, uploadedImage);
-    return newAsset;
-  });
-  return await Promise.all(allInfo);
-};

packages/asset-uploader/src/upload.ts

@@ -1,45 +1,12 @@
-import {
-  unstable_parseMultipartFormData,
-  type UploadHandlerPart,
-} from "@remix-run/node";
-import { s3UploadHandler } from "./targets/s3/handler";
-import { uploadToS3 } from "./targets/s3/upload";
-import { uploadToDisk } from "./targets/disk/upload";
-import { diskUploadHandler } from "./targets/disk/handler";
 import { AssetEnvVariables, S3EnvVariables } from "./schema";
-import { FILE_DIRECTORY } from "./targets/disk/file-path";
+import { uploadToFs } from "./targets/fs/upload";
+import { uploadToS3 } from "./targets/s3/upload";
 
 const isS3Upload = S3EnvVariables.safeParse(process.env).success;
-
 const commonUploadVars = AssetEnvVariables.parse(process.env);
 
 // user inputs the max value in mb and we transform it to bytes
-export const MAX_UPLOAD_SIZE =
-  parseFloat(commonUploadVars.MAX_UPLOAD_SIZE) * 1e6;
-
-const uploadHandler = () => {
-  if (isS3Upload) {
-    return (file: UploadHandlerPart) =>
-      s3UploadHandler({
-        file,
-        maxPartSize: MAX_UPLOAD_SIZE,
-      });
-  }
-
-  return diskUploadHandler({
-    maxPartSize: MAX_UPLOAD_SIZE,
-    directory: FILE_DIRECTORY,
-    file: ({ filename }) => filename,
-  });
-};
-
-const upload = async (options: { projectId: string; formData: FormData }) => {
-  if (isS3Upload) {
-    return await uploadToS3(options);
-  }
-
-  return await uploadToDisk(options);
-};
+const MAX_UPLOAD_SIZE = parseFloat(commonUploadVars.MAX_UPLOAD_SIZE) * 1e6;
 
 export const uploadAssets = async ({
   request,
@@ -49,11 +16,10 @@ export const uploadAssets = async ({
   projectId: string;
 }) => {
   try {
-    const formData = await unstable_parseMultipartFormData(
-      request,
-      uploadHandler()
-    );
-    return await upload({ projectId, formData });
+    if (isS3Upload) {
+      return await uploadToS3({ request, projectId, maxSize: MAX_UPLOAD_SIZE });
+    }
+    return await uploadToFs({ request, projectId, maxSize: MAX_UPLOAD_SIZE });
   } catch (error) {
     if (error instanceof Error && "maxBytes" in error) {
       throw new Error(

packages/asset-uploader/src/utils/array-buffer-helpers.test.ts

@@ -1,24 +0,0 @@
-import {
-  getFilenameAndExtension,
-  mergeUint8Arrays,
-} from "./array-buffer-helpers";
-
-describe("getFilenameAndExtension", () => {
-  test("gets correct file name when only one dot", () => {
-    expect(getFilenameAndExtension("index.js")).toEqual(["index", "js"]);
-  });
-  test("gets correct file name when more than one dot", () => {
-    expect(getFilenameAndExtension("index.test.js")).toEqual([
-      "index.test",
-      "js",
-    ]);
-  });
-});
-
-describe("mergeUint8Arrays", () => {
-  test("merges two array buffers", () => {
-    const a = new Uint8Array([1]);
-    const b = new Uint8Array([2]);
-    expect(mergeUint8Arrays(a, b)).toStrictEqual(new Uint8Array([1, 2]));
-  });
-});

packages/asset-uploader/src/utils/array-buffer-helpers.ts

@@ -1,27 +0,0 @@
-import path from "path";
-
-export const mergeUint8Arrays = (a: Uint8Array, b: Uint8Array) => {
-  const c = new Uint8Array(a.length + b.length);
-  c.set(a);
-  c.set(b, a.length);
-
-  return c;
-};
-
-export const getFilenameAndExtension = (filename: string) => {
-  return [
-    filename.substring(0, filename.lastIndexOf(".")),
-    path.extname(filename).split(".")[1],
-  ];
-};
-
-export const getArrayBufferFromIterable = async (
-  data: AsyncIterable<Uint8Array>
-) => {
-  let uint8FromIterable = new Uint8Array();
-  for await (const chunk of data) {
-    uint8FromIterable = mergeUint8Arrays(uint8FromIterable, chunk);
-  }
-
-  return uint8FromIterable;
-};

packages/asset-uploader/src/utils/get-asset-data.ts

@@ -0,0 +1,39 @@
+import sharp, { type Metadata } from "sharp";
+
+//type FontData = any;
+
+type ImageData = {
+  name: string;
+  size: number;
+  metadata: Metadata;
+};
+
+export type AssetData = ImageData; // | FontData;
+
+type AssetOptions = {
+  type: "image" | "font";
+  name: string;
+  size: number;
+  buffer: Buffer | Uint8Array;
+};
+
+export const getAssetData = async (
+  options: AssetOptions
+): Promise<AssetData> => {
+  let metadata;
+  if (options.type === "image") {
+    const sharpImage = sharp(options.buffer);
+    metadata = await sharpImage.metadata();
+  }
+
+  if (metadata === undefined) {
+    throw new Error("Could not get metadata for asset");
+  }
+
+  // @todo fonts
+  return {
+    name: options.name,
+    size: options.size,
+    metadata,
+  };
+};

packages/asset-uploader/src/utils/get-asset-path.ts

@@ -12,13 +12,12 @@ export const getAssetPath = (asset: DbAsset) => {
   }
 
   if (asset.location === Location.REMOTE && s3Envs.success) {
-    const s3Url = new URL(s3Envs.data.S3_ENDPOINT);
-
     if (s3Envs.data.ASSET_CDN_URL) {
       const cndUrl = new URL(s3Envs.data.ASSET_CDN_URL);
       cndUrl.pathname = asset.name;
       return cndUrl.toString();
     }
+    const s3Url = new URL(s3Envs.data.S3_ENDPOINT);
     s3Url.hostname = `${s3Envs.data.S3_BUCKET}.${s3Url.hostname}`;
     s3Url.pathname = asset.name;
     return s3Url.toString();

packages/asset-uploader/src/utils/get-unique-filename.ts

@@ -0,0 +1,13 @@
+import path from "path";
+import ObjectID from "bson-objectid";
+
+export const getUniqueFilename = (filename?: string): string => {
+  const id = ObjectID().toString();
+  if (filename === undefined) {
+    // @todo we need to actually detect the extesion || ".jpeg"
+    return `${id}.jpeg`;
+  }
+  const extension = path.extname(filename);
+  const name = path.basename(filename, extension);
+  return `${name}_${id}${extension}`;
+};

packages/asset-uploader/src/utils/to-buffer.ts

@@ -0,0 +1,7 @@
+export const toBuffer = async (data: AsyncIterable<Uint8Array>) => {
+  const result = [];
+  for await (const chunk of data) {
+    result.push(chunk);
+  }
+  return Buffer.concat(result);
+};

yarn.lock

@@ -5023,6 +5023,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190"
   integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==
 
+"@types/node@^18.7.16":
+  version "18.7.16"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.7.16.tgz#0eb3cce1e37c79619943d2fd903919fc30850601"
+  integrity sha512-EQHhixfu+mkqHMZl1R2Ovuvn47PUw18azMJOTwSZr9/fhzHNGXAJ0ma0dayRVchprpCj0Kc1K1xKoWaATWF1qg==
+
 "@types/normalize-package-data@^2.4.0":
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301"