Merge branch 'main' into feat-combobox-2

apps/designer/.env.example

@@ -6,4 +6,14 @@ DEV_LOGIN=true
 
 
 # relative to the public directory
-FILE_UPLOAD_PATH="uploads"
+FILE_UPLOAD_PATH="uploads"
+
+# S3 envs
+S3_BUCKET=
+S3_REGION=
+S3_ENDPOINT=
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
+
+# if using r2 this your worker url
+WORKER_URL=

apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx

@@ -2,8 +2,8 @@ import { ImageIcon } from "~/shared/icons";
 import { Button, Flex, Grid, Heading } from "~/shared/design-system";
 import { useRef } from "react";
 import { Form, useSubmit } from "@remix-run/react";
-import { type Asset } from "@webstudio-is/prisma-client";
 import { Image } from "~/shared/design-system/components/image";
+import { Asset } from "@webstudio-is/prisma-client";
 
 export const TabContent = ({ assets }: { assets: Array<Asset> }) => {
   const inputRef = useRef<HTMLInputElement | null>(null);

packages/asset-uploader/src/db/create.ts

@@ -1,16 +1,21 @@
-import { prisma, Project } from "@webstudio-is/prisma-client";
+import { type Location, prisma, Project } from "@webstudio-is/prisma-client";
 import sharp from "sharp";
 
 export const create = async (
   projectId: Project["id"],
-  values: { name: string; path: string; size: number; metadata: sharp.Metadata }
+  values: {
+    name: string;
+    size: number;
+    metadata: sharp.Metadata;
+    location: Location;
+  }
 ) => {
   const size = values.size || values.metadata.size || 0;
-  const { metadata, name, path } = values;
+  const { metadata, name, location } = values;
   const newAsset = await prisma.asset.create({
     data: {
+      location,
       name,
-      path,
       size,
       format: metadata.format,
       ...(metadata.width ? { width: metadata.width } : {}),

packages/asset-uploader/src/db/load.ts

@@ -1,6 +1,9 @@
-import { prisma, Project } from "@webstudio-is/prisma-client";
+import { prisma, Project, Asset } from "@webstudio-is/prisma-client";
+import { getAssetPath } from "../helpers/get-asset-path";
 
-export const loadByProject = async (projectId?: Project["id"]) => {
+export const loadByProject = async (
+  projectId?: Project["id"]
+): Promise<Asset[]> => {
   if (typeof projectId !== "string") {
     throw new Error("Tree ID required");
   }
@@ -12,5 +15,8 @@ export const loadByProject = async (projectId?: Project["id"]) => {
     },
   });
 
-  return assets;
+  return assets.map((asset) => ({
+    ...asset,
+    path: getAssetPath(asset),
+  }));
 };

packages/asset-uploader/src/helpers/get-asset-path.test.ts

@@ -0,0 +1,101 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+import { Decimal } from "@prisma/client/runtime";
+import { Location } from "@webstudio-is/prisma-client";
+
+const commonAsset = {
+  id: "sa-546",
+  width: new Decimal(200),
+  height: new Decimal(200),
+  projectId: "id",
+  size: 2135,
+  format: "png",
+  createdAt: new Date(),
+  alt: "",
+};
+
+describe("getAssetPath", () => {
+  const OLD_ENV = process.env;
+
+  beforeEach(() => {
+    jest.resetModules();
+    process.env = {
+      ...OLD_ENV,
+      S3_ACCESS_KEY_ID: "testid",
+      S3_SECRET_ACCESS_KEY: "testkey",
+      WORKER_URL: "",
+    };
+  });
+
+  afterAll(() => {
+    process.env = OLD_ENV;
+  });
+
+  describe("local", () => {
+    test("returns local path with no custom path", () => {
+      const { getAssetPath } = require("./get-asset-path");
+      expect(
+        getAssetPath({
+          ...commonAsset,
+          name: "test.png",
+          location: Location.FS,
+        })
+      ).toBe("/uploads/test.png");
+    });
+
+    test("return local path with custom path", () => {
+      process.env.FILE_UPLOAD_PATH = "assets";
+      const { getAssetPath } = require("./get-asset-path");
+      expect(
+        getAssetPath({
+          ...commonAsset,
+          name: "test.png",
+          location: Location.FS,
+        })
+      ).toBe("/assets/test.png");
+    });
+  });
+  describe("s3", () => {
+    test("returns s3 url", () => {
+      process.env.S3_ENDPOINT = "https://fr1.s3.com";
+      process.env.S3_BUCKET = "bucket";
+      process.env.S3_REGION = "fr1";
+      const { getAssetPath } = require("./get-asset-path");
+      expect(
+        getAssetPath({
+          ...commonAsset,
+          name: "test.png",
+          location: Location.REMOTE,
+        })
+      ).toBe("https://bucket.fr1.s3.com/test.png");
+    });
+    test("returns s3 url and fixes extra slashes", () => {
+      process.env.S3_ENDPOINT = "https://fr1.s3.com//";
+      process.env.S3_BUCKET = "bucket";
+      process.env.S3_REGION = "fr1";
+      const { getAssetPath } = require("./get-asset-path");
+      expect(
+        getAssetPath({
+          ...commonAsset,
+          name: "test.png",
+          location: Location.REMOTE,
+        })
+      ).toBe("https://bucket.fr1.s3.com/test.png");
+    });
+  });
+  describe("R2", () => {
+    test("returns r2 url", () => {
+      process.env.S3_ENDPOINT = "https://userid.r2.cloudflarestorage.com";
+      process.env.S3_BUCKET = "bucket";
+      process.env.S3_REGION = "auto";
+      process.env.WORKER_URL = "https://worker-test.workers.dev";
+      const { getAssetPath } = require("./get-asset-path");
+      expect(
+        getAssetPath({
+          ...commonAsset,
+          name: "test.png",
+          location: Location.REMOTE,
+        })
+      ).toBe("https://worker-test.workers.dev/test.png");
+    });
+  });
+});

packages/asset-uploader/src/helpers/get-asset-path.ts

@@ -0,0 +1,27 @@
+import { DbAsset, Location } from "@webstudio-is/prisma-client";
+import { fsEnvVariables, s3EnvVariables } from "../schema";
+import path from "path";
+
+const s3Envs = s3EnvVariables.safeParse(process.env);
+const fsEnvs = fsEnvVariables.parse(process.env);
+
+export const getAssetPath = (asset: DbAsset) => {
+  if (asset.location === Location.FS) {
+    return path.join("/", fsEnvs.FILE_UPLOAD_PATH, asset.name);
+  }
+
+  if (asset.location === Location.REMOTE && s3Envs.success) {
+    const s3Url = new URL(s3Envs.data.S3_ENDPOINT);
+
+    if (s3Envs.data.WORKER_URL) {
+      const r2Url = new URL(s3Envs.data.WORKER_URL);
+      r2Url.pathname = asset.name;
+      return r2Url.toString();
+    }
+    s3Url.hostname = `${s3Envs.data.S3_BUCKET}.${s3Url.hostname}`;
+    s3Url.pathname = asset.name;
+    return s3Url.toString();
+  }
+
+  return "";
+};

packages/asset-uploader/src/schema.ts

@@ -21,6 +21,7 @@ export const s3EnvVariables = z.object({
   S3_SECRET_ACCESS_KEY: z.string(),
   S3_BUCKET: z.string(),
   S3_ACL: z.string().optional().default("public-read"),
+  WORKER_URL: z.string().optional(),
 });
 
 export const fsEnvVariables = z.object({

packages/asset-uploader/src/targets/disk/upload.ts

@@ -1,4 +1,4 @@
-import path from "path";
+import { Location } from "@webstudio-is/prisma-client";
 import sharp from "sharp";
 import { create } from "../../";
 import { ImagesUpload } from "../../schema";
@@ -6,11 +6,9 @@ import { ImagesUpload } from "../../schema";
 export const uploadToDisk = async ({
   formData,
   projectId,
-  folderInPublic,
 }: {
   formData: FormData;
   projectId: string;
-  folderInPublic: string;
 }) => {
   const imagesInfo = ImagesUpload.parse(formData.getAll("image"));
   const allInfo = imagesInfo.map(async (image) => {
@@ -21,9 +19,9 @@ export const uploadToDisk = async ({
 
     const data = {
       name: image.name,
-      path: path.join("/", folderInPublic, image.name),
       size: image.size,
       metadata,
+      location: Location.FS,
     };
 
     const newAsset = await create(projectId, data);

packages/asset-uploader/src/targets/s3/handler.ts

@@ -13,6 +13,7 @@ import {
   getArrayBufferFromIterable,
   getFilenameAndExtension,
 } from "../../helpers/array-buffer-helpers";
+import { Location } from "@webstudio-is/prisma-client";
 
 type S3UploadHandler = ({
   file,
@@ -40,11 +41,14 @@ export const s3UploadHandler: S3UploadHandler = async ({
   }
 
   const [name, extension] = getFilenameAndExtension(filename);
-  const key = `${name}_${Date.now()}.${extension}`;
+  const key = encodeURI(`${name}_${Date.now()}.${extension}`);
+
+  // r2 does not have ACL support so in that case we pass nothing there
+  const ACL = s3Envs.WORKER_URL ? {} : { ACL: s3Envs.S3_ACL };
 
   const params: PutObjectCommandInput = {
     Bucket: s3Envs.S3_BUCKET,
-    ACL: s3Envs.S3_ACL,
+    ...ACL,
     Key: key,
     Body: uint8Array,
     ContentType: contentType,
@@ -64,7 +68,7 @@ export const s3UploadHandler: S3UploadHandler = async ({
 
   const upload = new Upload({ client, params });
 
-  const newFile = ImagesUploadedSuccess.parse(await upload.done());
+  ImagesUploadedSuccess.parse(await upload.done());
   const image = sharp(uint8Array);
   const metadata = await image.metadata();
 
@@ -72,6 +76,6 @@ export const s3UploadHandler: S3UploadHandler = async ({
   return JSON.stringify({
     metadata,
     name: key,
-    path: newFile.Location,
+    location: Location.REMOTE,
   });
 };

packages/asset-uploader/src/upload.ts

@@ -49,7 +49,6 @@ export const uploadAssets = async ({
     return await uploadToDisk({
       projectId,
       formData,
-      folderInPublic: fsUploadVars.FILE_UPLOAD_PATH,
     });
   }
 };

packages/prisma-client/prisma/migrations/20220722131820_remove_path/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `path` on the `Asset` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "Asset" DROP COLUMN "path";

packages/prisma-client/prisma/migrations/20220722132445_add_location/migration.sql

@@ -0,0 +1,11 @@
+/*
+  Warnings:
+
+  - Added the required column `location` to the `Asset` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- CreateEnum
+CREATE TYPE "Location" AS ENUM ('FS', 'REMOTE');
+
+-- AlterTable
+ALTER TABLE "Asset" ADD COLUMN     "location" "Location" NOT NULL;

packages/prisma-client/prisma/schema.prisma

@@ -15,6 +15,11 @@ model Team {
   users User[]
 }
 
+enum Location {
+  FS
+  REMOTE
+}
+
 model Asset {
   id        String   @id @default(uuid())
   project   Project  @relation(fields: [projectId], references: [id])
@@ -24,8 +29,8 @@ model Asset {
   width     Decimal?
   height    Decimal?
   name      String
-  path      String
   alt       String?
+  location  Location
   createdAt DateTime @default(now())
 }
 

packages/prisma-client/src/types.ts

@@ -1,7 +1,6 @@
-export type {
-  InstanceProps,
-  Project,
-  User,
-  Breakpoints,
-  Asset,
-} from "@prisma/client";
+import type { Asset as DbAsset } from "@prisma/client";
+export { Location } from "@prisma/client";
+export type { InstanceProps, Project, User, Breakpoints } from "@prisma/client";
+
+export type Asset = DbAsset & { path: string };
+export type { DbAsset };