diff --git a/apps/designer/.env.example b/apps/designer/.env.example index c9e6ea3653..354ce8b964 100644 --- a/apps/designer/.env.example +++ b/apps/designer/.env.example @@ -6,4 +6,14 @@ DEV_LOGIN=true # relative to the public directory -FILE_UPLOAD_PATH="uploads" \ No newline at end of file +FILE_UPLOAD_PATH="uploads" + +# S3 envs +S3_BUCKET= +S3_REGION= +S3_ENDPOINT= +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= + +# if using r2 this your worker url +WORKER_URL= \ No newline at end of file diff --git a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx index 9f6aa33fd3..7771ac6268 100644 --- a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx +++ b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx @@ -2,8 +2,8 @@ import { ImageIcon } from "~/shared/icons"; import { Button, Flex, Grid, Heading } from "~/shared/design-system"; import { useRef } from "react"; import { Form, useSubmit } from "@remix-run/react"; -import { type Asset } from "@webstudio-is/prisma-client"; import { Image } from "~/shared/design-system/components/image"; +import { Asset } from "@webstudio-is/prisma-client"; export const TabContent = ({ assets }: { assets: Array }) => { const inputRef = useRef(null); diff --git a/packages/asset-uploader/src/db/create.ts b/packages/asset-uploader/src/db/create.ts index 639e9d4fe8..0cb2be322b 100644 --- a/packages/asset-uploader/src/db/create.ts +++ b/packages/asset-uploader/src/db/create.ts @@ -1,16 +1,21 @@ -import { prisma, Project } from "@webstudio-is/prisma-client"; +import { type Location, prisma, Project } from "@webstudio-is/prisma-client"; import sharp from "sharp"; export const create = async ( projectId: Project["id"], - values: { name: string; path: string; size: number; metadata: sharp.Metadata } + values: { + name: string; + size: number; + metadata: sharp.Metadata; + location: Location; + } ) => { const size = values.size || values.metadata.size || 0; - const { metadata, name, path } = values; + const { metadata, name, location } = values; const newAsset = await prisma.asset.create({ data: { + location, name, - path, size, format: metadata.format, ...(metadata.width ? { width: metadata.width } : {}), diff --git a/packages/asset-uploader/src/db/load.ts b/packages/asset-uploader/src/db/load.ts index 13c670dc3f..25cfa43872 100644 --- a/packages/asset-uploader/src/db/load.ts +++ b/packages/asset-uploader/src/db/load.ts @@ -1,6 +1,9 @@ -import { prisma, Project } from "@webstudio-is/prisma-client"; +import { prisma, Project, Asset } from "@webstudio-is/prisma-client"; +import { getAssetPath } from "../helpers/get-asset-path"; -export const loadByProject = async (projectId?: Project["id"]) => { +export const loadByProject = async ( + projectId?: Project["id"] +): Promise => { if (typeof projectId !== "string") { throw new Error("Tree ID required"); } @@ -12,5 +15,8 @@ export const loadByProject = async (projectId?: Project["id"]) => { }, }); - return assets; + return assets.map((asset) => ({ + ...asset, + path: getAssetPath(asset), + })); }; diff --git a/packages/asset-uploader/src/helpers/get-asset-path.test.ts b/packages/asset-uploader/src/helpers/get-asset-path.test.ts new file mode 100644 index 0000000000..4c56a26a43 --- /dev/null +++ b/packages/asset-uploader/src/helpers/get-asset-path.test.ts @@ -0,0 +1,101 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +import { Decimal } from "@prisma/client/runtime"; +import { Location } from "@webstudio-is/prisma-client"; + +const commonAsset = { + id: "sa-546", + width: new Decimal(200), + height: new Decimal(200), + projectId: "id", + size: 2135, + format: "png", + createdAt: new Date(), + alt: "", +}; + +describe("getAssetPath", () => { + const OLD_ENV = process.env; + + beforeEach(() => { + jest.resetModules(); + process.env = { + ...OLD_ENV, + S3_ACCESS_KEY_ID: "testid", + S3_SECRET_ACCESS_KEY: "testkey", + WORKER_URL: "", + }; + }); + + afterAll(() => { + process.env = OLD_ENV; + }); + + describe("local", () => { + test("returns local path with no custom path", () => { + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: Location.FS, + }) + ).toBe("/uploads/test.png"); + }); + + test("return local path with custom path", () => { + process.env.FILE_UPLOAD_PATH = "assets"; + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: Location.FS, + }) + ).toBe("/assets/test.png"); + }); + }); + describe("s3", () => { + test("returns s3 url", () => { + process.env.S3_ENDPOINT = "https://fr1.s3.com"; + process.env.S3_BUCKET = "bucket"; + process.env.S3_REGION = "fr1"; + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: Location.REMOTE, + }) + ).toBe("https://bucket.fr1.s3.com/test.png"); + }); + test("returns s3 url and fixes extra slashes", () => { + process.env.S3_ENDPOINT = "https://fr1.s3.com//"; + process.env.S3_BUCKET = "bucket"; + process.env.S3_REGION = "fr1"; + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: Location.REMOTE, + }) + ).toBe("https://bucket.fr1.s3.com/test.png"); + }); + }); + describe("R2", () => { + test("returns r2 url", () => { + process.env.S3_ENDPOINT = "https://userid.r2.cloudflarestorage.com"; + process.env.S3_BUCKET = "bucket"; + process.env.S3_REGION = "auto"; + process.env.WORKER_URL = "https://worker-test.workers.dev"; + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: Location.REMOTE, + }) + ).toBe("https://worker-test.workers.dev/test.png"); + }); + }); +}); diff --git a/packages/asset-uploader/src/helpers/get-asset-path.ts b/packages/asset-uploader/src/helpers/get-asset-path.ts new file mode 100644 index 0000000000..9a69969b63 --- /dev/null +++ b/packages/asset-uploader/src/helpers/get-asset-path.ts @@ -0,0 +1,27 @@ +import { DbAsset, Location } from "@webstudio-is/prisma-client"; +import { fsEnvVariables, s3EnvVariables } from "../schema"; +import path from "path"; + +const s3Envs = s3EnvVariables.safeParse(process.env); +const fsEnvs = fsEnvVariables.parse(process.env); + +export const getAssetPath = (asset: DbAsset) => { + if (asset.location === Location.FS) { + return path.join("/", fsEnvs.FILE_UPLOAD_PATH, asset.name); + } + + if (asset.location === Location.REMOTE && s3Envs.success) { + const s3Url = new URL(s3Envs.data.S3_ENDPOINT); + + if (s3Envs.data.WORKER_URL) { + const r2Url = new URL(s3Envs.data.WORKER_URL); + r2Url.pathname = asset.name; + return r2Url.toString(); + } + s3Url.hostname = `${s3Envs.data.S3_BUCKET}.${s3Url.hostname}`; + s3Url.pathname = asset.name; + return s3Url.toString(); + } + + return ""; +}; diff --git a/packages/asset-uploader/src/schema.ts b/packages/asset-uploader/src/schema.ts index 5045025e53..0a8e3a2301 100644 --- a/packages/asset-uploader/src/schema.ts +++ b/packages/asset-uploader/src/schema.ts @@ -21,6 +21,7 @@ export const s3EnvVariables = z.object({ S3_SECRET_ACCESS_KEY: z.string(), S3_BUCKET: z.string(), S3_ACL: z.string().optional().default("public-read"), + WORKER_URL: z.string().optional(), }); export const fsEnvVariables = z.object({ diff --git a/packages/asset-uploader/src/targets/disk/upload.ts b/packages/asset-uploader/src/targets/disk/upload.ts index 123c7ad33a..98fb57d26d 100644 --- a/packages/asset-uploader/src/targets/disk/upload.ts +++ b/packages/asset-uploader/src/targets/disk/upload.ts @@ -1,4 +1,4 @@ -import path from "path"; +import { Location } from "@webstudio-is/prisma-client"; import sharp from "sharp"; import { create } from "../../"; import { ImagesUpload } from "../../schema"; @@ -6,11 +6,9 @@ import { ImagesUpload } from "../../schema"; export const uploadToDisk = async ({ formData, projectId, - folderInPublic, }: { formData: FormData; projectId: string; - folderInPublic: string; }) => { const imagesInfo = ImagesUpload.parse(formData.getAll("image")); const allInfo = imagesInfo.map(async (image) => { @@ -21,9 +19,9 @@ export const uploadToDisk = async ({ const data = { name: image.name, - path: path.join("/", folderInPublic, image.name), size: image.size, metadata, + location: Location.FS, }; const newAsset = await create(projectId, data); diff --git a/packages/asset-uploader/src/targets/s3/handler.ts b/packages/asset-uploader/src/targets/s3/handler.ts index 2a6f86b536..ab99c93665 100644 --- a/packages/asset-uploader/src/targets/s3/handler.ts +++ b/packages/asset-uploader/src/targets/s3/handler.ts @@ -13,6 +13,7 @@ import { getArrayBufferFromIterable, getFilenameAndExtension, } from "../../helpers/array-buffer-helpers"; +import { Location } from "@webstudio-is/prisma-client"; type S3UploadHandler = ({ file, @@ -40,11 +41,14 @@ export const s3UploadHandler: S3UploadHandler = async ({ } const [name, extension] = getFilenameAndExtension(filename); - const key = `${name}_${Date.now()}.${extension}`; + const key = encodeURI(`${name}_${Date.now()}.${extension}`); + + // r2 does not have ACL support so in that case we pass nothing there + const ACL = s3Envs.WORKER_URL ? {} : { ACL: s3Envs.S3_ACL }; const params: PutObjectCommandInput = { Bucket: s3Envs.S3_BUCKET, - ACL: s3Envs.S3_ACL, + ...ACL, Key: key, Body: uint8Array, ContentType: contentType, @@ -64,7 +68,7 @@ export const s3UploadHandler: S3UploadHandler = async ({ const upload = new Upload({ client, params }); - const newFile = ImagesUploadedSuccess.parse(await upload.done()); + ImagesUploadedSuccess.parse(await upload.done()); const image = sharp(uint8Array); const metadata = await image.metadata(); @@ -72,6 +76,6 @@ export const s3UploadHandler: S3UploadHandler = async ({ return JSON.stringify({ metadata, name: key, - path: newFile.Location, + location: Location.REMOTE, }); }; diff --git a/packages/asset-uploader/src/upload.ts b/packages/asset-uploader/src/upload.ts index d7795466f7..34fcf6075e 100644 --- a/packages/asset-uploader/src/upload.ts +++ b/packages/asset-uploader/src/upload.ts @@ -49,7 +49,6 @@ export const uploadAssets = async ({ return await uploadToDisk({ projectId, formData, - folderInPublic: fsUploadVars.FILE_UPLOAD_PATH, }); } }; diff --git a/packages/prisma-client/prisma/migrations/20220722131820_remove_path/migration.sql b/packages/prisma-client/prisma/migrations/20220722131820_remove_path/migration.sql new file mode 100644 index 0000000000..56cf9cbda7 --- /dev/null +++ b/packages/prisma-client/prisma/migrations/20220722131820_remove_path/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - You are about to drop the column `path` on the `Asset` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "Asset" DROP COLUMN "path"; diff --git a/packages/prisma-client/prisma/migrations/20220722132445_add_location/migration.sql b/packages/prisma-client/prisma/migrations/20220722132445_add_location/migration.sql new file mode 100644 index 0000000000..ae5ab4e639 --- /dev/null +++ b/packages/prisma-client/prisma/migrations/20220722132445_add_location/migration.sql @@ -0,0 +1,11 @@ +/* + Warnings: + + - Added the required column `location` to the `Asset` table without a default value. This is not possible if the table is not empty. + +*/ +-- CreateEnum +CREATE TYPE "Location" AS ENUM ('FS', 'REMOTE'); + +-- AlterTable +ALTER TABLE "Asset" ADD COLUMN "location" "Location" NOT NULL; diff --git a/packages/prisma-client/prisma/schema.prisma b/packages/prisma-client/prisma/schema.prisma index 9946c6fe12..e6cd47389d 100644 --- a/packages/prisma-client/prisma/schema.prisma +++ b/packages/prisma-client/prisma/schema.prisma @@ -15,6 +15,11 @@ model Team { users User[] } +enum Location { + FS + REMOTE +} + model Asset { id String @id @default(uuid()) project Project @relation(fields: [projectId], references: [id]) @@ -24,8 +29,8 @@ model Asset { width Decimal? height Decimal? name String - path String alt String? + location Location createdAt DateTime @default(now()) } diff --git a/packages/prisma-client/src/types.ts b/packages/prisma-client/src/types.ts index 29fbef7a4c..6fe42524f1 100644 --- a/packages/prisma-client/src/types.ts +++ b/packages/prisma-client/src/types.ts @@ -1,7 +1,6 @@ -export type { - InstanceProps, - Project, - User, - Breakpoints, - Asset, -} from "@prisma/client"; +import type { Asset as DbAsset } from "@prisma/client"; +export { Location } from "@prisma/client"; +export type { InstanceProps, Project, User, Breakpoints } from "@prisma/client"; + +export type Asset = DbAsset & { path: string }; +export type { DbAsset };