From 0dfa87d3d577a142b093b53bb4c39fb3affc8b9e Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 16:21:38 +0200 Subject: [PATCH 01/13] add support for r2 --- apps/designer/env-check.js | 2 +- packages/asset-uploader/src/db/create.ts | 11 +++++--- packages/asset-uploader/src/db/load.ts | 6 ++++- .../src/helpers/get-asset-path.ts | 27 +++++++++++++++++++ packages/asset-uploader/src/schema.ts | 1 + .../asset-uploader/src/targets/disk/upload.ts | 5 +--- .../asset-uploader/src/targets/s3/handler.ts | 8 +++--- packages/asset-uploader/src/upload.ts | 1 - .../20220722131820_remove_path/migration.sql | 8 ++++++ .../20220722132445_add_location/migration.sql | 11 ++++++++ packages/prisma-client/prisma/schema.prisma | 7 ++++- packages/prisma-client/src/types.ts | 1 + 12 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 packages/asset-uploader/src/helpers/get-asset-path.ts create mode 100644 packages/prisma-client/prisma/migrations/20220722131820_remove_path/migration.sql create mode 100644 packages/prisma-client/prisma/migrations/20220722132445_add_location/migration.sql diff --git a/apps/designer/env-check.js b/apps/designer/env-check.js index 470108dcc8..85d818579e 100644 --- a/apps/designer/env-check.js +++ b/apps/designer/env-check.js @@ -11,7 +11,7 @@ * if not it will read the environment variables set by your hosting provider like remix does. */ // eslint-disable-next-line @typescript-eslint/no-var-requires -require("dotenv").config({ path: "../../.env" }); +require("dotenv").config(); const REQUIRED_ENVS = ["DATABASE_URL", "AUTH_SECRET"]; const S3_KEYS = [ diff --git a/packages/asset-uploader/src/db/create.ts b/packages/asset-uploader/src/db/create.ts index 639e9d4fe8..1842726ebf 100644 --- a/packages/asset-uploader/src/db/create.ts +++ b/packages/asset-uploader/src/db/create.ts @@ -3,14 +3,19 @@ import sharp from "sharp"; export const create = async ( projectId: Project["id"], - values: { name: string; path: string; size: number; metadata: sharp.Metadata } + values: { + name: string; + size: number; + metadata: sharp.Metadata; + location: "FS" | "REMOTE"; + } ) => { const size = values.size || values.metadata.size || 0; - const { metadata, name, path } = values; + const { metadata, name, location } = values; const newAsset = await prisma.asset.create({ data: { + location, name, - path, size, format: metadata.format, ...(metadata.width ? { width: metadata.width } : {}), diff --git a/packages/asset-uploader/src/db/load.ts b/packages/asset-uploader/src/db/load.ts index 13c670dc3f..b451d502e0 100644 --- a/packages/asset-uploader/src/db/load.ts +++ b/packages/asset-uploader/src/db/load.ts @@ -1,4 +1,5 @@ import { prisma, Project } from "@webstudio-is/prisma-client"; +import { getAssetPath } from "../helpers/get-asset-path"; export const loadByProject = async (projectId?: Project["id"]) => { if (typeof projectId !== "string") { @@ -12,5 +13,8 @@ export const loadByProject = async (projectId?: Project["id"]) => { }, }); - return assets; + return assets.map((asset) => ({ + ...asset, + path: getAssetPath(asset), + })); }; diff --git a/packages/asset-uploader/src/helpers/get-asset-path.ts b/packages/asset-uploader/src/helpers/get-asset-path.ts new file mode 100644 index 0000000000..ce103e2283 --- /dev/null +++ b/packages/asset-uploader/src/helpers/get-asset-path.ts @@ -0,0 +1,27 @@ +import { Asset } from "@webstudio-is/prisma-client"; +import { fsEnvVariables, s3EnvVariables } from "../schema"; +import path from "path"; + +const s3Envs = s3EnvVariables.safeParse(process.env); +const fsEnvs = fsEnvVariables.parse(process.env); + +export const getAssetPath = (asset: Asset) => { + if (asset.location === "FS") { + return path.join(fsEnvs.FILE_UPLOAD_PATH, asset.name); + } + + if (asset.location === "REMOTE" && s3Envs.success) { + const s3Url = new URL(s3Envs.data.S3_ENDPOINT); + + if (s3Envs.data.WORKER_URL) { + const r2Url = new URL(s3Envs.data.WORKER_URL); + r2Url.pathname = asset.name; + return r2Url.toString(); + } + s3Url.hostname = `${s3Envs.data.S3_BUCKET}.${s3Url.hostname}`; + s3Url.pathname = asset.name; + return s3Url.toString(); + } + + return ""; +}; diff --git a/packages/asset-uploader/src/schema.ts b/packages/asset-uploader/src/schema.ts index 5045025e53..0a8e3a2301 100644 --- a/packages/asset-uploader/src/schema.ts +++ b/packages/asset-uploader/src/schema.ts @@ -21,6 +21,7 @@ export const s3EnvVariables = z.object({ S3_SECRET_ACCESS_KEY: z.string(), S3_BUCKET: z.string(), S3_ACL: z.string().optional().default("public-read"), + WORKER_URL: z.string().optional(), }); export const fsEnvVariables = z.object({ diff --git a/packages/asset-uploader/src/targets/disk/upload.ts b/packages/asset-uploader/src/targets/disk/upload.ts index 123c7ad33a..32e022c749 100644 --- a/packages/asset-uploader/src/targets/disk/upload.ts +++ b/packages/asset-uploader/src/targets/disk/upload.ts @@ -1,4 +1,3 @@ -import path from "path"; import sharp from "sharp"; import { create } from "../../"; import { ImagesUpload } from "../../schema"; @@ -6,11 +5,9 @@ import { ImagesUpload } from "../../schema"; export const uploadToDisk = async ({ formData, projectId, - folderInPublic, }: { formData: FormData; projectId: string; - folderInPublic: string; }) => { const imagesInfo = ImagesUpload.parse(formData.getAll("image")); const allInfo = imagesInfo.map(async (image) => { @@ -21,9 +18,9 @@ export const uploadToDisk = async ({ const data = { name: image.name, - path: path.join("/", folderInPublic, image.name), size: image.size, metadata, + location: "FS" as "FS" | "REMOTE", }; const newAsset = await create(projectId, data); diff --git a/packages/asset-uploader/src/targets/s3/handler.ts b/packages/asset-uploader/src/targets/s3/handler.ts index 2a6f86b536..b5919c10cf 100644 --- a/packages/asset-uploader/src/targets/s3/handler.ts +++ b/packages/asset-uploader/src/targets/s3/handler.ts @@ -42,9 +42,11 @@ export const s3UploadHandler: S3UploadHandler = async ({ const [name, extension] = getFilenameAndExtension(filename); const key = `${name}_${Date.now()}.${extension}`; + const ACL = s3Envs.WORKER_URL ? {} : { ACL: s3Envs.S3_ACL }; + const params: PutObjectCommandInput = { Bucket: s3Envs.S3_BUCKET, - ACL: s3Envs.S3_ACL, + ...ACL, Key: key, Body: uint8Array, ContentType: contentType, @@ -64,7 +66,7 @@ export const s3UploadHandler: S3UploadHandler = async ({ const upload = new Upload({ client, params }); - const newFile = ImagesUploadedSuccess.parse(await upload.done()); + ImagesUploadedSuccess.parse(await upload.done()); const image = sharp(uint8Array); const metadata = await image.metadata(); @@ -72,6 +74,6 @@ export const s3UploadHandler: S3UploadHandler = async ({ return JSON.stringify({ metadata, name: key, - path: newFile.Location, + location: "REMOTE", }); }; diff --git a/packages/asset-uploader/src/upload.ts b/packages/asset-uploader/src/upload.ts index d7795466f7..34fcf6075e 100644 --- a/packages/asset-uploader/src/upload.ts +++ b/packages/asset-uploader/src/upload.ts @@ -49,7 +49,6 @@ export const uploadAssets = async ({ return await uploadToDisk({ projectId, formData, - folderInPublic: fsUploadVars.FILE_UPLOAD_PATH, }); } }; diff --git a/packages/prisma-client/prisma/migrations/20220722131820_remove_path/migration.sql b/packages/prisma-client/prisma/migrations/20220722131820_remove_path/migration.sql new file mode 100644 index 0000000000..56cf9cbda7 --- /dev/null +++ b/packages/prisma-client/prisma/migrations/20220722131820_remove_path/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - You are about to drop the column `path` on the `Asset` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "Asset" DROP COLUMN "path"; diff --git a/packages/prisma-client/prisma/migrations/20220722132445_add_location/migration.sql b/packages/prisma-client/prisma/migrations/20220722132445_add_location/migration.sql new file mode 100644 index 0000000000..ae5ab4e639 --- /dev/null +++ b/packages/prisma-client/prisma/migrations/20220722132445_add_location/migration.sql @@ -0,0 +1,11 @@ +/* + Warnings: + + - Added the required column `location` to the `Asset` table without a default value. This is not possible if the table is not empty. + +*/ +-- CreateEnum +CREATE TYPE "Location" AS ENUM ('FS', 'REMOTE'); + +-- AlterTable +ALTER TABLE "Asset" ADD COLUMN "location" "Location" NOT NULL; diff --git a/packages/prisma-client/prisma/schema.prisma b/packages/prisma-client/prisma/schema.prisma index 9946c6fe12..e6cd47389d 100644 --- a/packages/prisma-client/prisma/schema.prisma +++ b/packages/prisma-client/prisma/schema.prisma @@ -15,6 +15,11 @@ model Team { users User[] } +enum Location { + FS + REMOTE +} + model Asset { id String @id @default(uuid()) project Project @relation(fields: [projectId], references: [id]) @@ -24,8 +29,8 @@ model Asset { width Decimal? height Decimal? name String - path String alt String? + location Location createdAt DateTime @default(now()) } diff --git a/packages/prisma-client/src/types.ts b/packages/prisma-client/src/types.ts index 29fbef7a4c..bbd3b0264d 100644 --- a/packages/prisma-client/src/types.ts +++ b/packages/prisma-client/src/types.ts @@ -4,4 +4,5 @@ export type { User, Breakpoints, Asset, + Location, } from "@prisma/client"; From 495b4399a0eb0381996f4a50c50de037bfda3b9a Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 16:30:12 +0200 Subject: [PATCH 02/13] start adding tests --- .../src/helpers/get-asset-path.test.ts | 51 +++++++++++++++++++ .../src/helpers/get-asset-path.ts | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 packages/asset-uploader/src/helpers/get-asset-path.test.ts diff --git a/packages/asset-uploader/src/helpers/get-asset-path.test.ts b/packages/asset-uploader/src/helpers/get-asset-path.test.ts new file mode 100644 index 0000000000..68190db7c3 --- /dev/null +++ b/packages/asset-uploader/src/helpers/get-asset-path.test.ts @@ -0,0 +1,51 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +import { Decimal } from "@prisma/client/runtime"; + +const commonAsset = { + id: "sa-546", + width: new Decimal(200), + height: new Decimal(200), + projectId: "jdhajk", + size: 2135, + format: "png", + createdAt: new Date(), + alt: "", +}; + +describe("getAssetPath", () => { + const OLD_ENV = process.env; + + beforeEach(() => { + jest.resetModules(); // Most important - it clears the cache + process.env = { ...OLD_ENV }; // Make a copy + }); + + afterAll(() => { + process.env = OLD_ENV; // Restore old environment + }); + + describe("local", () => { + test("return local path with no custom path", () => { + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: "FS", + }) + ).toBe("/uploads/test.png"); + }); + + test("return local path with custom path", () => { + process.env.FILE_UPLOAD_PATH = "assets"; + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: "FS", + }) + ).toBe("/assets/test.png"); + }); + }); +}); diff --git a/packages/asset-uploader/src/helpers/get-asset-path.ts b/packages/asset-uploader/src/helpers/get-asset-path.ts index ce103e2283..bfefab552e 100644 --- a/packages/asset-uploader/src/helpers/get-asset-path.ts +++ b/packages/asset-uploader/src/helpers/get-asset-path.ts @@ -7,7 +7,7 @@ const fsEnvs = fsEnvVariables.parse(process.env); export const getAssetPath = (asset: Asset) => { if (asset.location === "FS") { - return path.join(fsEnvs.FILE_UPLOAD_PATH, asset.name); + return path.join("/", fsEnvs.FILE_UPLOAD_PATH, asset.name); } if (asset.location === "REMOTE" && s3Envs.success) { From 884353b338b747b6eb56a4d3064d0aefb474302c Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 16:35:42 +0200 Subject: [PATCH 03/13] fix ts --- .../designer/features/sidebar-left/panels/asset-manager.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx index 9f6aa33fd3..fed7f5dd35 100644 --- a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx +++ b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx @@ -5,7 +5,11 @@ import { Form, useSubmit } from "@remix-run/react"; import { type Asset } from "@webstudio-is/prisma-client"; import { Image } from "~/shared/design-system/components/image"; -export const TabContent = ({ assets }: { assets: Array }) => { +export const TabContent = ({ + assets, +}: { + assets: Array; +}) => { const inputRef = useRef(null); const submit = useSubmit(); From a5aebed829229dca51b79ca3e9fba086f90ddc13 Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 16:40:24 +0200 Subject: [PATCH 04/13] better ts fix --- .../features/sidebar-left/panels/asset-manager.tsx | 8 ++------ packages/asset-uploader/src/db/load.ts | 5 ++++- packages/asset-uploader/src/index.ts | 1 + packages/asset-uploader/src/schema.ts | 3 +++ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx index fed7f5dd35..51f7e8976a 100644 --- a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx +++ b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx @@ -2,14 +2,10 @@ import { ImageIcon } from "~/shared/icons"; import { Button, Flex, Grid, Heading } from "~/shared/design-system"; import { useRef } from "react"; import { Form, useSubmit } from "@remix-run/react"; -import { type Asset } from "@webstudio-is/prisma-client"; import { Image } from "~/shared/design-system/components/image"; +import type { AssetWithPath } from "@webstudio-is/asset-uploader/src/schema"; -export const TabContent = ({ - assets, -}: { - assets: Array; -}) => { +export const TabContent = ({ assets }: { assets: Array }) => { const inputRef = useRef(null); const submit = useSubmit(); diff --git a/packages/asset-uploader/src/db/load.ts b/packages/asset-uploader/src/db/load.ts index b451d502e0..d43c8bca29 100644 --- a/packages/asset-uploader/src/db/load.ts +++ b/packages/asset-uploader/src/db/load.ts @@ -1,7 +1,10 @@ import { prisma, Project } from "@webstudio-is/prisma-client"; +import { AssetWithPath } from "../schema"; import { getAssetPath } from "../helpers/get-asset-path"; -export const loadByProject = async (projectId?: Project["id"]) => { +export const loadByProject = async ( + projectId?: Project["id"] +): Promise => { if (typeof projectId !== "string") { throw new Error("Tree ID required"); } diff --git a/packages/asset-uploader/src/index.ts b/packages/asset-uploader/src/index.ts index 22321d4d77..ce94c1a7f0 100644 --- a/packages/asset-uploader/src/index.ts +++ b/packages/asset-uploader/src/index.ts @@ -1,2 +1,3 @@ export * from "./upload"; export * from "./db"; +export * from "./schema"; diff --git a/packages/asset-uploader/src/schema.ts b/packages/asset-uploader/src/schema.ts index 0a8e3a2301..ba0d2be6eb 100644 --- a/packages/asset-uploader/src/schema.ts +++ b/packages/asset-uploader/src/schema.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { NodeOnDiskFile } from "@remix-run/node"; import { DEFAULT_UPLPOAD_PATH } from "./constants"; +import { Asset } from "@webstudio-is/prisma-client"; const SingleImageInUpload = z.instanceof(NodeOnDiskFile); @@ -31,3 +32,5 @@ export const fsEnvVariables = z.object({ export const assetEnvVariables = z.object({ MAX_UPLOAD_SIZE: z.string().optional().default("10"), }); + +export type AssetWithPath = Asset & { path: string }; From eded44b8bba4c63b5d5cf176fef0508134123513 Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 16:41:42 +0200 Subject: [PATCH 05/13] get asset from prisma client --- .../designer/features/sidebar-left/panels/asset-manager.tsx | 2 +- packages/asset-uploader/src/db/load.ts | 3 +-- packages/asset-uploader/src/index.ts | 1 - packages/asset-uploader/src/schema.ts | 3 --- packages/prisma-client/src/types.ts | 3 +++ 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx index 51f7e8976a..a4c8c4f44c 100644 --- a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx +++ b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx @@ -3,7 +3,7 @@ import { Button, Flex, Grid, Heading } from "~/shared/design-system"; import { useRef } from "react"; import { Form, useSubmit } from "@remix-run/react"; import { Image } from "~/shared/design-system/components/image"; -import type { AssetWithPath } from "@webstudio-is/asset-uploader/src/schema"; +import { AssetWithPath } from "@webstudio-is/prisma-client"; export const TabContent = ({ assets }: { assets: Array }) => { const inputRef = useRef(null); diff --git a/packages/asset-uploader/src/db/load.ts b/packages/asset-uploader/src/db/load.ts index d43c8bca29..5d1874c8f2 100644 --- a/packages/asset-uploader/src/db/load.ts +++ b/packages/asset-uploader/src/db/load.ts @@ -1,5 +1,4 @@ -import { prisma, Project } from "@webstudio-is/prisma-client"; -import { AssetWithPath } from "../schema"; +import { prisma, Project, AssetWithPath } from "@webstudio-is/prisma-client"; import { getAssetPath } from "../helpers/get-asset-path"; export const loadByProject = async ( diff --git a/packages/asset-uploader/src/index.ts b/packages/asset-uploader/src/index.ts index ce94c1a7f0..22321d4d77 100644 --- a/packages/asset-uploader/src/index.ts +++ b/packages/asset-uploader/src/index.ts @@ -1,3 +1,2 @@ export * from "./upload"; export * from "./db"; -export * from "./schema"; diff --git a/packages/asset-uploader/src/schema.ts b/packages/asset-uploader/src/schema.ts index ba0d2be6eb..0a8e3a2301 100644 --- a/packages/asset-uploader/src/schema.ts +++ b/packages/asset-uploader/src/schema.ts @@ -1,7 +1,6 @@ import { z } from "zod"; import { NodeOnDiskFile } from "@remix-run/node"; import { DEFAULT_UPLPOAD_PATH } from "./constants"; -import { Asset } from "@webstudio-is/prisma-client"; const SingleImageInUpload = z.instanceof(NodeOnDiskFile); @@ -32,5 +31,3 @@ export const fsEnvVariables = z.object({ export const assetEnvVariables = z.object({ MAX_UPLOAD_SIZE: z.string().optional().default("10"), }); - -export type AssetWithPath = Asset & { path: string }; diff --git a/packages/prisma-client/src/types.ts b/packages/prisma-client/src/types.ts index bbd3b0264d..5cebb0dfc6 100644 --- a/packages/prisma-client/src/types.ts +++ b/packages/prisma-client/src/types.ts @@ -1,3 +1,5 @@ +import { Asset } from "@prisma/client"; + export type { InstanceProps, Project, @@ -6,3 +8,4 @@ export type { Asset, Location, } from "@prisma/client"; +export type AssetWithPath = Asset & { path: string }; From c3703a911540cb6128aba29c73991bf863fd277b Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 16:46:59 +0200 Subject: [PATCH 06/13] fix ts --- apps/designer/app/designer/designer.tsx | 4 ++-- .../app/designer/features/sidebar-left/sidebar-left.tsx | 4 ++-- apps/designer/app/routes/designer/$id.tsx | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/designer/app/designer/designer.tsx b/apps/designer/app/designer/designer.tsx index 43b14a34b8..88f510de26 100644 --- a/apps/designer/app/designer/designer.tsx +++ b/apps/designer/app/designer/designer.tsx @@ -2,7 +2,7 @@ import { useCallback, useState } from "react"; import { DndProvider } from "react-dnd"; import { TouchBackend } from "react-dnd-touch-backend"; import { useSubscribe, usePublish } from "@webstudio-is/react-sdk"; -import { type Project, type Asset } from "@webstudio-is/prisma-client"; +import { type Project, type AssetWithPath } from "@webstudio-is/prisma-client"; import type { Config } from "~/config"; import type { HoveredInstanceData, @@ -173,7 +173,7 @@ const dndOptions = { enableMouseEvents: true }; type DesignerProps = { config: Config; project: Project; - assets: Array; + assets: Array; }; export const Designer = ({ config, project, assets }: DesignerProps) => { diff --git a/apps/designer/app/designer/features/sidebar-left/sidebar-left.tsx b/apps/designer/app/designer/features/sidebar-left/sidebar-left.tsx index 65b9cd96b8..8879a7fe92 100644 --- a/apps/designer/app/designer/features/sidebar-left/sidebar-left.tsx +++ b/apps/designer/app/designer/features/sidebar-left/sidebar-left.tsx @@ -1,6 +1,6 @@ import { useState, useCallback } from "react"; import { useSubscribe, type Publish } from "@webstudio-is/react-sdk"; -import type { Asset } from "@webstudio-is/prisma-client"; +import type { AssetWithPath } from "@webstudio-is/prisma-client"; import { Box, SidebarTabs, @@ -28,7 +28,7 @@ const none = { TabContent: () => null }; type SidebarLeftProps = { onDragChange: (isDragging: boolean) => void; publish: Publish; - assets: Array; + assets: Array; }; export const SidebarLeft = ({ diff --git a/apps/designer/app/routes/designer/$id.tsx b/apps/designer/app/routes/designer/$id.tsx index 50f6f2e296..556f337973 100644 --- a/apps/designer/app/routes/designer/$id.tsx +++ b/apps/designer/app/routes/designer/$id.tsx @@ -1,6 +1,6 @@ import { useActionData, useLoaderData } from "@remix-run/react"; import { ActionFunction, LoaderFunction } from "@remix-run/node"; -import type { Project, Asset } from "@webstudio-is/prisma-client"; +import type { Project, AssetWithPath } from "@webstudio-is/prisma-client"; import { Designer, links } from "~/designer"; import * as db from "~/shared/db"; import config from "~/config"; @@ -26,7 +26,7 @@ export const loader: LoaderFunction = async ({ params }) => { type Data = { config: typeof config; project: Project; - assets: Asset[]; + assets: AssetWithPath[]; }; type Error = { From 7b36ceb103752f8b44a235a6d0b742ab7977964b Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 16:53:31 +0200 Subject: [PATCH 07/13] add comemnts --- apps/designer/.env.example | 12 +++++++++++- packages/asset-uploader/src/targets/s3/handler.ts | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/designer/.env.example b/apps/designer/.env.example index c9e6ea3653..354ce8b964 100644 --- a/apps/designer/.env.example +++ b/apps/designer/.env.example @@ -6,4 +6,14 @@ DEV_LOGIN=true # relative to the public directory -FILE_UPLOAD_PATH="uploads" \ No newline at end of file +FILE_UPLOAD_PATH="uploads" + +# S3 envs +S3_BUCKET= +S3_REGION= +S3_ENDPOINT= +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= + +# if using r2 this your worker url +WORKER_URL= \ No newline at end of file diff --git a/packages/asset-uploader/src/targets/s3/handler.ts b/packages/asset-uploader/src/targets/s3/handler.ts index b5919c10cf..91862cb896 100644 --- a/packages/asset-uploader/src/targets/s3/handler.ts +++ b/packages/asset-uploader/src/targets/s3/handler.ts @@ -42,6 +42,7 @@ export const s3UploadHandler: S3UploadHandler = async ({ const [name, extension] = getFilenameAndExtension(filename); const key = `${name}_${Date.now()}.${extension}`; + // r2 does not have ACL support so in that case we pass nothing there const ACL = s3Envs.WORKER_URL ? {} : { ACL: s3Envs.S3_ACL }; const params: PutObjectCommandInput = { From c249f7c992e4ba5cfa1a2d8dd471283519d1607c Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 18:22:36 +0200 Subject: [PATCH 08/13] add more tests --- .../src/helpers/get-asset-path.test.ts | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/packages/asset-uploader/src/helpers/get-asset-path.test.ts b/packages/asset-uploader/src/helpers/get-asset-path.test.ts index 68190db7c3..6d4c084c4e 100644 --- a/packages/asset-uploader/src/helpers/get-asset-path.test.ts +++ b/packages/asset-uploader/src/helpers/get-asset-path.test.ts @@ -5,7 +5,7 @@ const commonAsset = { id: "sa-546", width: new Decimal(200), height: new Decimal(200), - projectId: "jdhajk", + projectId: "id", size: 2135, format: "png", createdAt: new Date(), @@ -16,16 +16,20 @@ describe("getAssetPath", () => { const OLD_ENV = process.env; beforeEach(() => { - jest.resetModules(); // Most important - it clears the cache - process.env = { ...OLD_ENV }; // Make a copy + jest.resetModules(); + process.env = { + ...OLD_ENV, + S3_ACCESS_KEY_ID: "testid", + S3_SECRET_ACCESS_KEY: "testkey", + }; }); afterAll(() => { - process.env = OLD_ENV; // Restore old environment + process.env = OLD_ENV; }); describe("local", () => { - test("return local path with no custom path", () => { + test("returns local path with no custom path", () => { const { getAssetPath } = require("./get-asset-path"); expect( getAssetPath({ @@ -48,4 +52,48 @@ describe("getAssetPath", () => { ).toBe("/assets/test.png"); }); }); + describe("s3", () => { + test("returns s3 url", () => { + process.env.S3_ENDPOINT = "https://fr1.s3.com"; + process.env.S3_BUCKET = "bucket"; + process.env.S3_REGION = "fr1"; + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: "REMOTE", + }) + ).toBe("https://bucket.fr1.s3.com/test.png"); + }); + test("returns s3 url and fixes extra slashes", () => { + process.env.S3_ENDPOINT = "https://fr1.s3.com//"; + process.env.S3_BUCKET = "bucket"; + process.env.S3_REGION = "fr1"; + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: "REMOTE", + }) + ).toBe("https://bucket.fr1.s3.com/test.png"); + }); + }); + describe("R2", () => { + test("returns r2 url", () => { + process.env.S3_ENDPOINT = "https://userid.r2.cloudflarestorage.com"; + process.env.S3_BUCKET = "bucket"; + process.env.S3_REGION = "auto"; + process.env.WORKER_URL = "https://worker-test.workers.dev"; + const { getAssetPath } = require("./get-asset-path"); + expect( + getAssetPath({ + ...commonAsset, + name: "test.png", + location: "REMOTE", + }) + ).toBe("https://worker-test.workers.dev/test.png"); + }); + }); }); From 684aeab1ced7934829fe457b6e3c0bdb8f51c368 Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 18:26:17 +0200 Subject: [PATCH 09/13] improve type usage --- packages/asset-uploader/src/db/create.ts | 4 ++-- packages/asset-uploader/src/helpers/get-asset-path.test.ts | 1 + packages/asset-uploader/src/helpers/get-asset-path.ts | 6 +++--- packages/asset-uploader/src/targets/disk/upload.ts | 3 ++- packages/asset-uploader/src/targets/s3/handler.ts | 3 ++- packages/prisma-client/src/types.ts | 3 +-- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/asset-uploader/src/db/create.ts b/packages/asset-uploader/src/db/create.ts index 1842726ebf..0cb2be322b 100644 --- a/packages/asset-uploader/src/db/create.ts +++ b/packages/asset-uploader/src/db/create.ts @@ -1,4 +1,4 @@ -import { prisma, Project } from "@webstudio-is/prisma-client"; +import { type Location, prisma, Project } from "@webstudio-is/prisma-client"; import sharp from "sharp"; export const create = async ( @@ -7,7 +7,7 @@ export const create = async ( name: string; size: number; metadata: sharp.Metadata; - location: "FS" | "REMOTE"; + location: Location; } ) => { const size = values.size || values.metadata.size || 0; diff --git a/packages/asset-uploader/src/helpers/get-asset-path.test.ts b/packages/asset-uploader/src/helpers/get-asset-path.test.ts index 6d4c084c4e..464838a1c7 100644 --- a/packages/asset-uploader/src/helpers/get-asset-path.test.ts +++ b/packages/asset-uploader/src/helpers/get-asset-path.test.ts @@ -21,6 +21,7 @@ describe("getAssetPath", () => { ...OLD_ENV, S3_ACCESS_KEY_ID: "testid", S3_SECRET_ACCESS_KEY: "testkey", + WORKER_URL: "", }; }); diff --git a/packages/asset-uploader/src/helpers/get-asset-path.ts b/packages/asset-uploader/src/helpers/get-asset-path.ts index bfefab552e..1bca6686db 100644 --- a/packages/asset-uploader/src/helpers/get-asset-path.ts +++ b/packages/asset-uploader/src/helpers/get-asset-path.ts @@ -1,4 +1,4 @@ -import { Asset } from "@webstudio-is/prisma-client"; +import { Asset, Location } from "@webstudio-is/prisma-client"; import { fsEnvVariables, s3EnvVariables } from "../schema"; import path from "path"; @@ -6,11 +6,11 @@ const s3Envs = s3EnvVariables.safeParse(process.env); const fsEnvs = fsEnvVariables.parse(process.env); export const getAssetPath = (asset: Asset) => { - if (asset.location === "FS") { + if (asset.location === Location.FS) { return path.join("/", fsEnvs.FILE_UPLOAD_PATH, asset.name); } - if (asset.location === "REMOTE" && s3Envs.success) { + if (asset.location === Location.REMOTE && s3Envs.success) { const s3Url = new URL(s3Envs.data.S3_ENDPOINT); if (s3Envs.data.WORKER_URL) { diff --git a/packages/asset-uploader/src/targets/disk/upload.ts b/packages/asset-uploader/src/targets/disk/upload.ts index 32e022c749..98fb57d26d 100644 --- a/packages/asset-uploader/src/targets/disk/upload.ts +++ b/packages/asset-uploader/src/targets/disk/upload.ts @@ -1,3 +1,4 @@ +import { Location } from "@webstudio-is/prisma-client"; import sharp from "sharp"; import { create } from "../../"; import { ImagesUpload } from "../../schema"; @@ -20,7 +21,7 @@ export const uploadToDisk = async ({ name: image.name, size: image.size, metadata, - location: "FS" as "FS" | "REMOTE", + location: Location.FS, }; const newAsset = await create(projectId, data); diff --git a/packages/asset-uploader/src/targets/s3/handler.ts b/packages/asset-uploader/src/targets/s3/handler.ts index 91862cb896..ed65e67a09 100644 --- a/packages/asset-uploader/src/targets/s3/handler.ts +++ b/packages/asset-uploader/src/targets/s3/handler.ts @@ -13,6 +13,7 @@ import { getArrayBufferFromIterable, getFilenameAndExtension, } from "../../helpers/array-buffer-helpers"; +import { Location } from "@webstudio-is/prisma-client"; type S3UploadHandler = ({ file, @@ -75,6 +76,6 @@ export const s3UploadHandler: S3UploadHandler = async ({ return JSON.stringify({ metadata, name: key, - location: "REMOTE", + location: Location.REMOTE, }); }; diff --git a/packages/prisma-client/src/types.ts b/packages/prisma-client/src/types.ts index 5cebb0dfc6..482ebaaee9 100644 --- a/packages/prisma-client/src/types.ts +++ b/packages/prisma-client/src/types.ts @@ -1,11 +1,10 @@ import { Asset } from "@prisma/client"; - +export { Location } from "@prisma/client"; export type { InstanceProps, Project, User, Breakpoints, Asset, - Location, } from "@prisma/client"; export type AssetWithPath = Asset & { path: string }; From 85f79fafa4d2c9af93886b18d7f3ea10ab80694b Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 18:27:04 +0200 Subject: [PATCH 10/13] use enum everywhere --- .../asset-uploader/src/helpers/get-asset-path.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/asset-uploader/src/helpers/get-asset-path.test.ts b/packages/asset-uploader/src/helpers/get-asset-path.test.ts index 464838a1c7..4c56a26a43 100644 --- a/packages/asset-uploader/src/helpers/get-asset-path.test.ts +++ b/packages/asset-uploader/src/helpers/get-asset-path.test.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/no-var-requires */ import { Decimal } from "@prisma/client/runtime"; +import { Location } from "@webstudio-is/prisma-client"; const commonAsset = { id: "sa-546", @@ -36,7 +37,7 @@ describe("getAssetPath", () => { getAssetPath({ ...commonAsset, name: "test.png", - location: "FS", + location: Location.FS, }) ).toBe("/uploads/test.png"); }); @@ -48,7 +49,7 @@ describe("getAssetPath", () => { getAssetPath({ ...commonAsset, name: "test.png", - location: "FS", + location: Location.FS, }) ).toBe("/assets/test.png"); }); @@ -63,7 +64,7 @@ describe("getAssetPath", () => { getAssetPath({ ...commonAsset, name: "test.png", - location: "REMOTE", + location: Location.REMOTE, }) ).toBe("https://bucket.fr1.s3.com/test.png"); }); @@ -76,7 +77,7 @@ describe("getAssetPath", () => { getAssetPath({ ...commonAsset, name: "test.png", - location: "REMOTE", + location: Location.REMOTE, }) ).toBe("https://bucket.fr1.s3.com/test.png"); }); @@ -92,7 +93,7 @@ describe("getAssetPath", () => { getAssetPath({ ...commonAsset, name: "test.png", - location: "REMOTE", + location: Location.REMOTE, }) ).toBe("https://worker-test.workers.dev/test.png"); }); From 91cef3796dd217b9a24db7d61d556fc8016c7b93 Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 18:31:37 +0200 Subject: [PATCH 11/13] more code cleanup --- apps/designer/app/designer/designer.tsx | 4 ++-- .../features/sidebar-left/panels/asset-manager.tsx | 4 ++-- .../features/sidebar-left/sidebar-left.tsx | 4 ++-- apps/designer/app/routes/designer/$id.tsx | 4 ++-- packages/asset-uploader/src/db/load.ts | 4 ++-- .../asset-uploader/src/helpers/get-asset-path.ts | 4 ++-- packages/prisma-client/src/types.ts | 14 +++++--------- 7 files changed, 17 insertions(+), 21 deletions(-) diff --git a/apps/designer/app/designer/designer.tsx b/apps/designer/app/designer/designer.tsx index 88f510de26..43b14a34b8 100644 --- a/apps/designer/app/designer/designer.tsx +++ b/apps/designer/app/designer/designer.tsx @@ -2,7 +2,7 @@ import { useCallback, useState } from "react"; import { DndProvider } from "react-dnd"; import { TouchBackend } from "react-dnd-touch-backend"; import { useSubscribe, usePublish } from "@webstudio-is/react-sdk"; -import { type Project, type AssetWithPath } from "@webstudio-is/prisma-client"; +import { type Project, type Asset } from "@webstudio-is/prisma-client"; import type { Config } from "~/config"; import type { HoveredInstanceData, @@ -173,7 +173,7 @@ const dndOptions = { enableMouseEvents: true }; type DesignerProps = { config: Config; project: Project; - assets: Array; + assets: Array; }; export const Designer = ({ config, project, assets }: DesignerProps) => { diff --git a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx index a4c8c4f44c..7771ac6268 100644 --- a/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx +++ b/apps/designer/app/designer/features/sidebar-left/panels/asset-manager.tsx @@ -3,9 +3,9 @@ import { Button, Flex, Grid, Heading } from "~/shared/design-system"; import { useRef } from "react"; import { Form, useSubmit } from "@remix-run/react"; import { Image } from "~/shared/design-system/components/image"; -import { AssetWithPath } from "@webstudio-is/prisma-client"; +import { Asset } from "@webstudio-is/prisma-client"; -export const TabContent = ({ assets }: { assets: Array }) => { +export const TabContent = ({ assets }: { assets: Array }) => { const inputRef = useRef(null); const submit = useSubmit(); diff --git a/apps/designer/app/designer/features/sidebar-left/sidebar-left.tsx b/apps/designer/app/designer/features/sidebar-left/sidebar-left.tsx index 8879a7fe92..65b9cd96b8 100644 --- a/apps/designer/app/designer/features/sidebar-left/sidebar-left.tsx +++ b/apps/designer/app/designer/features/sidebar-left/sidebar-left.tsx @@ -1,6 +1,6 @@ import { useState, useCallback } from "react"; import { useSubscribe, type Publish } from "@webstudio-is/react-sdk"; -import type { AssetWithPath } from "@webstudio-is/prisma-client"; +import type { Asset } from "@webstudio-is/prisma-client"; import { Box, SidebarTabs, @@ -28,7 +28,7 @@ const none = { TabContent: () => null }; type SidebarLeftProps = { onDragChange: (isDragging: boolean) => void; publish: Publish; - assets: Array; + assets: Array; }; export const SidebarLeft = ({ diff --git a/apps/designer/app/routes/designer/$id.tsx b/apps/designer/app/routes/designer/$id.tsx index 556f337973..50f6f2e296 100644 --- a/apps/designer/app/routes/designer/$id.tsx +++ b/apps/designer/app/routes/designer/$id.tsx @@ -1,6 +1,6 @@ import { useActionData, useLoaderData } from "@remix-run/react"; import { ActionFunction, LoaderFunction } from "@remix-run/node"; -import type { Project, AssetWithPath } from "@webstudio-is/prisma-client"; +import type { Project, Asset } from "@webstudio-is/prisma-client"; import { Designer, links } from "~/designer"; import * as db from "~/shared/db"; import config from "~/config"; @@ -26,7 +26,7 @@ export const loader: LoaderFunction = async ({ params }) => { type Data = { config: typeof config; project: Project; - assets: AssetWithPath[]; + assets: Asset[]; }; type Error = { diff --git a/packages/asset-uploader/src/db/load.ts b/packages/asset-uploader/src/db/load.ts index 5d1874c8f2..25cfa43872 100644 --- a/packages/asset-uploader/src/db/load.ts +++ b/packages/asset-uploader/src/db/load.ts @@ -1,9 +1,9 @@ -import { prisma, Project, AssetWithPath } from "@webstudio-is/prisma-client"; +import { prisma, Project, Asset } from "@webstudio-is/prisma-client"; import { getAssetPath } from "../helpers/get-asset-path"; export const loadByProject = async ( projectId?: Project["id"] -): Promise => { +): Promise => { if (typeof projectId !== "string") { throw new Error("Tree ID required"); } diff --git a/packages/asset-uploader/src/helpers/get-asset-path.ts b/packages/asset-uploader/src/helpers/get-asset-path.ts index 1bca6686db..9a69969b63 100644 --- a/packages/asset-uploader/src/helpers/get-asset-path.ts +++ b/packages/asset-uploader/src/helpers/get-asset-path.ts @@ -1,11 +1,11 @@ -import { Asset, Location } from "@webstudio-is/prisma-client"; +import { DbAsset, Location } from "@webstudio-is/prisma-client"; import { fsEnvVariables, s3EnvVariables } from "../schema"; import path from "path"; const s3Envs = s3EnvVariables.safeParse(process.env); const fsEnvs = fsEnvVariables.parse(process.env); -export const getAssetPath = (asset: Asset) => { +export const getAssetPath = (asset: DbAsset) => { if (asset.location === Location.FS) { return path.join("/", fsEnvs.FILE_UPLOAD_PATH, asset.name); } diff --git a/packages/prisma-client/src/types.ts b/packages/prisma-client/src/types.ts index 482ebaaee9..a6cf5d409c 100644 --- a/packages/prisma-client/src/types.ts +++ b/packages/prisma-client/src/types.ts @@ -1,10 +1,6 @@ -import { Asset } from "@prisma/client"; +import { type Asset as DbAsset } from "@prisma/client"; export { Location } from "@prisma/client"; -export type { - InstanceProps, - Project, - User, - Breakpoints, - Asset, -} from "@prisma/client"; -export type AssetWithPath = Asset & { path: string }; +export type { InstanceProps, Project, User, Breakpoints } from "@prisma/client"; + +export type Asset = DbAsset & { path: string }; +export { DbAsset }; From 6a3748b92c5ece82a61074e38448b5a02646b371 Mon Sep 17 00:00:00 2001 From: Sara Vieira Date: Fri, 22 Jul 2022 18:35:23 +0200 Subject: [PATCH 12/13] fix ts --- packages/prisma-client/src/types.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/prisma-client/src/types.ts b/packages/prisma-client/src/types.ts index a6cf5d409c..6fe42524f1 100644 --- a/packages/prisma-client/src/types.ts +++ b/packages/prisma-client/src/types.ts @@ -1,6 +1,6 @@ -import { type Asset as DbAsset } from "@prisma/client"; +import type { Asset as DbAsset } from "@prisma/client"; export { Location } from "@prisma/client"; export type { InstanceProps, Project, User, Breakpoints } from "@prisma/client"; export type Asset = DbAsset & { path: string }; -export { DbAsset }; +export type { DbAsset }; From e9e7c8dcccf5eb5cb7c9de443368833eb7a7d5ed Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Fri, 22 Jul 2022 18:53:15 +0000 Subject: [PATCH 13/13] fix: encodeURI --- packages/asset-uploader/src/targets/s3/handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/asset-uploader/src/targets/s3/handler.ts b/packages/asset-uploader/src/targets/s3/handler.ts index ed65e67a09..ab99c93665 100644 --- a/packages/asset-uploader/src/targets/s3/handler.ts +++ b/packages/asset-uploader/src/targets/s3/handler.ts @@ -41,7 +41,7 @@ export const s3UploadHandler: S3UploadHandler = async ({ } const [name, extension] = getFilenameAndExtension(filename); - const key = `${name}_${Date.now()}.${extension}`; + const key = encodeURI(`${name}_${Date.now()}.${extension}`); // r2 does not have ACL support so in that case we pass nothing there const ACL = s3Envs.WORKER_URL ? {} : { ACL: s3Envs.S3_ACL };