From 279e2ec09380263cbbe04c70dadabe593fa87aa0 Mon Sep 17 00:00:00 2001 From: istarkov Date: Mon, 21 Nov 2022 06:20:33 +0000 Subject: [PATCH] Add sanitize --- .../designer/shared/assets/sanitize-s3-key.ts | 26 +++++++++++++++++++ .../app/designer/shared/assets/use-assets.ts | 5 +++- .../asset-uploader/src/targets/s3/upload.ts | 9 ++++--- .../src/utils/get-asset-path.ts | 4 +-- .../src/utils/sanitize-s3-key.ts | 24 +++++++++++++++++ 5 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 apps/designer/app/designer/shared/assets/sanitize-s3-key.ts create mode 100644 packages/asset-uploader/src/utils/sanitize-s3-key.ts diff --git a/apps/designer/app/designer/shared/assets/sanitize-s3-key.ts b/apps/designer/app/designer/shared/assets/sanitize-s3-key.ts new file mode 100644 index 0000000000..d5435fa28b --- /dev/null +++ b/apps/designer/app/designer/shared/assets/sanitize-s3-key.ts @@ -0,0 +1,26 @@ +/** + * TEMPORARY: This function is a temporary fix for https://github.com/remix-run/remix/issues/4443 + * + * Based on this: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + * Characters that might require special handling + * ASCII character ranges 00–1F hex (0–31 decimal) and 7F (127 decimal) + * "&","$", "@", "=", ";" "/", ":", "+", Significant sequences of spaces might be lost in some uses (especially multiple spaces) + * ",", "?" + * Characters to avoid + * Non-printable ASCII characters (128–255 decimal characters) + * "\", "{", "^", "}", "%", "`", "]", ', ", “, ”, ">", "[", "~", "<", "#", "|" + **/ + +// eslint-disable-next-line no-control-regex +const specialRegexps = [/[\x00-\x1F\x7F]+/g, /[&$@=;/:+\s,?]+/g]; + +// eslint-disable-next-line no-control-regex +const avoidRegexps = [/[\x80-\xFF]+/g, /[\\{^}%`\]'"“”>[~<#|]+/g]; + +const allRegexps = [...specialRegexps, ...avoidRegexps]; + +const REPLACE_CHAR = "_"; + +export const sanitizeS3Key = (str: string): string => { + return allRegexps.reduce((r, reg) => r.replace(reg, REPLACE_CHAR), str); +}; diff --git a/apps/designer/app/designer/shared/assets/use-assets.ts b/apps/designer/app/designer/shared/assets/use-assets.ts index ba32d05c9b..933b63ddd7 100644 --- a/apps/designer/app/designer/shared/assets/use-assets.ts +++ b/apps/designer/app/designer/shared/assets/use-assets.ts @@ -10,6 +10,7 @@ import ObjectID from "bson-objectid"; import { useMemo } from "react"; import { restAssetsPath } from "~/shared/router-utils"; import { useAssets as useAssetsState, useProject } from "../nano-states"; +import { sanitizeS3Key } from "./sanitize-s3-key"; import { PreviewAsset } from "./types"; const toPreviewAssets = (formData: FormData): Promise => { @@ -54,7 +55,9 @@ const toFormData = (type: AssetType, input: HTMLInputElement) => { ); continue; } - formData.append(type, file, file.name); + + // sanitizeS3Key here is just because of https://github.com/remix-run/remix/issues/4443 + formData.append(type, file, sanitizeS3Key(file.name)); } return formData; }; diff --git a/packages/asset-uploader/src/targets/s3/upload.ts b/packages/asset-uploader/src/targets/s3/upload.ts index fdb8524354..75815f2791 100644 --- a/packages/asset-uploader/src/targets/s3/upload.ts +++ b/packages/asset-uploader/src/targets/s3/upload.ts @@ -14,6 +14,7 @@ import { createMany } from "../../db"; import type { Asset } from "../../types"; import { getUniqueFilename } from "../../utils/get-unique-filename"; import { getS3Client } from "./client"; +import { sanitizeS3Key } from "../../utils/sanitize-s3-key"; const AssetsUploadedSuccess = z.object({ Location: z.string(), @@ -72,7 +73,9 @@ const uploadHandler = async ({ throw new Error("Filename is required"); } - const uniqueFilename = getUniqueFilename(file.filename); + const fileName = sanitizeS3Key(file.filename); + + const uniqueFilename = getUniqueFilename(fileName); const s3Env = S3Env.parse(process.env); @@ -82,12 +85,12 @@ const uploadHandler = async ({ const params: PutObjectCommandInput = { ...ACL, Bucket: s3Env.S3_BUCKET, - Key: encodeURIComponent(uniqueFilename), + Key: uniqueFilename, Body: data, ContentType: file.contentType, Metadata: { // encodeURIComponent is needed to support special characters like Cyrillic - filename: "meta", // encodeURIComponent(file.filename) || "unnamed", + filename: encodeURIComponent(fileName) || "unnamed", }, }; diff --git a/packages/asset-uploader/src/utils/get-asset-path.ts b/packages/asset-uploader/src/utils/get-asset-path.ts index ad6e835a4f..a12c48a13f 100644 --- a/packages/asset-uploader/src/utils/get-asset-path.ts +++ b/packages/asset-uploader/src/utils/get-asset-path.ts @@ -28,13 +28,13 @@ export const getAssetPath = (asset: DbAsset) => { // see /packages/asset-uploader/src/targets/s3/upload.ts "Key: encodeURIComponent(uniqueFilename)" // here we need to encode file name, i.e cndUrl.pathname = encodeURIComponent(fileName) // but having that fileName = encodeURIComponent(asset.name) we are doing double encoding - cndUrl.pathname = encodeURIComponent(encodeURIComponent(asset.name)); + cndUrl.pathname = asset.name; return cndUrl.toString(); } const s3Url = new URL(s3Envs.data.S3_ENDPOINT); s3Url.hostname = `${s3Envs.data.S3_BUCKET}.${s3Url.hostname}`; // See above comment about double encoding - s3Url.pathname = encodeURIComponent(encodeURIComponent(asset.name)); + s3Url.pathname = asset.name; return s3Url.toString(); } diff --git a/packages/asset-uploader/src/utils/sanitize-s3-key.ts b/packages/asset-uploader/src/utils/sanitize-s3-key.ts new file mode 100644 index 0000000000..5102c3032d --- /dev/null +++ b/packages/asset-uploader/src/utils/sanitize-s3-key.ts @@ -0,0 +1,24 @@ +/** + * Based on this: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + * Characters that might require special handling + * ASCII character ranges 00–1F hex (0–31 decimal) and 7F (127 decimal) + * "&","$", "@", "=", ";" "/", ":", "+", Significant sequences of spaces might be lost in some uses (especially multiple spaces) + * ",", "?" + * Characters to avoid + * Non-printable ASCII characters (128–255 decimal characters) + * "\", "{", "^", "}", "%", "`", "]", ', ", “, ”, ">", "[", "~", "<", "#", "|" + **/ + +// eslint-disable-next-line no-control-regex +const specialRegexps = [/[\x00-\x1F\x7F]+/g, /[&$@=;/:+\s,?]+/g]; + +// eslint-disable-next-line no-control-regex +const avoidRegexps = [/[\x80-\xFF]+/g, /[\\{^}%`\]'"“”>[~<#|]+/g]; + +const allRegexps = [...specialRegexps, ...avoidRegexps]; + +const REPLACE_CHAR = "_"; + +export const sanitizeS3Key = (str: string): string => { + return allRegexps.reduce((r, reg) => r.replace(reg, REPLACE_CHAR), str); +};