From aaaf57da9f24da69ff12a9469bb20d15fc207ef4 Mon Sep 17 00:00:00 2001
From: Ivan Starkov <istarkov@gmail.com>
Date: Thu, 2 Nov 2023 09:39:06 +0200
Subject: [PATCH] feat: Add Twitter card support & exclude pages from
 search/index (#2545)

## Description

ref #2544

Corresponding SAAS PR
https://github.com/webstudio-is/webstudio-saas/pull/215

- Support of page property - `excludePageFromSearch`
- Introduced functionality to enable the meta robot noindex tag for the
entire website.
- Implemented robots.txt support, which appears to be essential for
Twitter meta cards functionality.

Needs corresponding PR in SAAS deploy.

## Steps for reproduction

1. click button
2. expect xyz

## Code Review

- [ ] hi @kof, I need you to do
  - conceptual review (architecture, feature-correctness)
  - detailed review (read every line)
  - test it on preview

## Before requesting a review

- [ ] made a self-review
- [ ] added inline comments where things may be not obvious (the "why",
not "what")

## Before merging

- [ ] tested locally and on preview environment (preview dev login:
5de6)
- [ ] updated [test
cases](https://github.com/webstudio-is/webstudio/blob/main/apps/builder/docs/test-cases.md)
document
- [ ] added tests
- [ ] if any new env variables are added, added them to `.env.example`
and the `builder/env-check.js` if mandatory
---
 .../sidebar-left/panels/pages/settings.tsx    | 13 +++++++---
 .../app/extension.ts                          |  9 +++++++
 .../app/routes/[robots.txt].tsx               | 25 +++++++++++++++++++
 .../app/routes/_index.tsx                     | 16 +++++++++++-
 .../webstudio-custom-template/remix.env.d.ts  |  1 +
 .../app/extension.ts                          |  9 +++++++
 .../app/routes/[robots.txt].tsx               | 25 +++++++++++++++++++
 .../app/routes/_index.tsx                     | 16 +++++++++++-
 .../remix.env.d.ts                            |  1 +
 .../app/extension.ts                          |  9 +++++++
 .../app/routes/[robots.txt].tsx               | 25 +++++++++++++++++++
 .../app/routes/_index.tsx                     | 16 +++++++++++-
 .../remix.env.d.ts                            |  1 +
 .../webstudio-remix-vercel/app/extension.ts   |  9 +++++++
 .../routes/[_route_with_symbols_]._index.tsx  | 16 +++++++++++-
 .../app/routes/[form]._index.tsx              | 16 +++++++++++-
 .../app/routes/[heading-with-id]._index.tsx   | 16 +++++++++++-
 .../app/routes/[radix]._index.tsx             | 16 +++++++++++-
 .../app/routes/[robots.txt].tsx               | 25 +++++++++++++++++++
 .../app/routes/_index.tsx                     | 16 +++++++++++-
 .../webstudio-remix-vercel/remix.env.d.ts     |  1 +
 packages/cli/src/prebuild.ts                  | 16 ++++++------
 .../cli/templates/defaults/app/extension.ts   |  9 +++++++
 .../defaults/app/routes/[robots.txt].tsx      | 25 +++++++++++++++++++
 .../cli/templates/defaults/remix.env.d.ts     |  1 +
 packages/cli/templates/route-template.tsx     | 16 +++++++++++-
 26 files changed, 327 insertions(+), 21 deletions(-)
 create mode 100644 fixtures/webstudio-custom-template/app/extension.ts
 create mode 100644 fixtures/webstudio-custom-template/app/routes/[robots.txt].tsx
 create mode 100644 fixtures/webstudio-remix-netlify-edge-functions/app/extension.ts
 create mode 100644 fixtures/webstudio-remix-netlify-edge-functions/app/routes/[robots.txt].tsx
 create mode 100644 fixtures/webstudio-remix-netlify-functions/app/extension.ts
 create mode 100644 fixtures/webstudio-remix-netlify-functions/app/routes/[robots.txt].tsx
 create mode 100644 fixtures/webstudio-remix-vercel/app/extension.ts
 create mode 100644 fixtures/webstudio-remix-vercel/app/routes/[robots.txt].tsx
 create mode 100644 packages/cli/templates/defaults/app/extension.ts
 create mode 100644 packages/cli/templates/defaults/app/routes/[robots.txt].tsx

diff --git a/apps/builder/app/builder/features/sidebar-left/panels/pages/settings.tsx b/apps/builder/app/builder/features/sidebar-left/panels/pages/settings.tsx
index 1c080f7aa8..10db394238 100644
--- a/apps/builder/app/builder/features/sidebar-left/panels/pages/settings.tsx
+++ b/apps/builder/app/builder/features/sidebar-left/panels/pages/settings.tsx
@@ -60,7 +60,6 @@ import { ImageControl } from "~/builder/features/seo/image-control";
 import { ImageInfo } from "./image-info";
 import { SocialPreview } from "./social-preview";
 import { useEffectEvent } from "~/builder/features/ai/hooks/effect-event";
-import { getPublishedUrl } from "~/shared/router-utils";
 
 const fieldDefaultValues = {
   name: "Untitled",
@@ -178,13 +177,19 @@ const CopyPageDomainAndPathButton = ({
     >
       <Button
         color="ghost"
+        type="button"
         onPointerDown={(event) => {
-          navigator.clipboard.writeText("ddd");
+          navigator.clipboard.writeText(`https://${pageDomainAndPath}`);
           setPathIconState("checkmark");
           // Prevent tooltip to be closed
           event.stopPropagation();
         }}
-        prefix={pathIcon}
+        // Recreating Icon without pointer-events: none cause mouse leave/enter event to be fired again
+        prefix={
+          <Grid align="center" css={{ pointerEvents: "none" }}>
+            {pathIcon}
+          </Grid>
+        }
         css={{ justifySelf: "start" }}
         onMouseEnter={() => {
           setPathIconState("copy");
@@ -228,7 +233,7 @@ const FormFields = ({
   const faviconUrl = faviconAsset?.type === "image" ? faviconAsset.name : "";
 
   const project = projectStore.get();
-  const publishedUrl = new URL(getPublishedUrl(project?.domain ?? ""));
+  const publishedUrl = new URL(`https://${project?.domain}`);
 
   const pageDomainAndPath = [publishedUrl.host, values?.path]
     .filter(Boolean)
diff --git a/fixtures/webstudio-custom-template/app/extension.ts b/fixtures/webstudio-custom-template/app/extension.ts
new file mode 100644
index 0000000000..c42becab9a
--- /dev/null
+++ b/fixtures/webstudio-custom-template/app/extension.ts
@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}
diff --git a/fixtures/webstudio-custom-template/app/routes/[robots.txt].tsx b/fixtures/webstudio-custom-template/app/routes/[robots.txt].tsx
new file mode 100644
index 0000000000..1b8d531fd6
--- /dev/null
+++ b/fixtures/webstudio-custom-template/app/routes/[robots.txt].tsx
@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};
diff --git a/fixtures/webstudio-custom-template/app/routes/_index.tsx b/fixtures/webstudio-custom-template/app/routes/_index.tsx
index d67253fa9f..9858cc75de 100644
--- a/fixtures/webstudio-custom-template/app/routes/_index.tsx
+++ b/fixtures/webstudio-custom-template/app/routes/_index.tsx
@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",
diff --git a/fixtures/webstudio-custom-template/remix.env.d.ts b/fixtures/webstudio-custom-template/remix.env.d.ts
index dcf8c45e1d..8bf8b6dd71 100644
--- a/fixtures/webstudio-custom-template/remix.env.d.ts
+++ b/fixtures/webstudio-custom-template/remix.env.d.ts
@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+
diff --git a/fixtures/webstudio-remix-netlify-edge-functions/app/extension.ts b/fixtures/webstudio-remix-netlify-edge-functions/app/extension.ts
new file mode 100644
index 0000000000..c42becab9a
--- /dev/null
+++ b/fixtures/webstudio-remix-netlify-edge-functions/app/extension.ts
@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}
diff --git a/fixtures/webstudio-remix-netlify-edge-functions/app/routes/[robots.txt].tsx b/fixtures/webstudio-remix-netlify-edge-functions/app/routes/[robots.txt].tsx
new file mode 100644
index 0000000000..1b8d531fd6
--- /dev/null
+++ b/fixtures/webstudio-remix-netlify-edge-functions/app/routes/[robots.txt].tsx
@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};
diff --git a/fixtures/webstudio-remix-netlify-edge-functions/app/routes/_index.tsx b/fixtures/webstudio-remix-netlify-edge-functions/app/routes/_index.tsx
index d67253fa9f..9858cc75de 100644
--- a/fixtures/webstudio-remix-netlify-edge-functions/app/routes/_index.tsx
+++ b/fixtures/webstudio-remix-netlify-edge-functions/app/routes/_index.tsx
@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",
diff --git a/fixtures/webstudio-remix-netlify-edge-functions/remix.env.d.ts b/fixtures/webstudio-remix-netlify-edge-functions/remix.env.d.ts
index dcf8c45e1d..8bf8b6dd71 100644
--- a/fixtures/webstudio-remix-netlify-edge-functions/remix.env.d.ts
+++ b/fixtures/webstudio-remix-netlify-edge-functions/remix.env.d.ts
@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+
diff --git a/fixtures/webstudio-remix-netlify-functions/app/extension.ts b/fixtures/webstudio-remix-netlify-functions/app/extension.ts
new file mode 100644
index 0000000000..c42becab9a
--- /dev/null
+++ b/fixtures/webstudio-remix-netlify-functions/app/extension.ts
@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}
diff --git a/fixtures/webstudio-remix-netlify-functions/app/routes/[robots.txt].tsx b/fixtures/webstudio-remix-netlify-functions/app/routes/[robots.txt].tsx
new file mode 100644
index 0000000000..1b8d531fd6
--- /dev/null
+++ b/fixtures/webstudio-remix-netlify-functions/app/routes/[robots.txt].tsx
@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};
diff --git a/fixtures/webstudio-remix-netlify-functions/app/routes/_index.tsx b/fixtures/webstudio-remix-netlify-functions/app/routes/_index.tsx
index d67253fa9f..9858cc75de 100644
--- a/fixtures/webstudio-remix-netlify-functions/app/routes/_index.tsx
+++ b/fixtures/webstudio-remix-netlify-functions/app/routes/_index.tsx
@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",
diff --git a/fixtures/webstudio-remix-netlify-functions/remix.env.d.ts b/fixtures/webstudio-remix-netlify-functions/remix.env.d.ts
index dcf8c45e1d..8bf8b6dd71 100644
--- a/fixtures/webstudio-remix-netlify-functions/remix.env.d.ts
+++ b/fixtures/webstudio-remix-netlify-functions/remix.env.d.ts
@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+
diff --git a/fixtures/webstudio-remix-vercel/app/extension.ts b/fixtures/webstudio-remix-vercel/app/extension.ts
new file mode 100644
index 0000000000..c42becab9a
--- /dev/null
+++ b/fixtures/webstudio-remix-vercel/app/extension.ts
@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}
diff --git a/fixtures/webstudio-remix-vercel/app/routes/[_route_with_symbols_]._index.tsx b/fixtures/webstudio-remix-vercel/app/routes/[_route_with_symbols_]._index.tsx
index 1a0a54288b..fc3210c542 100644
--- a/fixtures/webstudio-remix-vercel/app/routes/[_route_with_symbols_]._index.tsx
+++ b/fixtures/webstudio-remix-vercel/app/routes/[_route_with_symbols_]._index.tsx
@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",
diff --git a/fixtures/webstudio-remix-vercel/app/routes/[form]._index.tsx b/fixtures/webstudio-remix-vercel/app/routes/[form]._index.tsx
index 5d8e4720ce..755d77a645 100644
--- a/fixtures/webstudio-remix-vercel/app/routes/[form]._index.tsx
+++ b/fixtures/webstudio-remix-vercel/app/routes/[form]._index.tsx
@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",
diff --git a/fixtures/webstudio-remix-vercel/app/routes/[heading-with-id]._index.tsx b/fixtures/webstudio-remix-vercel/app/routes/[heading-with-id]._index.tsx
index be7bf26b3f..9ee9d035af 100644
--- a/fixtures/webstudio-remix-vercel/app/routes/[heading-with-id]._index.tsx
+++ b/fixtures/webstudio-remix-vercel/app/routes/[heading-with-id]._index.tsx
@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",
diff --git a/fixtures/webstudio-remix-vercel/app/routes/[radix]._index.tsx b/fixtures/webstudio-remix-vercel/app/routes/[radix]._index.tsx
index 6d843561df..a34cc204d7 100644
--- a/fixtures/webstudio-remix-vercel/app/routes/[radix]._index.tsx
+++ b/fixtures/webstudio-remix-vercel/app/routes/[radix]._index.tsx
@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",
diff --git a/fixtures/webstudio-remix-vercel/app/routes/[robots.txt].tsx b/fixtures/webstudio-remix-vercel/app/routes/[robots.txt].tsx
new file mode 100644
index 0000000000..1b8d531fd6
--- /dev/null
+++ b/fixtures/webstudio-remix-vercel/app/routes/[robots.txt].tsx
@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};
diff --git a/fixtures/webstudio-remix-vercel/app/routes/_index.tsx b/fixtures/webstudio-remix-vercel/app/routes/_index.tsx
index d67253fa9f..9858cc75de 100644
--- a/fixtures/webstudio-remix-vercel/app/routes/_index.tsx
+++ b/fixtures/webstudio-remix-vercel/app/routes/_index.tsx
@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",
diff --git a/fixtures/webstudio-remix-vercel/remix.env.d.ts b/fixtures/webstudio-remix-vercel/remix.env.d.ts
index dcf8c45e1d..8bf8b6dd71 100644
--- a/fixtures/webstudio-remix-vercel/remix.env.d.ts
+++ b/fixtures/webstudio-remix-vercel/remix.env.d.ts
@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+
diff --git a/packages/cli/src/prebuild.ts b/packages/cli/src/prebuild.ts
index 78bc695a73..fbecb41e14 100644
--- a/packages/cli/src/prebuild.ts
+++ b/packages/cli/src/prebuild.ts
@@ -214,6 +214,14 @@ export const prebuild = async (options: {
 
   spinner.text = "Generating files";
 
+  const appRoot = "app";
+
+  const generatedDir = join(appRoot, "__generated__");
+  await rm(generatedDir, { recursive: true, force: true });
+
+  const routesDir = join(appRoot, "routes");
+  await rm(routesDir, { recursive: true, force: true });
+
   await copyTemplates();
 
   if (options.template !== undefined) {
@@ -390,14 +398,6 @@ export const prebuild = async (options: {
 
   spinner.text = "Generating routes and pages";
 
-  const appRoot = "app";
-
-  const generatedDir = join(appRoot, "__generated__");
-  await rm(generatedDir, { recursive: true, force: true });
-
-  const routesDir = join(appRoot, "routes");
-  await rm(routesDir, { recursive: true, force: true });
-
   const routeFileTemplate = await readFile(
     normalize(
       join(
diff --git a/packages/cli/templates/defaults/app/extension.ts b/packages/cli/templates/defaults/app/extension.ts
new file mode 100644
index 0000000000..c42becab9a
--- /dev/null
+++ b/packages/cli/templates/defaults/app/extension.ts
@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}
diff --git a/packages/cli/templates/defaults/app/routes/[robots.txt].tsx b/packages/cli/templates/defaults/app/routes/[robots.txt].tsx
new file mode 100644
index 0000000000..1b8d531fd6
--- /dev/null
+++ b/packages/cli/templates/defaults/app/routes/[robots.txt].tsx
@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};
diff --git a/packages/cli/templates/defaults/remix.env.d.ts b/packages/cli/templates/defaults/remix.env.d.ts
index dcf8c45e1d..8bf8b6dd71 100644
--- a/packages/cli/templates/defaults/remix.env.d.ts
+++ b/packages/cli/templates/defaults/remix.env.d.ts
@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+
diff --git a/packages/cli/templates/route-template.tsx b/packages/cli/templates/route-template.tsx
index 5abdb3b5e0..68c9f3a597 100644
--- a/packages/cli/templates/route-template.tsx
+++ b/packages/cli/templates/route-template.tsx
@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",