feat: Add Twitter card support & exclude pages from search/index (#2545)

## Description

ref #2544

Corresponding SAAS PR
https://github.com/webstudio-is/webstudio-saas/pull/215

- Support of page property - `excludePageFromSearch`
- Introduced functionality to enable the meta robot noindex tag for the
entire website.
- Implemented robots.txt support, which appears to be essential for
Twitter meta cards functionality.

Needs corresponding PR in SAAS deploy.

## Steps for reproduction

1. click button
2. expect xyz

## Code Review

- [ ] hi @kof, I need you to do
  - conceptual review (architecture, feature-correctness)
  - detailed review (read every line)
  - test it on preview

## Before requesting a review

- [ ] made a self-review
- [ ] added inline comments where things may be not obvious (the "why",
not "what")

## Before merging

- [ ] tested locally and on preview environment (preview dev login:
5de6)
- [ ] updated [test
cases](https://github.com/webstudio-is/webstudio/blob/main/apps/builder/docs/test-cases.md)
document
- [ ] added tests
- [ ] if any new env variables are added, added them to `.env.example`
and the `builder/env-check.js` if mandatory

apps/builder/app/builder/features/sidebar-left/panels/pages/settings.tsx

@@ -60,7 +60,6 @@ import { ImageControl } from "~/builder/features/seo/image-control";
 import { ImageInfo } from "./image-info";
 import { SocialPreview } from "./social-preview";
 import { useEffectEvent } from "~/builder/features/ai/hooks/effect-event";
-import { getPublishedUrl } from "~/shared/router-utils";
 
 const fieldDefaultValues = {
   name: "Untitled",
@@ -178,13 +177,19 @@ const CopyPageDomainAndPathButton = ({
     >
       <Button
         color="ghost"
+        type="button"
         onPointerDown={(event) => {
-          navigator.clipboard.writeText("ddd");
+          navigator.clipboard.writeText(`https://${pageDomainAndPath}`);
           setPathIconState("checkmark");
           // Prevent tooltip to be closed
           event.stopPropagation();
         }}
-        prefix={pathIcon}
+        // Recreating Icon without pointer-events: none cause mouse leave/enter event to be fired again
+        prefix={
+          <Grid align="center" css={{ pointerEvents: "none" }}>
+            {pathIcon}
+          </Grid>
+        }
         css={{ justifySelf: "start" }}
         onMouseEnter={() => {
           setPathIconState("copy");
@@ -228,7 +233,7 @@ const FormFields = ({
   const faviconUrl = faviconAsset?.type === "image" ? faviconAsset.name : "";
 
   const project = projectStore.get();
-  const publishedUrl = new URL(getPublishedUrl(project?.domain ?? ""));
+  const publishedUrl = new URL(`https://${project?.domain}`);
 
   const pageDomainAndPath = [publishedUrl.host, values?.path]
     .filter(Boolean)

fixtures/webstudio-custom-template/app/extension.ts

@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}

fixtures/webstudio-custom-template/app/routes/[robots.txt].tsx

@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};

fixtures/webstudio-custom-template/app/routes/_index.tsx

@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",

fixtures/webstudio-custom-template/remix.env.d.ts

@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+

fixtures/webstudio-remix-netlify-edge-functions/app/extension.ts

@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}

fixtures/webstudio-remix-netlify-edge-functions/app/routes/[robots.txt].tsx

@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};

fixtures/webstudio-remix-netlify-edge-functions/app/routes/_index.tsx

@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",

fixtures/webstudio-remix-netlify-edge-functions/remix.env.d.ts

@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+

fixtures/webstudio-remix-netlify-functions/app/extension.ts

@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}

fixtures/webstudio-remix-netlify-functions/app/routes/[robots.txt].tsx

@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};

fixtures/webstudio-remix-netlify-functions/app/routes/_index.tsx

@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",

fixtures/webstudio-remix-netlify-functions/remix.env.d.ts

@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+

fixtures/webstudio-remix-vercel/app/extension.ts

@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}

fixtures/webstudio-remix-vercel/app/routes/[_route_with_symbols_]._index.tsx

@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",

fixtures/webstudio-remix-vercel/app/routes/[form]._index.tsx

@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",

fixtures/webstudio-remix-vercel/app/routes/[heading-with-id]._index.tsx

@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",

fixtures/webstudio-remix-vercel/app/routes/[radix]._index.tsx

@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",

fixtures/webstudio-remix-vercel/app/routes/[robots.txt].tsx

@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};

fixtures/webstudio-remix-vercel/app/routes/_index.tsx

@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",

fixtures/webstudio-remix-vercel/remix.env.d.ts

@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+

packages/cli/src/prebuild.ts

@@ -214,6 +214,14 @@ export const prebuild = async (options: {
 
   spinner.text = "Generating files";
 
+  const appRoot = "app";
+
+  const generatedDir = join(appRoot, "__generated__");
+  await rm(generatedDir, { recursive: true, force: true });
+
+  const routesDir = join(appRoot, "routes");
+  await rm(routesDir, { recursive: true, force: true });
+
   await copyTemplates();
 
   if (options.template !== undefined) {
@@ -390,14 +398,6 @@ export const prebuild = async (options: {
 
   spinner.text = "Generating routes and pages";
 
-  const appRoot = "app";
-
-  const generatedDir = join(appRoot, "__generated__");
-  await rm(generatedDir, { recursive: true, force: true });
-
-  const routesDir = join(appRoot, "routes");
-  await rm(routesDir, { recursive: true, force: true });
-
   const routeFileTemplate = await readFile(
     normalize(
       join(

packages/cli/templates/defaults/app/extension.ts

@@ -0,0 +1,9 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { AppLoadContext } from "@remix-run/server-runtime";
+
+declare module "@remix-run/server-runtime" {
+  interface AppLoadContext {
+    EXCLUDE_FROM_SEARCH: boolean;
+  }
+}

packages/cli/templates/defaults/app/routes/[robots.txt].tsx

@@ -0,0 +1,25 @@
+import type { LoaderArgs } from "@remix-run/server-runtime";
+
+export const loader = (arg: LoaderArgs) => {
+  const host =
+    arg.request.headers.get("x-forwarded-host") ||
+    arg.request.headers.get("host") ||
+    "";
+
+  return new Response(
+    `
+User-agent: *
+Disallow: /api/
+
+# Uncomment
+# Sitemap: https://${host}/sitemap.xml
+
+  `,
+    {
+      headers: {
+        "Content-Type": "text/plain",
+      },
+      status: 200,
+    }
+  );
+};

packages/cli/templates/defaults/remix.env.d.ts

@@ -1,2 +1,3 @@
 /// <reference types="@remix-run/dev" />
 /// <reference types="@remix-run/node" />
+

packages/cli/templates/route-template.tsx

@@ -39,8 +39,15 @@ export const loader = async (arg: LoaderArgs) => {
   url.host = host;
   url.protocol = "https";
 
+  // typecheck
+  arg.context.EXCLUDE_FROM_SEARCH satisfies boolean;
+
   return json(
-    { host, url: url.href },
+    {
+      host,
+      url: url.href,
+      excludeFromSearch: arg.context.EXCLUDE_FROM_SEARCH,
+    },
     // No way for current information to change, so add cache for 10 minutes
     // In case of CRM Data, this should be set to 0
     { headers: { "Cache-Control": "public, max-age=600" } }
@@ -93,6 +100,13 @@ export const meta: V2_ServerRuntimeMetaFunction<typeof loader> = ({ data }) => {
     });
   }
 
+  if (page.meta.excludePageFromSearch || data?.excludeFromSearch) {
+    metas.push({
+      name: "robots",
+      content: "noindex, nofollow",
+    });
+  }
+
   if (page.meta.description) {
     metas.push({
       name: "description",