fix(seo): noindex non-production sim.ai hosts, redirect indexed 404s (#5988)

* fix(seo): noindex non-production sim.ai hosts, redirect indexed 404s

Non-production deployments (dev.sim.ai, staging.sim.ai) serve the same
build as www.sim.ai and were fully crawlable. Send X-Robots-Tag noindex
for those hosts, and 301 seven marketing paths that an external SEO
audit found returning 404.

* fix(seo): parse only the first entry of a comma-joined forwarded host

A multi-value x-forwarded-host survives the port split intact, so
endsWith('.sim.ai') matched on the trailing entry and could apply
noindex to the canonical site. Normalize with the same
split(',')[0].trim() the rest of the repo uses for forwarded headers.

* fix(seo): drop the /security redirect, complete the urls test mock

security.txt advertises /security as its RFC 9116 Policy URI, so a
permanent redirect to marketing would mislead that link and shadow a
real policy page added later.

urlsMock is installed globally and documents itself as carrying every
real export, but was missing CANONICAL_SITE_HOST and
isNonCanonicalSimHost — any test loading proxy.ts would have called
undefined in track().

Author: waleedlatif1

apps/sim/lib/core/utils/urls.test.ts

@@ -17,6 +17,7 @@ import {
   getBrowserOrigin,
   getSocketUrl,
   isLocalhostUrl,
+  isNonCanonicalSimHost,
   isSafeHttpUrl,
   parseOriginList,
 } from '@/lib/core/utils/urls'
@@ -163,3 +164,41 @@ describe('isSafeHttpUrl', () => {
     expect(isSafeHttpUrl('http://')).toBe(false)
   })
 })
+
+describe('isNonCanonicalSimHost', () => {
+  it.each(['www.sim.ai', 'sim.ai', 'WWW.SIM.AI', 'www.sim.ai:443'])(
+    'treats %s as the canonical marketing site',
+    (host) => {
+      expect(isNonCanonicalSimHost(host)).toBe(false)
+    }
+  )
+
+  it.each(['dev.sim.ai', 'www.dev.sim.ai', 'staging.sim.ai', 'prod.sockets.sim.ai'])(
+    'treats %s as non-canonical',
+    (host) => {
+      expect(isNonCanonicalSimHost(host)).toBe(true)
+    }
+  )
+
+  it.each(['sim.example.com', 'localhost:3000', 'notsim.ai', 'sim.ai.evil.com'])(
+    'leaves %s alone',
+    (host) => {
+      expect(isNonCanonicalSimHost(host)).toBe(false)
+    }
+  )
+
+  it.each(['www.sim.ai, dev.sim.ai', 'sim.ai,dev.sim.ai', '  www.sim.ai , staging.sim.ai'])(
+    'classifies a comma-joined forwarded host by its first entry (%s)',
+    (host) => {
+      expect(isNonCanonicalSimHost(host)).toBe(false)
+    }
+  )
+
+  it('still flags a comma-joined host whose first entry is non-canonical', () => {
+    expect(isNonCanonicalSimHost('dev.sim.ai, www.sim.ai')).toBe(true)
+  })
+
+  it('does not throw on an empty host', () => {
+    expect(isNonCanonicalSimHost('')).toBe(false)
+  })
+})

apps/sim/lib/core/utils/urls.ts

@@ -4,6 +4,9 @@ import { isProd } from '@/lib/core/config/env-flags'
 /** Canonical base URL for the public-facing marketing site. No trailing slash. */
 export const SITE_URL = 'https://www.sim.ai'
 
+/** Host of the canonical marketing site, e.g. `www.sim.ai`. */
+export const CANONICAL_SITE_HOST = new URL(SITE_URL).host
+
 function hasHttpProtocol(url: string): boolean {
   return /^https?:\/\//i.test(url)
 }
@@ -88,14 +91,36 @@ export function getBaseDomain(): string {
   }
 }
 
+/** Drops a leading `www.` label, e.g. `www.sim.ai` -> `sim.ai`. */
+function stripWwwPrefix(host: string): string {
+  return host.startsWith('www.') ? host.slice(4) : host
+}
+
+/**
+ * True for a sim.ai host that is not the canonical marketing site — dev.sim.ai,
+ * staging.sim.ai, and their www variants serve the same build as www.sim.ai, so
+ * search engines treat them as duplicates unless told otherwise.
+ *
+ * `sim.ai` and `www.sim.ai` are both canonical. Self-hosted domains return
+ * false, as do lookalikes such as `notsim.ai`.
+ *
+ * Takes the first entry of a comma-joined forwarded host so a chained proxy
+ * can't make the canonical site look non-canonical via a trailing entry.
+ */
+export function isNonCanonicalSimHost(host: string): boolean {
+  const first = host.split(',')[0]?.trim() ?? ''
+  const hostname = stripWwwPrefix(first.toLowerCase().split(':')[0])
+  const canonical = stripWwwPrefix(CANONICAL_SITE_HOST)
+  return hostname !== canonical && hostname.endsWith(`.${canonical}`)
+}
+
 /**
  * Returns the domain for email addresses, stripping www subdomain for Resend compatibility
  * @returns The email domain (e.g., 'sim.ai' instead of 'www.sim.ai')
  */
 export function getEmailDomain(): string {
   try {
-    const baseDomain = getBaseDomain()
-    return baseDomain.startsWith('www.') ? baseDomain.substring(4) : baseDomain
+    return stripWwwPrefix(getBaseDomain())
   } catch (_e) {
     return isProd ? 'sim.ai' : 'localhost:3000'
   }

apps/sim/next.config.ts

@@ -531,6 +531,28 @@ const nextConfig: NextConfig = {
       }
     )
 
+    /**
+     * Indexed 404s from an external SEO audit. The capability paths read as
+     * tool/feature pages and map to the integrations catalog; the rest have no
+     * closer successor than the homepage.
+     *
+     * `/security` is deliberately excluded: security.txt advertises it as the
+     * RFC 9116 `Policy` URI, so a permanent redirect to marketing would both
+     * mislead that link and shadow a real policy page added later.
+     */
+    redirects.push(
+      ...['read', 'research', 'scrape'].map((slug) => ({
+        source: `/${slug}`,
+        destination: '/integrations',
+        permanent: true,
+      })),
+      ...['actions', 'crawl', 'fast'].map((slug) => ({
+        source: `/${slug}`,
+        destination: '/',
+        permanent: true,
+      }))
+    )
+
     return redirects
   },
   async rewrites() {

apps/sim/proxy.ts

@@ -6,6 +6,7 @@ import { getEnv } from './lib/core/config/env'
 import { isAuthDisabled, isDev, isHosted } from './lib/core/config/env-flags'
 import { generateRuntimeCSP } from './lib/core/security/csp'
 import { getClientIp } from './lib/core/utils/request'
+import { isNonCanonicalSimHost } from './lib/core/utils/urls'
 
 const logger = createLogger('Proxy')
 
@@ -297,10 +298,31 @@ export async function proxy(request: NextRequest) {
   return track(request, response)
 }
 
+/**
+ * Keeps non-production sim.ai deployments out of search results.
+ *
+ * `noindex` rather than a robots.txt `Disallow` is deliberate: a disallowed URL
+ * can still be indexed when linked externally, and blocking the crawl stops
+ * search engines from ever seeing the directive that removes pages already in
+ * the index. robots.txt is excluded from this proxy's matcher so it keeps
+ * serving the crawlable rules this header depends on.
+ */
+function applyIndexingPolicy(request: NextRequest, response: NextResponse): void {
+  const host =
+    request.headers.get('x-forwarded-host')?.split(',')[0]?.trim() ||
+    request.headers.get('host') ||
+    request.nextUrl.host
+
+  if (isNonCanonicalSimHost(host)) {
+    response.headers.set('X-Robots-Tag', 'noindex, nofollow')
+  }
+}
+
 /**
  * Sends request data to Profound analytics (fire-and-forget) and returns the response.
  */
 function track(request: NextRequest, response: NextResponse): NextResponse {
+  applyIndexingPolicy(request, response)
   sendToProfound(request, response.status)
   return response
 }

packages/testing/src/mocks/urls.mock.ts

@@ -10,6 +10,9 @@ export const LOCALHOST_HOSTNAMES_MOCK: ReadonlySet<string> = new Set([
   '::1',
 ])
 
+/** Mirrors the real `CANONICAL_SITE_HOST` from `@/lib/core/utils/urls`. */
+export const CANONICAL_SITE_HOST_MOCK = 'www.sim.ai'
+
 const DEFAULT_SOCKET_URL = 'http://localhost:3002'
 const DEFAULT_OLLAMA_URL = 'http://localhost:11434'
 
@@ -72,6 +75,18 @@ function isLoopbackHostnameImpl(hostname: string): boolean {
   return LOCALHOST_HOSTNAMES_MOCK.has(hostname)
 }
 
+/** Mirrors the real `stripWwwPrefix` from `@/lib/core/utils/urls`. */
+function stripWwwPrefix(host: string): string {
+  return host.startsWith('www.') ? host.slice(4) : host
+}
+
+function isNonCanonicalSimHostImpl(host: string): boolean {
+  const first = host.split(',')[0]?.trim() ?? ''
+  const hostname = stripWwwPrefix(first.toLowerCase().split(':')[0])
+  const canonical = stripWwwPrefix(CANONICAL_SITE_HOST_MOCK)
+  return hostname !== canonical && hostname.endsWith(`.${canonical}`)
+}
+
 function parseOriginListImpl(
   raw: string | undefined | null,
   onInvalid?: (value: string) => void
@@ -156,6 +171,7 @@ export const urlsMockFns = {
   mockGetBaseDomain: vi.fn(getBaseDomainImpl),
   mockGetEmailDomain: vi.fn(getEmailDomainImpl),
   mockIsLoopbackHostname: vi.fn(isLoopbackHostnameImpl),
+  mockIsNonCanonicalSimHost: vi.fn(isNonCanonicalSimHostImpl),
   mockParseOriginList: vi.fn(parseOriginListImpl),
   mockIsLocalhostUrl: vi.fn(isLocalhostUrlImpl),
   mockGetBrowserOrigin: vi.fn(getBrowserOriginImpl),
@@ -176,6 +192,7 @@ export function resetUrlsMock(): void {
   urlsMockFns.mockGetBaseDomain.mockReset().mockImplementation(getBaseDomainImpl)
   urlsMockFns.mockGetEmailDomain.mockReset().mockImplementation(getEmailDomainImpl)
   urlsMockFns.mockIsLoopbackHostname.mockReset().mockImplementation(isLoopbackHostnameImpl)
+  urlsMockFns.mockIsNonCanonicalSimHost.mockReset().mockImplementation(isNonCanonicalSimHostImpl)
   urlsMockFns.mockParseOriginList.mockReset().mockImplementation(parseOriginListImpl)
   urlsMockFns.mockIsLocalhostUrl.mockReset().mockImplementation(isLocalhostUrlImpl)
   urlsMockFns.mockGetBrowserOrigin.mockReset().mockImplementation(getBrowserOriginImpl)
@@ -197,12 +214,14 @@ export function resetUrlsMock(): void {
 export const urlsMock = {
   SITE_URL: 'https://www.sim.ai',
   LOCALHOST_HOSTNAMES: LOCALHOST_HOSTNAMES_MOCK,
+  CANONICAL_SITE_HOST: CANONICAL_SITE_HOST_MOCK,
   getBaseUrl: urlsMockFns.mockGetBaseUrl,
   getInternalApiBaseUrl: urlsMockFns.mockGetInternalApiBaseUrl,
   ensureAbsoluteUrl: urlsMockFns.mockEnsureAbsoluteUrl,
   getBaseDomain: urlsMockFns.mockGetBaseDomain,
   getEmailDomain: urlsMockFns.mockGetEmailDomain,
   isLoopbackHostname: urlsMockFns.mockIsLoopbackHostname,
+  isNonCanonicalSimHost: urlsMockFns.mockIsNonCanonicalSimHost,
   parseOriginList: urlsMockFns.mockParseOriginList,
   isLocalhostUrl: urlsMockFns.mockIsLocalhostUrl,
   getBrowserOrigin: urlsMockFns.mockGetBrowserOrigin,