From ae6204fbd7ff311f3619656cee1ec6184442ccb3 Mon Sep 17 00:00:00 2001
From: Morgan Dean <morgandeanhi@gmail.com>
Date: Thu, 10 Jul 2025 16:51:15 -0700
Subject: [PATCH] Setup llms.txt & *.mdx routes

---
 docs/next.config.mjs                       |  9 +++++++-
 docs/package.json                          |  5 ++++-
 docs/pnpm-lock.yaml                        |  9 ++++++++
 docs/src/app/llms.mdx/[[...slug]]/route.ts | 21 +++++++++++++++++
 docs/src/app/llms.txt/route.ts             | 12 ++++++++++
 docs/src/lib/llms.ts                       | 26 ++++++++++++++++++++++
 6 files changed, 80 insertions(+), 2 deletions(-)
 create mode 100644 docs/src/app/llms.mdx/[[...slug]]/route.ts
 create mode 100644 docs/src/app/llms.txt/route.ts
 create mode 100644 docs/src/lib/llms.ts

diff --git a/docs/next.config.mjs b/docs/next.config.mjs
index 5d9846df..8f8a29ac 100644
--- a/docs/next.config.mjs
+++ b/docs/next.config.mjs
@@ -5,7 +5,14 @@ const withMDX = createMDX();
 /** @type {import('next').NextConfig} */
 const config = {
   reactStrictMode: true,
-
+  async rewrites() {
+    return [
+      {
+        source: '/home/:path*.mdx',
+        destination: '/llms.mdx/:path*',
+      },
+    ];
+  },
   images: {
     dangerouslyAllowSVG: true,
     remotePatterns: [
diff --git a/docs/package.json b/docs/package.json
index eec9850b..c6c083c9 100644
--- a/docs/package.json
+++ b/docs/package.json
@@ -18,17 +18,20 @@
     "next-themes": "^0.4.6",
     "react": "^19.1.0",
     "react-dom": "^19.1.0",
+    "remark": "^15.0.1",
+    "remark-gfm": "^4.0.1",
+    "remark-mdx": "^3.1.0",
     "tailwind-merge": "^3.3.1",
     "zod": "^3.25.76"
   },
   "devDependencies": {
-    "prettier": "^3.6.2",
     "@tailwindcss/postcss": "^4.1.8",
     "@types/mdx": "^2.0.13",
     "@types/node": "22.15.28",
     "@types/react": "^19.1.6",
     "@types/react-dom": "^19.1.5",
     "postcss": "^8.5.4",
+    "prettier": "^3.6.2",
     "tailwindcss": "^4.1.8",
     "typescript": "^5.8.3"
   },
diff --git a/docs/pnpm-lock.yaml b/docs/pnpm-lock.yaml
index e91224b1..21945e6b 100644
--- a/docs/pnpm-lock.yaml
+++ b/docs/pnpm-lock.yaml
@@ -35,6 +35,15 @@ importers:
       react-dom:
         specifier: ^19.1.0
         version: 19.1.0(react@19.1.0)
+      remark:
+        specifier: ^15.0.1
+        version: 15.0.1
+      remark-gfm:
+        specifier: ^4.0.1
+        version: 4.0.1
+      remark-mdx:
+        specifier: ^3.1.0
+        version: 3.1.0
       tailwind-merge:
         specifier: ^3.3.1
         version: 3.3.1
diff --git a/docs/src/app/llms.mdx/[[...slug]]/route.ts b/docs/src/app/llms.mdx/[[...slug]]/route.ts
new file mode 100644
index 00000000..45079884
--- /dev/null
+++ b/docs/src/app/llms.mdx/[[...slug]]/route.ts
@@ -0,0 +1,21 @@
+import { type NextRequest, NextResponse } from 'next/server';
+import { getLLMText } from '@/lib/llms';
+import { source } from '@/lib/source';
+import { notFound } from 'next/navigation';
+
+export const revalidate = false;
+
+export async function GET(
+  _req: NextRequest,
+  { params }: { params: Promise<{ slug?: string[] }> }
+) {
+  const { slug } = await params;
+  const page = source.getPage(['home', ...slug!]);
+  if (!page) notFound();
+
+  return new NextResponse(await getLLMText(page));
+}
+
+export function generateStaticParams() {
+  return source.generateParams();
+}
diff --git a/docs/src/app/llms.txt/route.ts b/docs/src/app/llms.txt/route.ts
new file mode 100644
index 00000000..5c913bf1
--- /dev/null
+++ b/docs/src/app/llms.txt/route.ts
@@ -0,0 +1,12 @@
+import { source } from '@/lib/source';
+import { getLLMText } from '@/lib/llms';
+
+// cached forever
+export const revalidate = false;
+
+export async function GET() {
+  const scan = source.getPages().map(getLLMText);
+  const scanned = await Promise.all(scan);
+
+  return new Response(scanned.join('\n\n'));
+}
diff --git a/docs/src/lib/llms.ts b/docs/src/lib/llms.ts
new file mode 100644
index 00000000..e485bed2
--- /dev/null
+++ b/docs/src/lib/llms.ts
@@ -0,0 +1,26 @@
+import { remark } from 'remark';
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+import { remarkInclude } from 'fumadocs-mdx/config';
+import { source } from '@/lib/source';
+import type { InferPageType } from 'fumadocs-core/source';
+
+const processor = remark()
+  .use(remarkMdx)
+  // needed for Fumadocs MDX
+  .use(remarkInclude)
+  .use(remarkGfm);
+
+export async function getLLMText(page: InferPageType<typeof source>) {
+  const processed = await processor.process({
+    path: page.data._file.absolutePath,
+    value: page.data.content,
+  });
+
+  return `# ${page.data.title}
+URL: ${page.url}
+
+${page.data.description}
+
+${processed.value}`;
+}