fix: add back sync docs script (#2123)

frontend/app/src/next/lib/docs/generated/_meta.ts

@@ -0,0 +1,58 @@
+// Generated from frontend/docs/pages/_meta.js
+const meta = {
+  home: {
+    title: 'User Guide',
+    type: 'page',
+    theme: {
+      toc: false,
+    },
+  },
+  _setup: {
+    display: 'hidden',
+  },
+  compute: {
+    title: 'Managed Compute',
+    type: 'page',
+    href: '/home/compute',
+    index: 'Overview',
+    'getting-started': 'Getting Started',
+    cpu: 'CPU Machine Types',
+    gpu: 'GPU Machine Types',
+  },
+  self_hosting: {
+    title: 'Self Hosting',
+    type: 'page',
+    theme: {
+      toc: false,
+    },
+  },
+  blog: {
+    title: 'Blog',
+    type: 'page',
+  },
+  contributing: {
+    title: 'Contributing',
+    type: 'page',
+    display: 'hidden',
+    theme: {
+      toc: false,
+    },
+  },
+  sdks: {
+    title: 'SDK Reference',
+    type: 'menu',
+    items: {
+      python: {
+        title: 'Python',
+        href: '/sdks/python/client',
+        type: 'page',
+      },
+    },
+  },
+  v0: {
+    title: 'V0 (Old docs)',
+    type: 'page',
+    href: 'https://v0-docs.hatchet.run',
+  },
+};
+export default meta;

frontend/app/src/next/lib/docs/generated/blog/_meta.ts

@@ -0,0 +1,49 @@
+// Generated from frontend/docs/pages/blog/_meta.js
+const meta = {
+  background_tasks_fastapi_hatchet: {
+    title: 'Background Tasks: From FastAPI to Hatchet',
+    href: '/blog/background-tasks-fastapi-hatchet',
+  },
+  go_agents: {
+    title: 'Why Go is a good fit for agents',
+    href: '/blog/go-agents',
+  },
+  warning_event_loop_blocked: {
+    title: 'Warning! The Event Loop May Be Blocked',
+    href: '/blog/warning-event-loop-blocked',
+  },
+  fastest_postgres_inserts: {
+    title: 'The fastest Postgres inserts',
+    href: '/blog/fastest-postgres-inserts',
+  },
+  task_queue_modern_python: {
+    title: 'A task queue for modern Python applications',
+    href: '/blog/task-queue-modern-python',
+  },
+  postgres_events_table: {
+    title: 'Use Postgres for your events table',
+    href: '/blog/postgres-events-table',
+  },
+  migrating_off_prisma: {
+    title: 'Why we moved off Prisma',
+    href: '/blog/migrating-off-prisma',
+  },
+  problems_with_celery: {
+    title: 'The problems with Celery',
+    display: 'hidden',
+    href: '/blog/problems-with-celery',
+  },
+  multi_tenant_queues: {
+    title: 'An unfair advantage: multi-tenant queues in Postgres',
+    href: '/blog/multi-tenant-queues',
+  },
+  '--migration-guides': {
+    title: 'Migration Guides',
+    type: 'separator',
+  },
+  mergent_migration_guide: {
+    title: 'Migrating from Mergent',
+    href: '/blog/mergent-migration-guide',
+  },
+};
+export default meta;

frontend/app/src/next/lib/docs/generated/contributing/_meta.ts

@@ -0,0 +1,16 @@
+// Generated from frontend/docs/pages/contributing/_meta.js
+const meta = {
+  index: {
+    title: 'Contributing',
+    href: '/contributing/',
+  },
+  github_app_setup: {
+    title: 'GitHub App Setup',
+    href: '/contributing/github-app-setup',
+  },
+  sdks: {
+    title: 'SDKs',
+    href: '/contributing/sdks',
+  },
+};
+export default meta;

frontend/app/src/next/lib/docs/generated/home/compute/_meta.ts

@@ -0,0 +1,32 @@
+// Generated from frontend/docs/pages/home/compute/_meta.js
+const meta = {
+  index: {
+    title: 'Overview',
+    href: '/home/compute/',
+  },
+  getting_started: {
+    title: 'Getting Started',
+    href: '/home/compute/getting-started',
+  },
+  cpu: {
+    title: 'CPU Machine Types',
+    href: '/home/compute/cpu',
+  },
+  gpu: {
+    title: 'GPU Machine Types',
+    href: '/home/compute/gpu',
+  },
+  git_ops: {
+    title: 'GitOps',
+    href: '/home/compute/git-ops',
+  },
+  auto_scaling: {
+    title: 'Auto Scaling',
+    href: '/home/compute/auto-scaling',
+  },
+  environment_variables: {
+    title: 'Environment Variables',
+    href: '/home/compute/environment-variables',
+  },
+};
+export default meta;

frontend/app/src/next/lib/docs/generated/index.ts

@@ -0,0 +1,21 @@
+// Generated index file for meta-data
+import root from './_meta';
+import blog from './blog/_meta';
+import contributing from './contributing/_meta';
+import home from './home/_meta';
+import homecompute from './home/compute/_meta';
+import sdks from './sdks/_meta';
+import sdkspython from './sdks/python/_meta';
+import sdkspythonfeature_clients from './sdks/python/feature-clients/_meta';
+import self_hosting from './self-hosting/_meta';
+export {
+  root,
+  blog,
+  contributing,
+  home,
+  homecompute,
+  sdks,
+  sdkspython,
+  sdkspythonfeature_clients,
+  self_hosting,
+};

frontend/app/src/next/lib/docs/generated/sdks/_meta.ts

@@ -0,0 +1,11 @@
+// Generated from frontend/docs/pages/sdks/_meta.js
+const meta = {
+  python: {
+    title: 'Python SDK',
+    type: 'page',
+    theme: {
+      toc: true,
+    },
+  },
+};
+export default meta;

frontend/app/src/next/lib/docs/generated/sdks/python/_meta.ts

@@ -0,0 +1,32 @@
+// Generated from frontend/docs/pages/sdks/python/_meta.js
+const meta = {
+  client: {
+    title: 'Client',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/client',
+  },
+  context: {
+    title: 'Context',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/context',
+  },
+  feature_clients: {
+    title: 'Feature Clients',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients',
+  },
+  runnables: {
+    title: 'Runnables',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/runnables',
+  },
+};
+export default meta;

frontend/app/src/next/lib/docs/generated/sdks/python/feature-clients/_meta.ts

@@ -0,0 +1,67 @@
+// Generated from frontend/docs/pages/sdks/python/feature-clients/_meta.js
+const meta = {
+  cron: {
+    title: 'Cron',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients/cron',
+  },
+  filters: {
+    title: 'Filters',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients/filters',
+  },
+  logs: {
+    title: 'Logs',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients/logs',
+  },
+  metrics: {
+    title: 'Metrics',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients/metrics',
+  },
+  rate_limits: {
+    title: 'Rate Limits',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients/rate_limits',
+  },
+  runs: {
+    title: 'Runs',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients/runs',
+  },
+  scheduled: {
+    title: 'Scheduled',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients/scheduled',
+  },
+  workers: {
+    title: 'Workers',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients/workers',
+  },
+  workflows: {
+    title: 'Workflows',
+    theme: {
+      toc: true,
+    },
+    href: '/sdks/python/feature-clients/workflows',
+  },
+};
+export default meta;

frontend/app/src/next/lib/docs/generated/self-hosting/_meta.ts

@@ -0,0 +1,87 @@
+// Generated from frontend/docs/pages/self-hosting/_meta.js
+const meta = {
+  index: {
+    title: 'Introduction',
+    href: '/self-hosting/',
+  },
+  '-- Docker': {
+    type: 'separator',
+    title: 'Docker',
+  },
+  hatchet_lite: {
+    title: 'Hatchet Lite',
+    href: '/self-hosting/hatchet-lite',
+  },
+  docker_compose: {
+    title: 'Docker Compose',
+    href: '/self-hosting/docker-compose',
+  },
+  '-- Kubernetes': {
+    type: 'separator',
+    title: 'Kubernetes',
+  },
+  kubernetes_quickstart: {
+    title: 'Quickstart',
+    href: '/self-hosting/kubernetes-quickstart',
+  },
+  kubernetes_glasskube: {
+    title: 'Installing with Glasskube',
+    href: '/self-hosting/kubernetes-glasskube',
+  },
+  networking: {
+    title: 'Networking',
+    href: '/self-hosting/networking',
+  },
+  kubernetes_helm_configuration: {
+    title: 'Configuring the Helm Chart',
+    href: '/self-hosting/kubernetes-helm-configuration',
+  },
+  kubernetes_external_database: {
+    title: 'Setting up an External Database',
+    href: '/self-hosting/kubernetes-external-database',
+  },
+  high_availability: {
+    title: 'High Availability',
+    href: '/self-hosting/high-availability',
+  },
+  '-- Managing Hatchet': {
+    type: 'separator',
+    title: 'Managing Hatchet',
+  },
+  configuration_options: {
+    title: 'Engine Configuration Options',
+    href: '/self-hosting/configuration-options',
+  },
+  prometheus_metrics: {
+    title: 'Prometheus Metrics',
+    theme: {
+      toc: true,
+    },
+    href: '/self-hosting/prometheus-metrics',
+  },
+  worker_configuration_options: {
+    title: 'Worker Configuration Options',
+    href: '/self-hosting/worker-configuration-options',
+  },
+  benchmarking: {
+    title: 'Benchmarking',
+    href: '/self-hosting/benchmarking',
+  },
+  data_retention: {
+    title: 'Data Retention',
+    href: '/self-hosting/data-retention',
+  },
+  improving_performance: {
+    title: 'Improving Performance',
+    href: '/self-hosting/improving-performance',
+  },
+  read_replicas: {
+    title: 'Read Replicas',
+    href: '/self-hosting/read-replicas',
+  },
+  sampling: {
+    title: 'Trace Sampling',
+    href: '/self-hosting/sampling',
+  },
+};
+export default meta;

frontend/app/src/next/lib/docs/snips.ts

@@ -0,0 +1,5 @@
+import * as snippets from './generated/snips/index';
+import { Snippet as SnippetType } from './generated/snips/types';
+
+export type Snippet = SnippetType;
+export default snippets;

frontend/app/src/next/lib/docs/sync-docs.ts

@@ -0,0 +1,332 @@
+#!/usr/bin/env node
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+// Get the directory paths
+const dirname = path.dirname(fileURLToPath(import.meta.url));
+const docsDir = path.resolve(dirname, '../../../../../docs/pages');
+const generatedDir = path.resolve(dirname, 'generated');
+
+// Make sure the generated directory exists
+if (!fs.existsSync(generatedDir)) {
+  fs.mkdirSync(generatedDir, { recursive: true });
+}
+
+// Keep track of found meta files for generating the index
+const metaFiles: { importName: string; importPath: string }[] = [];
+
+// Function to process a directory recursively
+function processDirectory(dirPath: string, relativePath: string = '') {
+  const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+
+  // First, find if there's a _meta.js file in this directory
+  const metaFile = entries.find(
+    (entry) => entry.isFile() && entry.name === '_meta.js',
+  );
+
+  if (metaFile) {
+    // Determine the target directory path
+    const targetDirPath = path.join(generatedDir, relativePath);
+
+    // Create the target directory if it doesn't exist
+    if (!fs.existsSync(targetDirPath)) {
+      fs.mkdirSync(targetDirPath, { recursive: true });
+    }
+
+    // Read the meta file
+    const metaFilePath = path.join(dirPath, metaFile.name);
+    const metaContent = fs.readFileSync(metaFilePath, 'utf8');
+
+    try {
+      // Build a dictionary from the meta.js file
+      const metaObject = parseMetaFile(metaContent);
+
+      // Process the object to ensure all entries have title and href
+      const processedObj = processMetaObject(metaObject, relativePath);
+
+      // Convert to well-formatted string
+      const formattedStr = formatObject(processedObj);
+      const formattedPath = metaFilePath.replace(/^.*?(frontend)/i, '$1');
+
+      // Create a TypeScript version with the proper format
+      const tsContent = `// Generated from ${formattedPath}
+const meta = ${formattedStr};
+export default meta;
+`;
+
+      // Write the TypeScript file
+      const targetFilePath = path.join(targetDirPath, '_meta.ts');
+      fs.writeFileSync(targetFilePath, tsContent, 'utf8');
+
+      // Add to the list of meta files for the index
+      let importName = relativePath.replace(/\//g, '') || 'root';
+
+      // Handle nested paths by creating camelCase names
+      if (importName !== 'root') {
+        importName = importName
+          .split('/')
+          .map((part, index) => {
+            if (index === 0) {
+              return part;
+            }
+            return part.charAt(0).toUpperCase() + part.slice(1);
+          })
+          .join('');
+      }
+
+      metaFiles.push({
+        importName: importName.replace(/-/g, '_'),
+        importPath: `./${relativePath ? relativePath + '/' : ''}_meta`,
+      });
+    } catch (err) {
+      console.error(`Error processing ${metaFilePath}:`, err);
+      const formattedPath = metaFilePath.replace(/^.*?(frontend)/i, '$1');
+
+      // Create a TypeScript version with the proper format
+      const tsContent = `// Generated from ${formattedPath}
+const meta = ${metaContent.replace('export default', '')};
+export default meta;
+`;
+      const targetFilePath = path.join(targetDirPath, '_meta.ts');
+      fs.writeFileSync(targetFilePath, tsContent, 'utf8');
+    }
+  }
+
+  // Process subdirectories
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      processDirectory(
+        path.join(dirPath, entry.name),
+        relativePath ? path.join(relativePath, entry.name) : entry.name,
+      );
+    }
+  }
+}
+
+// Function to parse a meta.js file into a dictionary
+function parseMetaFile(content: string): Record<string, any> {
+  // Remove the "export default" and trailing semicolon
+  const objectStr = content
+    .replace(/export\s+default\s*/, '')
+    .trim()
+    .replace(/;$/, '');
+
+  // Parse the object structure
+  const result: Record<string, any> = {};
+
+  // Simple regex-based parser for the object structure
+  // Extract key-value pairs from the object
+  let depth = 0;
+  let inString = false;
+  let stringDelimiter = '';
+
+  // Skip the first opening brace and last closing brace
+  const objContent = objectStr
+    .substring(objectStr.indexOf('{') + 1, objectStr.lastIndexOf('}'))
+    .trim();
+
+  // Split by commas that are not inside nested objects or strings
+  const entries: string[] = [];
+  let currentEntry = '';
+
+  for (let i = 0; i < objContent.length; i++) {
+    const char = objContent[i];
+
+    if (char === '{') {
+      depth++;
+    }
+    if (char === '}') {
+      depth--;
+    }
+
+    if (char === '"' || char === "'") {
+      if (!inString) {
+        inString = true;
+        stringDelimiter = char;
+      } else if (char === stringDelimiter && objContent[i - 1] !== '\\') {
+        inString = false;
+      }
+    }
+
+    if (char === ',' && depth === 0 && !inString) {
+      entries.push(currentEntry.trim());
+      currentEntry = '';
+    } else {
+      currentEntry += char;
+    }
+  }
+
+  if (currentEntry.trim()) {
+    entries.push(currentEntry.trim());
+  }
+
+  // Process each entry
+  for (const entry of entries) {
+    // Split by the first colon not in a string
+    let colonIndex = -1;
+    inString = false;
+    stringDelimiter = '';
+
+    for (let i = 0; i < entry.length; i++) {
+      const char = entry[i];
+
+      if (char === '"' || char === "'") {
+        if (!inString) {
+          inString = true;
+          stringDelimiter = char;
+        } else if (char === stringDelimiter && entry[i - 1] !== '\\') {
+          inString = false;
+        }
+      }
+
+      if (char === ':' && !inString && colonIndex === -1) {
+        colonIndex = i;
+        break;
+      }
+    }
+
+    if (colonIndex === -1) {
+      continue;
+    }
+
+    const keyPart = entry.substring(0, colonIndex).trim();
+    const valuePart = entry.substring(colonIndex + 1).trim();
+
+    // Extract the key (remove quotes if present)
+    let key = keyPart;
+    if (
+      (key.startsWith('"') && key.endsWith('"')) ||
+      (key.startsWith("'") && key.endsWith("'"))
+    ) {
+      key = key.substring(1, key.length - 1);
+    }
+
+    // Parse the value
+    let value: any;
+
+    if (valuePart === 'true') {
+      value = true;
+    } else if (valuePart === 'false') {
+      value = false;
+    } else if (valuePart === 'null') {
+      value = null;
+    } else if (valuePart === 'undefined') {
+      value = undefined;
+    } else if (valuePart.startsWith('{') && valuePart.endsWith('}')) {
+      // Nested object - recursively parse
+      value = parseMetaFile(`export default ${valuePart}`);
+    } else if (
+      (valuePart.startsWith('"') && valuePart.endsWith('"')) ||
+      (valuePart.startsWith("'") && valuePart.endsWith("'"))
+    ) {
+      // String value
+      value = valuePart.substring(1, valuePart.length - 1);
+    } else if (!isNaN(Number(valuePart))) {
+      // Number value
+      value = Number(valuePart);
+    } else {
+      // Unknown/complex value - keep as string
+      value = valuePart;
+    }
+
+    result[key] = value;
+  }
+
+  return result;
+}
+
+// Function to format an object as a string
+function formatObject(obj: Record<string, any>, indent = 0): string {
+  const spaces = ' '.repeat(indent);
+  let result = '{\n';
+
+  for (const [key, value] of Object.entries(obj)) {
+    // Format the key - add quotes for keys with special characters
+    const formattedKey = /^[a-zA-Z0-9_]+$/.test(key) ? key : `'${key}'`;
+
+    // Format the value based on type
+    let formattedValue: string;
+
+    if (value === null) {
+      formattedValue = 'null';
+    } else if (typeof value === 'object') {
+      formattedValue = formatObject(value, indent + 2);
+    } else if (typeof value === 'string') {
+      formattedValue = `'${value.replace(/'/g, "\\'")}'`;
+    } else {
+      formattedValue = String(value);
+    }
+
+    result += `${spaces}  ${formattedKey}: ${formattedValue},\n`;
+  }
+
+  if (result.endsWith(',\n')) {
+    result = result.slice(0, -2) + '\n';
+  }
+
+  result += `${spaces}}`;
+  return result;
+}
+
+// Function to recursively process the meta object
+function processMetaObject(
+  obj: Record<string, any>,
+  relativePath: string,
+): Record<string, any> {
+  const result: Record<string, any> = {};
+
+  // Process each key in the object
+  for (const [key, value] of Object.entries(obj)) {
+    // Sanitize the key - replace hyphens with underscores, except for --prefixed keys
+    const sanitizedKey = key.startsWith('--') ? key : key.replace(/-/g, '_');
+
+    // Skip --prefixed entries (but preserve them)
+    if (key.startsWith('--')) {
+      result[sanitizedKey] = value;
+      continue;
+    }
+
+    if (typeof value === 'string') {
+      // Convert string values to objects with title and href
+      result[sanitizedKey] = {
+        title: value,
+        href:
+          key === 'index'
+            ? `/${relativePath ? relativePath + '/' : ''}`
+            : `/${relativePath ? relativePath + '/' : ''}${key}`, // Keep original key for href
+      };
+    } else if (typeof value === 'object' && value !== null) {
+      // Already an object, make sure it has href if it has title
+      if (value.title && !value.href && !value.type) {
+        result[sanitizedKey] = {
+          ...value,
+          href: `/${relativePath ? relativePath + '/' : ''}${key}`, // Keep original key for href
+        };
+      } else {
+        // Just keep the original object
+        result[sanitizedKey] = value;
+      }
+    } else {
+      // For any other type, just pass it through
+      result[sanitizedKey] = value;
+    }
+  }
+
+  return result;
+}
+
+// Start processing from the root docs directory
+processDirectory(docsDir);
+
+// Generate the index.ts file
+const indexContent = `// Generated index file for meta-data
+${metaFiles.map((file) => `import ${file.importName} from '${file.importPath}';`).join('\n')}
+export { ${metaFiles.map((file) => file.importName).join(', ')} };
+`;
+
+// Write the index file
+fs.writeFileSync(path.join(generatedDir, 'index.ts'), indexContent, 'utf8');
+
+// eslint-disable-next-line no-console
+console.log(`Generated ${metaFiles.length} meta files in ${generatedDir}`);