From 8bcbf403d5edd220a5564a708130480872b3c369 Mon Sep 17 00:00:00 2001 From: MaxP <7832832+mmplisskin@users.noreply.github.com> Date: Tue, 31 Mar 2026 01:43:58 -0700 Subject: [PATCH] feat: add OpenAI Compatible LLM provider for local models (#81) Support Ollama, LM Studio, vLLM, and any OpenAI-compatible API via a configurable base URL. Reuses ChatOpenAI with custom baseURL, no new dependencies. Local models use direct JSON parsing instead of withStructuredOutput since many don't support function calling. Co-authored-by: FasterOP <7832832+mmplisskin@useres.noreply.github.com> --- ai/providers/llmProvider.ts | 32 ++++++++++++++---- app/(app)/unsorted/page.tsx | 3 +- app/(auth)/actions.ts | 4 ++- app/(auth)/self-hosted/setup-form-client.tsx | 10 ++++++ components/settings/llm-settings-form.tsx | 29 ++++++++++++----- forms/settings.ts | 5 ++- lib/llm-providers.ts | 34 +++++++++++++++++++- models/settings.ts | 11 ++++++- 8 files changed, 109 insertions(+), 19 deletions(-) diff --git a/ai/providers/llmProvider.ts b/ai/providers/llmProvider.ts index ec56100..edf2895 100644 --- a/ai/providers/llmProvider.ts +++ b/ai/providers/llmProvider.ts @@ -3,12 +3,13 @@ import { ChatGoogleGenerativeAI } from "@langchain/google-genai" import { ChatMistralAI } from "@langchain/mistralai" import { BaseMessage, HumanMessage } from "@langchain/core/messages" -export type LLMProvider = "openai" | "google" | "mistral" +export type LLMProvider = "openai" | "google" | "mistral" | "openai_compatible" export interface LLMConfig { provider: LLMProvider apiKey: string model: string + baseUrl?: string } export interface LLMSettings { @@ -50,6 +51,15 @@ async function requestLLMUnified(config: LLMConfig, req: LLMRequest): Promise 0) { const images = req.attachments.map((att) => ({ @@ -72,7 +80,15 @@ async function requestLLMUnified(config: LLMConfig, req: LLMRequest): Promise c.text || "").join("") + response = JSON.parse(text.replace(/```(?:json)?\s*/g, "").trim()) + } else { + const structuredModel = model.withStructuredOutput(req.schema, { name: "transaction" }) + response = await structuredModel.invoke(messages) + } return { output: response, @@ -89,8 +105,12 @@ async function requestLLMUnified(config: LLMConfig, req: LLMRequest): Promise { for (const config of settings.providers) { - if (!config.apiKey || !config.model) { - console.info("Skipping provider:", config.provider) + if (!config.model) { + console.info("Skipping provider:", config.provider, "(no model)") + continue + } + if (config.provider === "openai_compatible" ? !config.baseUrl : !config.apiKey) { + console.info("Skipping provider:", config.provider, "(not configured)") continue } console.info("Use provider:", config.provider) diff --git a/app/(app)/unsorted/page.tsx b/app/(app)/unsorted/page.tsx index be57c51..554a843 100644 --- a/app/(app)/unsorted/page.tsx +++ b/app/(app)/unsorted/page.tsx @@ -41,7 +41,8 @@ export default async function UnsortedPage() { {config.selfHosted.isEnabled && !settings.openai_api_key && !settings.google_api_key && - !settings.mistral_api_key && ( + !settings.mistral_api_key && + !settings.openai_compatible_base_url && (
diff --git a/app/(auth)/actions.ts b/app/(auth)/actions.ts index 033f3c5..659570d 100644 --- a/app/(auth)/actions.ts +++ b/app/(auth)/actions.ts @@ -16,7 +16,9 @@ export async function selfHostedGetStartedAction(formData: FormData) { const apiKeys = [ "openai_api_key", "google_api_key", - "mistral_api_key" + "mistral_api_key", + "openai_compatible_api_key", + "openai_compatible_base_url", ] for (const key of apiKeys) { diff --git a/app/(auth)/self-hosted/setup-form-client.tsx b/app/(auth)/self-hosted/setup-form-client.tsx index d3aef62..2f925bd 100644 --- a/app/(auth)/self-hosted/setup-form-client.tsx +++ b/app/(auth)/self-hosted/setup-form-client.tsx @@ -68,6 +68,16 @@ export default function SelfHostedSetupFormClient({ defaultProvider, defaultApiK
+ {selected.baseUrlName && ( +
+ +
+ )} diff --git a/components/settings/llm-settings-form.tsx b/components/settings/llm-settings-form.tsx index c4ab6b0..961d379 100644 --- a/components/settings/llm-settings-form.tsx +++ b/components/settings/llm-settings-form.tsx @@ -30,7 +30,7 @@ import { PROVIDERS } from "@/lib/llm-providers"; function getInitialProviderOrder(settings: Record) { let order: string[] = [] if (!settings.llm_providers) { - order = ['openai', 'google', 'mistral'] + order = ['openai', 'google', 'mistral', 'openai_compatible'] } else { order = settings.llm_providers.split(",").map(p => p.trim()) } @@ -51,17 +51,20 @@ export default function LLMSettingsForm({ // Controlled values for each provider const [providerValues, setProviderValues] = useState(() => { - const values: Record = {} + const values: Record = {} PROVIDERS.forEach((provider) => { values[provider.key] = { apiKey: settings[provider.apiKeyName], model: settings[provider.modelName] || provider.defaultModelName, + baseUrl: provider.baseUrlName + ? (settings[provider.baseUrlName] || provider.defaultBaseUrl || "") + : "", } }) return values }) - function handleProviderValueChange(providerKey: string, field: "apiKey" | "model", value: string) { + function handleProviderValueChange(providerKey: string, field: "apiKey" | "model" | "baseUrl", value: string) { setProviderValues((prev) => ({ ...prev, [providerKey]: { @@ -141,8 +144,8 @@ export default function LLMSettingsForm({ type DndProviderBlocksProps = { providerOrder: string[]; setProviderOrder: React.Dispatch>; - providerValues: Record; - handleProviderValueChange: (providerKey: string, field: "apiKey" | "model", value: string) => void; + providerValues: Record; + handleProviderValueChange: (providerKey: string, field: "apiKey" | "model" | "baseUrl", value: string) => void; }; function DndProviderBlocks({ providerOrder, setProviderOrder, providerValues, handleProviderValueChange }: DndProviderBlocksProps) { @@ -176,8 +179,8 @@ type SortableProviderBlockProps = { id: string; idx: number; providerKey: string; - value: { apiKey: string; model: string }; - handleValueChange: (providerKey: string, field: "apiKey" | "model", value: string) => void; + value: { apiKey: string; model: string; baseUrl: string }; + handleValueChange: (providerKey: string, field: "apiKey" | "model" | "baseUrl", value: string) => void; }; function SortableProviderBlock({ id, idx, providerKey, value, handleValueChange }: SortableProviderBlockProps) { @@ -214,7 +217,7 @@ function SortableProviderBlock({ id, idx, providerKey, value, handleValueChange value={value.apiKey} onChange={e => handleValueChange(provider.key, "apiKey", e.target.value)} className="flex-1 border rounded px-2 py-1" - placeholder="API key" + placeholder={provider.baseUrlName ? "API key (optional)" : "API key"} /> + {provider.baseUrlName && ( + handleValueChange(provider.key, "baseUrl", e.target.value)} + className="w-full border rounded px-2 py-1" + placeholder="Base URL (e.g. http://localhost:11434/v1)" + /> + )} {provider.apiDoc && ( Get your API key from{" "} diff --git a/forms/settings.ts b/forms/settings.ts index 08ab599..a8e8eca 100644 --- a/forms/settings.ts +++ b/forms/settings.ts @@ -12,7 +12,10 @@ export const settingsFormSchema = z.object({ google_model_name: z.string().default("gemini-2.5-flash"), mistral_api_key: z.string().optional(), mistral_model_name: z.string().default("mistral-medium-latest"), - llm_providers: z.string().default('openai,google,mistral'), + openai_compatible_api_key: z.string().optional(), + openai_compatible_model_name: z.string().optional(), + openai_compatible_base_url: z.string().optional(), + llm_providers: z.string().default('openai,google,mistral,openai_compatible'), prompt_analyse_new_file: z.string().optional(), is_welcome_message_hidden: z.string().optional(), }) diff --git a/lib/llm-providers.ts b/lib/llm-providers.ts index 8ffa670..cf9fc01 100644 --- a/lib/llm-providers.ts +++ b/lib/llm-providers.ts @@ -1,4 +1,19 @@ -export const PROVIDERS = [ +export interface ProviderMeta { + key: string + label: string + apiKeyName: string + modelName: string + defaultModelName: string + baseUrlName?: string + defaultBaseUrl?: string + apiDoc: string + apiDocLabel: string + placeholder: string + help: { url: string; label: string } + logo: string +} + +export const PROVIDERS: ProviderMeta[] = [ { key: "openai", label: "OpenAI", @@ -44,4 +59,21 @@ export const PROVIDERS = [ }, logo: "/logo/mistral.svg" }, + { + key: "openai_compatible", + label: "OpenAI Compatible", + apiKeyName: "openai_compatible_api_key", + modelName: "openai_compatible_model_name", + defaultModelName: "", + baseUrlName: "openai_compatible_base_url", + defaultBaseUrl: "http://localhost:11434/v1", + apiDoc: "", + apiDocLabel: "", + placeholder: "(optional)", + help: { + url: "https://github.com/ollama/ollama/blob/main/docs/openai.md", + label: "Works with Ollama, LM Studio, vLLM, LocalAI" + }, + logo: "/logo/openai.svg" + }, ] diff --git a/models/settings.ts b/models/settings.ts index e4cd71f..245f616 100644 --- a/models/settings.ts +++ b/models/settings.ts @@ -9,7 +9,7 @@ export type SettingsMap = Record * Helper to extract LLM provider settings from SettingsMap. */ export function getLLMSettings(settings: SettingsMap) { - const priorities = (settings.llm_providers || "openai,google,mistral").split(",").map(p => p.trim()).filter(Boolean) + const priorities = (settings.llm_providers || "openai,google,mistral,openai_compatible").split(",").map(p => p.trim()).filter(Boolean) const providers = priorities.map((provider) => { if (provider === "openai") { @@ -33,6 +33,15 @@ export function getLLMSettings(settings: SettingsMap) { model: settings.mistral_model_name || PROVIDERS[2]['defaultModelName'], } } + if (provider === "openai_compatible") { + const providerMeta = PROVIDERS.find(p => p.key === "openai_compatible") + return { + provider: provider as LLMProvider, + apiKey: settings.openai_compatible_api_key || "", + model: settings.openai_compatible_model_name || "", + baseUrl: settings.openai_compatible_base_url || providerMeta?.defaultBaseUrl || "", + } + } return null }).filter((provider): provider is NonNullable => provider !== null)