From b9a25b16e665a15fdde9d46131e865b861087d01 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 6 Aug 2025 21:56:05 +0200 Subject: [PATCH] feat: add reasoning effort and metadata to template (#5981) Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/openai/chat.go | 2 +- core/http/endpoints/openai/completion.go | 12 ++++++++---- core/http/endpoints/openai/edit.go | 8 +++++--- core/schema/openai.go | 4 ++++ core/templates/evaluator.go | 6 +++++- core/templates/evaluator_test.go | 6 +++--- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index f43254c15..0d96e04cd 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -305,7 +305,7 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, evaluat // If we are using the tokenizer template, we don't need to process the messages // unless we are processing functions if !config.TemplateConfig.UseTokenizerTemplate || shouldUseFn { - predInput = evaluator.TemplateMessages(input.Messages, config, funcs, shouldUseFn) + predInput = evaluator.TemplateMessages(*input, input.Messages, config, funcs, shouldUseFn) log.Debug().Msgf("Prompt (after templating): %s", predInput) if config.Grammar != "" { diff --git a/core/http/endpoints/openai/completion.go b/core/http/endpoints/openai/completion.go index 7dec8ca7e..654166a1c 100644 --- a/core/http/endpoints/openai/completion.go +++ b/core/http/endpoints/openai/completion.go @@ -109,8 +109,10 @@ func CompletionEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, e predInput := config.PromptStrings[0] templatedInput, err := evaluator.EvaluateTemplateForPrompt(templates.CompletionPromptTemplate, *config, templates.PromptTemplateData{ - Input: predInput, - SystemPrompt: config.SystemPrompt, + Input: predInput, + SystemPrompt: config.SystemPrompt, + ReasoningEffort: input.ReasoningEffort, + Metadata: input.Metadata, }) if err == nil { predInput = templatedInput @@ -160,8 +162,10 @@ func CompletionEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, e for k, i := range config.PromptStrings { templatedInput, err := evaluator.EvaluateTemplateForPrompt(templates.CompletionPromptTemplate, *config, templates.PromptTemplateData{ - SystemPrompt: config.SystemPrompt, - Input: i, + SystemPrompt: config.SystemPrompt, + Input: i, + ReasoningEffort: input.ReasoningEffort, + Metadata: input.Metadata, }) if err == nil { i = templatedInput diff --git a/core/http/endpoints/openai/edit.go b/core/http/endpoints/openai/edit.go index 5b29b1c9c..fbcd398d2 100644 --- a/core/http/endpoints/openai/edit.go +++ b/core/http/endpoints/openai/edit.go @@ -47,9 +47,11 @@ func EditEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, evaluat for _, i := range config.InputStrings { templatedInput, err := evaluator.EvaluateTemplateForPrompt(templates.EditPromptTemplate, *config, templates.PromptTemplateData{ - Input: i, - Instruction: input.Instruction, - SystemPrompt: config.SystemPrompt, + Input: i, + Instruction: input.Instruction, + SystemPrompt: config.SystemPrompt, + ReasoningEffort: input.ReasoningEffort, + Metadata: input.Metadata, }) if err == nil { i = templatedInput diff --git a/core/schema/openai.go b/core/schema/openai.go index 44b54d188..5506231e5 100644 --- a/core/schema/openai.go +++ b/core/schema/openai.go @@ -183,6 +183,10 @@ type OpenAIRequest struct { Backend string `json:"backend" yaml:"backend"` ModelBaseName string `json:"model_base_name" yaml:"model_base_name"` + + ReasoningEffort string `json:"reasoning_effort" yaml:"reasoning_effort"` + + Metadata map[string]string `json:"metadata" yaml:"metadata"` } type ModelsDataResponse struct { diff --git a/core/templates/evaluator.go b/core/templates/evaluator.go index 78de7582e..f9bd313af 100644 --- a/core/templates/evaluator.go +++ b/core/templates/evaluator.go @@ -21,6 +21,8 @@ type PromptTemplateData struct { Instruction string Functions []functions.Function MessageIndex int + ReasoningEffort string + Metadata map[string]string } type ChatMessageTemplateData struct { @@ -133,7 +135,7 @@ func (e *Evaluator) evaluateJinjaTemplateForPrompt(templateType TemplateType, te return e.cache.evaluateJinjaTemplate(templateType, templateName, conversation) } -func (e *Evaluator) TemplateMessages(messages []schema.Message, config *config.BackendConfig, funcs []functions.Function, shouldUseFn bool) string { +func (e *Evaluator) TemplateMessages(input schema.OpenAIRequest, messages []schema.Message, config *config.BackendConfig, funcs []functions.Function, shouldUseFn bool) string { if config.TemplateConfig.JinjaTemplate { var messageData []ChatMessageTemplateData @@ -283,6 +285,8 @@ func (e *Evaluator) TemplateMessages(messages []schema.Message, config *config.B SuppressSystemPrompt: suppressConfigSystemPrompt, Input: predInput, Functions: funcs, + ReasoningEffort: input.ReasoningEffort, + Metadata: input.Metadata, }) if err == nil { predInput = templatedInput diff --git a/core/templates/evaluator_test.go b/core/templates/evaluator_test.go index 04ef48d6e..41c17e6a5 100644 --- a/core/templates/evaluator_test.go +++ b/core/templates/evaluator_test.go @@ -219,7 +219,7 @@ var _ = Describe("Templates", func() { for key := range chatMLTestMatch { foo := chatMLTestMatch[key] It("renders correctly `"+key+"`", func() { - templated := evaluator.TemplateMessages(foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) + templated := evaluator.TemplateMessages(schema.OpenAIRequest{}, foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) Expect(templated).To(Equal(foo["expected"]), templated) }) } @@ -232,7 +232,7 @@ var _ = Describe("Templates", func() { for key := range llama3TestMatch { foo := llama3TestMatch[key] It("renders correctly `"+key+"`", func() { - templated := evaluator.TemplateMessages(foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) + templated := evaluator.TemplateMessages(schema.OpenAIRequest{}, foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) Expect(templated).To(Equal(foo["expected"]), templated) }) } @@ -245,7 +245,7 @@ var _ = Describe("Templates", func() { for key := range jinjaTest { foo := jinjaTest[key] It("renders correctly `"+key+"`", func() { - templated := evaluator.TemplateMessages(foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) + templated := evaluator.TemplateMessages(schema.OpenAIRequest{}, foo["messages"].([]schema.Message), foo["config"].(*config.BackendConfig), foo["functions"].([]functions.Function), foo["shouldUseFn"].(bool)) Expect(templated).To(Equal(foo["expected"]), templated) }) }