mirror of
https://github.com/mudler/LocalAI.git
synced 2025-12-30 22:20:20 -06:00
feat(agent-jobs): add multimedia support (#7398)
* feat(agent-jobs): add multimedia support Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Refactoring Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
committed by
GitHub
GitHub
parent
45ee10ec50
commit
a3423f33e1
@@ -147,7 +147,18 @@ func ExecuteJobEndpoint(app *application.Application) echo.HandlerFunc {
|
||||
req.Parameters = make(map[string]string)
|
||||
}
|
||||
|
||||
jobID, err := app.AgentJobService().ExecuteJob(req.TaskID, req.Parameters, "api")
|
||||
// Build multimedia struct from request
|
||||
var multimedia *schema.MultimediaAttachment
|
||||
if len(req.Images) > 0 || len(req.Videos) > 0 || len(req.Audios) > 0 || len(req.Files) > 0 {
|
||||
multimedia = &schema.MultimediaAttachment{
|
||||
Images: req.Images,
|
||||
Videos: req.Videos,
|
||||
Audios: req.Audios,
|
||||
Files: req.Files,
|
||||
}
|
||||
}
|
||||
|
||||
jobID, err := app.AgentJobService().ExecuteJob(req.TaskID, req.Parameters, "api", multimedia)
|
||||
if err != nil {
|
||||
return c.JSON(http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
}
|
||||
@@ -281,7 +292,7 @@ func ExecuteTaskByNameEndpoint(app *application.Application) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
name := c.Param("name")
|
||||
var params map[string]string
|
||||
|
||||
|
||||
// Try to bind parameters from request body
|
||||
// If body is empty or invalid, use empty params
|
||||
if c.Request().ContentLength > 0 {
|
||||
@@ -323,7 +334,7 @@ func ExecuteTaskByNameEndpoint(app *application.Application) echo.HandlerFunc {
|
||||
return c.JSON(http.StatusNotFound, map[string]string{"error": "Task not found: " + name})
|
||||
}
|
||||
|
||||
jobID, err := app.AgentJobService().ExecuteJob(task.ID, params, "api")
|
||||
jobID, err := app.AgentJobService().ExecuteJob(task.ID, params, "api", nil)
|
||||
if err != nil {
|
||||
return c.JSON(http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
}
|
||||
@@ -336,4 +347,3 @@ func ExecuteTaskByNameEndpoint(app *application.Application) echo.HandlerFunc {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -365,21 +365,40 @@
|
||||
x-cloak
|
||||
@click.away="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''"
|
||||
class="fixed inset-0 bg-black/50 flex items-center justify-center z-50">
|
||||
<div class="bg-[var(--color-bg-secondary)] border border-[var(--color-primary-border)]/20 rounded-xl p-8 max-w-2xl w-full mx-4">
|
||||
<div class="flex justify-between items-center mb-6">
|
||||
<div class="bg-[var(--color-bg-secondary)] border border-[var(--color-primary-border)]/20 rounded-xl max-w-2xl w-full mx-4 max-h-[90vh] flex flex-col">
|
||||
<div class="flex justify-between items-center p-8 pb-6 border-b border-[var(--color-primary-border)]/20">
|
||||
<h3 class="text-2xl font-semibold text-[var(--color-text-primary)]">Execute Task</h3>
|
||||
<button @click="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''"
|
||||
<button @click="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''; executionMultimedia = {images: '', videos: '', audios: '', files: ''}; executeModalTab = 'parameters'"
|
||||
class="text-[var(--color-text-secondary)] hover:text-[var(--color-text-primary)]">
|
||||
<i class="fas fa-times text-xl"></i>
|
||||
</button>
|
||||
</div>
|
||||
<template x-if="selectedTaskForExecution">
|
||||
<div class="space-y-4">
|
||||
<div class="flex flex-col flex-1 min-h-0">
|
||||
<div class="flex-1 overflow-y-auto px-8 py-6 space-y-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Task</label>
|
||||
<div class="text-[var(--color-text-secondary)]" x-text="selectedTaskForExecution.name"></div>
|
||||
</div>
|
||||
<div>
|
||||
|
||||
<!-- Tabs for Parameters and Multimedia -->
|
||||
<div class="border-b border-[var(--color-primary-border)]/20">
|
||||
<div class="flex space-x-4">
|
||||
<button @click="executeModalTab = 'parameters'"
|
||||
:class="executeModalTab === 'parameters' ? 'border-b-2 border-[var(--color-primary)] text-[var(--color-primary)]' : 'text-[var(--color-text-secondary)] hover:text-[var(--color-text-primary)]'"
|
||||
class="px-4 py-2 font-medium transition-colors">
|
||||
Parameters
|
||||
</button>
|
||||
<button @click="executeModalTab = 'multimedia'"
|
||||
:class="executeModalTab === 'multimedia' ? 'border-b-2 border-[var(--color-primary)] text-[var(--color-primary)]' : 'text-[var(--color-text-secondary)] hover:text-[var(--color-text-primary)]'"
|
||||
class="px-4 py-2 font-medium transition-colors">
|
||||
Multimedia
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Parameters Tab -->
|
||||
<div x-show="executeModalTab === 'parameters'">
|
||||
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Parameters</label>
|
||||
<p class="text-xs text-[var(--color-text-secondary)] mb-3">
|
||||
Enter parameters as key-value pairs (one per line, format: key=value).
|
||||
@@ -393,8 +412,61 @@
|
||||
Example: <code class="bg-[var(--color-bg-primary)] px-1 py-0.5 rounded text-[var(--color-primary)]">user_name=Alice</code>
|
||||
</p>
|
||||
</div>
|
||||
<div class="flex justify-end space-x-4">
|
||||
<button @click="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''"
|
||||
|
||||
<!-- Multimedia Tab -->
|
||||
<div x-show="executeModalTab === 'multimedia'" class="space-y-4">
|
||||
<p class="text-xs text-[var(--color-text-secondary)] mb-3">
|
||||
Provide multimedia content as URLs or base64-encoded data URIs. You can also upload files which will be converted to base64.
|
||||
</p>
|
||||
|
||||
<!-- Images -->
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Images</label>
|
||||
<textarea x-model="executionMultimedia.images"
|
||||
rows="3"
|
||||
placeholder="https://example.com/image.png data:image/png;base64,iVBORw0KG..."
|
||||
class="w-full bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded px-4 py-2 text-[var(--color-text-primary)] font-mono text-sm focus:border-[var(--color-primary-border)] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
|
||||
<input type="file" @change="handleFileUpload($event, 'image')" accept="image/*" multiple
|
||||
class="mt-2 text-sm text-[var(--color-text-secondary)] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[var(--color-primary)] file:text-white hover:file:bg-[var(--color-primary-hover)]">
|
||||
</div>
|
||||
|
||||
<!-- Videos -->
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Videos</label>
|
||||
<textarea x-model="executionMultimedia.videos"
|
||||
rows="3"
|
||||
placeholder="https://example.com/video.mp4 data:video/mp4;base64,..."
|
||||
class="w-full bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded px-4 py-2 text-[var(--color-text-primary)] font-mono text-sm focus:border-[var(--color-primary-border)] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
|
||||
<input type="file" @change="handleFileUpload($event, 'video')" accept="video/*" multiple
|
||||
class="mt-2 text-sm text-[var(--color-text-secondary)] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[var(--color-primary)] file:text-white hover:file:bg-[var(--color-primary-hover)]">
|
||||
</div>
|
||||
|
||||
<!-- Audios -->
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Audios</label>
|
||||
<textarea x-model="executionMultimedia.audios"
|
||||
rows="3"
|
||||
placeholder="https://example.com/audio.mp3 data:audio/mpeg;base64,..."
|
||||
class="w-full bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded px-4 py-2 text-[var(--color-text-primary)] font-mono text-sm focus:border-[var(--color-primary-border)] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
|
||||
<input type="file" @change="handleFileUpload($event, 'audio')" accept="audio/*" multiple
|
||||
class="mt-2 text-sm text-[var(--color-text-secondary)] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[var(--color-primary)] file:text-white hover:file:bg-[var(--color-primary-hover)]">
|
||||
</div>
|
||||
|
||||
<!-- Files -->
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[var(--color-text-primary)] mb-2">Files</label>
|
||||
<textarea x-model="executionMultimedia.files"
|
||||
rows="3"
|
||||
placeholder="https://example.com/file.pdf data:application/pdf;base64,..."
|
||||
class="w-full bg-[var(--color-bg-primary)] border border-[var(--color-primary-border)]/20 rounded px-4 py-2 text-[var(--color-text-primary)] font-mono text-sm focus:border-[var(--color-primary-border)] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
|
||||
<input type="file" @change="handleFileUpload($event, 'file')" multiple
|
||||
class="mt-2 text-sm text-[var(--color-text-secondary)] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[var(--color-primary)] file:text-white hover:file:bg-[var(--color-primary-hover)]">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex justify-end space-x-4 p-8 pt-6 border-t border-[var(--color-primary-border)]/20 bg-[var(--color-bg-secondary)]">
|
||||
<button @click="showExecuteTaskModal = false; selectedTaskForExecution = null; executionParameters = {}; executionParametersText = ''; executionMultimedia = {images: '', videos: '', audios: '', files: ''}; executeModalTab = 'parameters'"
|
||||
class="px-4 py-2 bg-[var(--color-bg-primary)] hover:bg-[#0A0E1A] text-[var(--color-text-primary)] rounded-lg transition-colors">
|
||||
Cancel
|
||||
</button>
|
||||
@@ -433,6 +505,13 @@
|
||||
selectedTaskForExecution: null,
|
||||
executionParameters: {},
|
||||
executionParametersText: '',
|
||||
executionMultimedia: {
|
||||
images: '',
|
||||
videos: '',
|
||||
audios: '',
|
||||
files: ''
|
||||
},
|
||||
executeModalTab: 'parameters',
|
||||
modelsConfig: [],
|
||||
hasModels: false,
|
||||
hasMCPModels: false,
|
||||
@@ -528,20 +607,36 @@
|
||||
// Parse parameters from text
|
||||
this.executionParameters = this.parseParameters(this.executionParametersText);
|
||||
|
||||
// Parse multimedia from text (split by newlines, filter empty)
|
||||
const parseMultimedia = (text) => {
|
||||
if (!text || !text.trim()) return [];
|
||||
return text.split('\n')
|
||||
.map(line => line.trim())
|
||||
.filter(line => line.length > 0);
|
||||
};
|
||||
|
||||
const requestBody = {
|
||||
task_id: this.selectedTaskForExecution.id,
|
||||
parameters: this.executionParameters,
|
||||
images: parseMultimedia(this.executionMultimedia.images),
|
||||
videos: parseMultimedia(this.executionMultimedia.videos),
|
||||
audios: parseMultimedia(this.executionMultimedia.audios),
|
||||
files: parseMultimedia(this.executionMultimedia.files)
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/agent/jobs/execute', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
task_id: this.selectedTaskForExecution.id,
|
||||
parameters: this.executionParameters
|
||||
})
|
||||
body: JSON.stringify(requestBody)
|
||||
});
|
||||
if (response.ok) {
|
||||
this.showExecuteTaskModal = false;
|
||||
this.selectedTaskForExecution = null;
|
||||
this.executionParameters = {};
|
||||
this.executionParametersText = '';
|
||||
this.executionMultimedia = {images: '', videos: '', audios: '', files: ''};
|
||||
this.executeModalTab = 'parameters';
|
||||
this.fetchJobs();
|
||||
} else {
|
||||
const error = await response.json();
|
||||
@@ -552,6 +647,34 @@
|
||||
alert('Failed to execute task: ' + error.message);
|
||||
}
|
||||
},
|
||||
|
||||
handleFileUpload(event, type) {
|
||||
const files = event.target.files;
|
||||
if (!files || files.length === 0) return;
|
||||
|
||||
const dataURIs = [];
|
||||
let processed = 0;
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
const reader = new FileReader();
|
||||
|
||||
reader.onload = (e) => {
|
||||
const dataURI = e.target.result;
|
||||
dataURIs.push(dataURI);
|
||||
processed++;
|
||||
|
||||
if (processed === files.length) {
|
||||
// Append to existing content
|
||||
const current = this.executionMultimedia[type + 's'] || '';
|
||||
const newContent = current ? current + '\n' + dataURIs.join('\n') : dataURIs.join('\n');
|
||||
this.executionMultimedia[type + 's'] = newContent;
|
||||
}
|
||||
};
|
||||
|
||||
reader.readAsDataURL(file);
|
||||
}
|
||||
},
|
||||
|
||||
async deleteTask(taskId) {
|
||||
if (!confirm('Are you sure you want to delete this task?')) return;
|
||||
|
||||
@@ -165,6 +165,59 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Multimedia Sources Configuration -->
|
||||
<div class="bg-[#1E293B] border border-[#38BDF8]/20 rounded-xl p-8">
|
||||
<h2 class="text-2xl font-semibold text-[#E5E7EB] mb-6">Multimedia Sources (Optional)</h2>
|
||||
<p class="text-sm text-[#94A3B8] mb-4">
|
||||
Configure multimedia sources (images, videos, audios, files) to fetch when cron jobs execute.
|
||||
Each source can have custom headers for authentication/authorization. These will be fetched and included in the job execution.
|
||||
</p>
|
||||
<div class="space-y-4">
|
||||
<template x-for="(source, index) in taskForm.multimedia_sources" :key="index">
|
||||
<div class="bg-[#101827] p-4 rounded border border-[#38BDF8]/10">
|
||||
<div class="flex justify-between items-center mb-4">
|
||||
<h3 class="text-lg font-semibold text-[#E5E7EB]">Multimedia Source <span x-text="index + 1"></span></h3>
|
||||
<button type="button" @click="taskForm.multimedia_sources.splice(index, 1)"
|
||||
class="text-red-400 hover:text-red-300">
|
||||
<i class="fas fa-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
<div class="space-y-4">
|
||||
<div>
|
||||
<label class="block text-[#E5E7EB] mb-2">Type *</label>
|
||||
<select x-model="source.type" required
|
||||
class="w-full bg-[#0A0E1A] border border-[#38BDF8]/20 rounded px-4 py-2 text-[#E5E7EB] focus:border-[#38BDF8] focus:ring-2 focus:ring-[#38BDF8]/50">
|
||||
<option value="">Select type...</option>
|
||||
<option value="image">Image</option>
|
||||
<option value="video">Video</option>
|
||||
<option value="audio">Audio</option>
|
||||
<option value="file">File</option>
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-[#E5E7EB] mb-2">URL *</label>
|
||||
<input type="url" x-model="source.url" required
|
||||
placeholder="https://example.com/image.png"
|
||||
class="w-full bg-[#0A0E1A] border border-[#38BDF8]/20 rounded px-4 py-2 text-[#E5E7EB] focus:border-[#38BDF8] focus:ring-2 focus:ring-[#38BDF8]/50">
|
||||
<p class="text-xs text-[#94A3B8] mt-1">URL where multimedia content will be fetched from</p>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-[#E5E7EB] mb-2">Headers (JSON)</label>
|
||||
<textarea x-model="source.headers_json" rows="3"
|
||||
placeholder='{"Authorization": "Bearer token"}'
|
||||
class="w-full bg-[#0A0E1A] border border-[#38BDF8]/20 rounded px-4 py-2 text-[#E5E7EB] font-mono text-sm focus:border-[#38BDF8] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
|
||||
<p class="text-xs text-[#94A3B8] mt-1">Custom headers for the HTTP request (e.g., Authorization)</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<button type="button" @click="addMultimediaSource()"
|
||||
class="w-full bg-[#101827] hover:bg-[#0A0E1A] border border-[#38BDF8]/20 border-dashed rounded-lg p-4 text-[#94A3B8] hover:text-[#E5E7EB] transition-colors">
|
||||
<i class="fas fa-plus mr-2"></i>Add Multimedia Source
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Webhook Configuration -->
|
||||
<div class="bg-[#1E293B] border border-[#38BDF8]/20 rounded-xl p-8">
|
||||
<h2 class="text-2xl font-semibold text-[#E5E7EB] mb-6">Webhooks (Optional)</h2>
|
||||
@@ -350,6 +403,33 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<!-- Execute Task with Multimedia -->
|
||||
<div>
|
||||
<h3 class="text-lg font-semibold text-[#E5E7EB] mb-3 flex items-center">
|
||||
<i class="fas fa-images text-[#38BDF8] mr-2"></i>
|
||||
Execute Task with Multimedia (Images)
|
||||
</h3>
|
||||
<div class="bg-[#101827] border border-[#38BDF8]/10 rounded-lg p-4">
|
||||
<pre class="text-xs text-[#E5E7EB] font-mono overflow-x-auto"><code>curl -X POST {{ .BaseURL }}api/agent/jobs/execute \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer YOUR_API_KEY" \
|
||||
-d '{
|
||||
"task_id": "<span x-text="task ? task.id : 'task-uuid'"></span>",
|
||||
"parameters": {
|
||||
"user_name": "Alice",
|
||||
"task_description": "Analyze this image"
|
||||
},
|
||||
"images": [
|
||||
"https://example.com/image.png",
|
||||
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
|
||||
]
|
||||
}'</code></pre>
|
||||
</div>
|
||||
<p class="text-xs text-[#94A3B8] mt-2">
|
||||
You can provide multimedia content as URLs or base64-encoded data URIs. Supported types: <code class="bg-[#101827] px-1 py-0.5 rounded text-[#38BDF8]">images</code>, <code class="bg-[#101827] px-1 py-0.5 rounded text-[#38BDF8]">videos</code>, <code class="bg-[#101827] px-1 py-0.5 rounded text-[#38BDF8]">audios</code>, and <code class="bg-[#101827] px-1 py-0.5 rounded text-[#38BDF8]">files</code>.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<!-- Check Job Status -->
|
||||
<div>
|
||||
<h3 class="text-lg font-semibold text-[#E5E7EB] mb-3 flex items-center">
|
||||
@@ -478,23 +558,42 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
<!-- Execute Task Modal -->
|
||||
<div x-show="showExecuteTaskModal"
|
||||
x-cloak
|
||||
@click.away="showExecuteTaskModal = false; executionParameters = {}; executionParametersText = ''"
|
||||
@click.away="showExecuteTaskModal = false; executionParameters = {}; executionParametersText = ''; executionMultimedia = {images: '', videos: '', audios: '', files: ''}; executeModalTab = 'parameters'"
|
||||
class="fixed inset-0 bg-black/50 flex items-center justify-center z-50">
|
||||
<div class="bg-[#1E293B] border border-[#38BDF8]/20 rounded-xl p-8 max-w-2xl w-full mx-4">
|
||||
<div class="flex justify-between items-center mb-6">
|
||||
<div class="bg-[#1E293B] border border-[#38BDF8]/20 rounded-xl max-w-2xl w-full mx-4 max-h-[90vh] flex flex-col" @click.stop>
|
||||
<div class="flex justify-between items-center p-8 pb-6 border-b border-[#38BDF8]/20">
|
||||
<h3 class="text-2xl font-semibold text-[#E5E7EB]">Execute Task</h3>
|
||||
<button @click="showExecuteTaskModal = false; executionParameters = {}; executionParametersText = ''"
|
||||
<button @click="showExecuteTaskModal = false; executionParameters = {}; executionParametersText = ''; executionMultimedia = {images: '', videos: '', audios: '', files: ''}; executeModalTab = 'parameters'"
|
||||
class="text-[#94A3B8] hover:text-[#E5E7EB]">
|
||||
<i class="fas fa-times text-xl"></i>
|
||||
</button>
|
||||
</div>
|
||||
<template x-if="task">
|
||||
<div class="space-y-4">
|
||||
<div class="flex flex-col flex-1 min-h-0">
|
||||
<div class="flex-1 overflow-y-auto px-8 py-6 space-y-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[#E5E7EB] mb-2">Task</label>
|
||||
<div class="text-[#94A3B8]" x-text="task.name"></div>
|
||||
</div>
|
||||
<div>
|
||||
|
||||
<!-- Tabs for Parameters and Multimedia -->
|
||||
<div class="border-b border-[#38BDF8]/20">
|
||||
<div class="flex space-x-4">
|
||||
<button @click="executeModalTab = 'parameters'"
|
||||
:class="executeModalTab === 'parameters' ? 'border-b-2 border-[#38BDF8] text-[#38BDF8]' : 'text-[#94A3B8] hover:text-[#E5E7EB]'"
|
||||
class="px-4 py-2 font-medium transition-colors">
|
||||
Parameters
|
||||
</button>
|
||||
<button @click="executeModalTab = 'multimedia'"
|
||||
:class="executeModalTab === 'multimedia' ? 'border-b-2 border-[#38BDF8] text-[#38BDF8]' : 'text-[#94A3B8] hover:text-[#E5E7EB]'"
|
||||
class="px-4 py-2 font-medium transition-colors">
|
||||
Multimedia
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Parameters Tab -->
|
||||
<div x-show="executeModalTab === 'parameters'">
|
||||
<label class="block text-sm font-medium text-[#E5E7EB] mb-2">Parameters</label>
|
||||
<p class="text-xs text-[#94A3B8] mb-3">
|
||||
Enter parameters as key-value pairs (one per line, format: key=value).
|
||||
@@ -508,8 +607,61 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
Example: <code class="bg-[#101827] px-1 py-0.5 rounded text-[#38BDF8]">user_name=Alice</code>
|
||||
</p>
|
||||
</div>
|
||||
<div class="flex justify-end space-x-4">
|
||||
<button @click="showExecuteTaskModal = false; executionParameters = {}; executionParametersText = ''"
|
||||
|
||||
<!-- Multimedia Tab -->
|
||||
<div x-show="executeModalTab === 'multimedia'" class="space-y-4">
|
||||
<p class="text-xs text-[#94A3B8] mb-3">
|
||||
Provide multimedia content as URLs or base64-encoded data URIs. You can also upload files which will be converted to base64.
|
||||
</p>
|
||||
|
||||
<!-- Images -->
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[#E5E7EB] mb-2">Images</label>
|
||||
<textarea x-model="executionMultimedia.images"
|
||||
rows="3"
|
||||
placeholder="https://example.com/image.png data:image/png;base64,iVBORw0KG..."
|
||||
class="w-full bg-[#101827] border border-[#38BDF8]/20 rounded px-4 py-2 text-[#E5E7EB] font-mono text-sm focus:border-[#38BDF8] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
|
||||
<input type="file" @change="handleFileUpload($event, 'image')" accept="image/*" multiple
|
||||
class="mt-2 text-sm text-[#94A3B8] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[#38BDF8] file:text-white hover:file:bg-[#38BDF8]/80">
|
||||
</div>
|
||||
|
||||
<!-- Videos -->
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[#E5E7EB] mb-2">Videos</label>
|
||||
<textarea x-model="executionMultimedia.videos"
|
||||
rows="3"
|
||||
placeholder="https://example.com/video.mp4 data:video/mp4;base64,..."
|
||||
class="w-full bg-[#101827] border border-[#38BDF8]/20 rounded px-4 py-2 text-[#E5E7EB] font-mono text-sm focus:border-[#38BDF8] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
|
||||
<input type="file" @change="handleFileUpload($event, 'video')" accept="video/*" multiple
|
||||
class="mt-2 text-sm text-[#94A3B8] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[#38BDF8] file:text-white hover:file:bg-[#38BDF8]/80">
|
||||
</div>
|
||||
|
||||
<!-- Audios -->
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[#E5E7EB] mb-2">Audios</label>
|
||||
<textarea x-model="executionMultimedia.audios"
|
||||
rows="3"
|
||||
placeholder="https://example.com/audio.mp3 data:audio/mpeg;base64,..."
|
||||
class="w-full bg-[#101827] border border-[#38BDF8]/20 rounded px-4 py-2 text-[#E5E7EB] font-mono text-sm focus:border-[#38BDF8] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
|
||||
<input type="file" @change="handleFileUpload($event, 'audio')" accept="audio/*" multiple
|
||||
class="mt-2 text-sm text-[#94A3B8] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[#38BDF8] file:text-white hover:file:bg-[#38BDF8]/80">
|
||||
</div>
|
||||
|
||||
<!-- Files -->
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-[#E5E7EB] mb-2">Files</label>
|
||||
<textarea x-model="executionMultimedia.files"
|
||||
rows="3"
|
||||
placeholder="https://example.com/file.pdf data:application/pdf;base64,..."
|
||||
class="w-full bg-[#101827] border border-[#38BDF8]/20 rounded px-4 py-2 text-[#E5E7EB] font-mono text-sm focus:border-[#38BDF8] focus:ring-2 focus:ring-[#38BDF8]/50"></textarea>
|
||||
<input type="file" @change="handleFileUpload($event, 'file')" multiple
|
||||
class="mt-2 text-sm text-[#94A3B8] file:mr-4 file:py-2 file:px-4 file:rounded file:border-0 file:text-sm file:font-semibold file:bg-[#38BDF8] file:text-white hover:file:bg-[#38BDF8]/80">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex justify-end space-x-4 p-8 pt-6 border-t border-[#38BDF8]/20 bg-[#1E293B]">
|
||||
<button @click="showExecuteTaskModal = false; executionParameters = {}; executionParametersText = ''; executionMultimedia = {images: '', videos: '', audios: '', files: ''}; executeModalTab = 'parameters'"
|
||||
class="px-4 py-2 bg-[#101827] hover:bg-[#0A0E1A] text-[#E5E7EB] rounded-lg transition-colors">
|
||||
Cancel
|
||||
</button>
|
||||
@@ -533,6 +685,13 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
showExecuteTaskModal: false,
|
||||
executionParameters: {},
|
||||
executionParametersText: '',
|
||||
executionMultimedia: {
|
||||
images: '',
|
||||
videos: '',
|
||||
audios: '',
|
||||
files: ''
|
||||
},
|
||||
executeModalTab: 'parameters',
|
||||
isNewTask: false,
|
||||
isEditMode: false,
|
||||
taskForm: {
|
||||
@@ -543,7 +702,8 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
enabled: true,
|
||||
cron: '',
|
||||
cron_parameters: {},
|
||||
webhooks: []
|
||||
webhooks: [],
|
||||
multimedia_sources: []
|
||||
},
|
||||
cronError: '',
|
||||
cronParametersText: '',
|
||||
@@ -600,6 +760,15 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
// Note: Legacy fields (webhook_url, webhook_auth, webhook_template) should be migrated
|
||||
// by the backend, so we don't need to handle them here
|
||||
|
||||
// Handle multimedia sources
|
||||
let multimediaSources = [];
|
||||
if (this.task.multimedia_sources && Array.isArray(this.task.multimedia_sources) && this.task.multimedia_sources.length > 0) {
|
||||
multimediaSources = this.task.multimedia_sources.map(ms => ({
|
||||
...ms,
|
||||
headers_json: JSON.stringify(ms.headers || {}, null, 2)
|
||||
}));
|
||||
}
|
||||
|
||||
// Convert cron_parameters to text format
|
||||
let cronParamsText = '';
|
||||
if (this.task.cron_parameters && Object.keys(this.task.cron_parameters).length > 0) {
|
||||
@@ -616,7 +785,8 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
enabled: this.task.enabled !== undefined ? this.task.enabled : true,
|
||||
cron: this.task.cron || '',
|
||||
cron_parameters: this.task.cron_parameters || {},
|
||||
webhooks: webhooks
|
||||
webhooks: webhooks,
|
||||
multimedia_sources: multimediaSources
|
||||
};
|
||||
this.cronParametersText = cronParamsText;
|
||||
} else {
|
||||
@@ -667,6 +837,19 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
this.taskForm.webhooks.push(webhook);
|
||||
},
|
||||
|
||||
addMultimediaSource() {
|
||||
const source = {
|
||||
type: '',
|
||||
url: '',
|
||||
headers: {},
|
||||
headers_json: '{}'
|
||||
};
|
||||
if (!this.taskForm.multimedia_sources) {
|
||||
this.taskForm.multimedia_sources = [];
|
||||
}
|
||||
this.taskForm.multimedia_sources.push(source);
|
||||
},
|
||||
|
||||
updateCronParameters() {
|
||||
// Parse text input into parameters object
|
||||
const params = {};
|
||||
@@ -791,6 +974,8 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
showExecuteModal() {
|
||||
this.executionParameters = {};
|
||||
this.executionParametersText = '';
|
||||
this.executionMultimedia = {images: '', videos: '', audios: '', files: ''};
|
||||
this.executeModalTab = 'parameters';
|
||||
this.showExecuteTaskModal = true;
|
||||
},
|
||||
|
||||
@@ -821,19 +1006,35 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
// Parse parameters from text
|
||||
this.executionParameters = this.parseParameters(this.executionParametersText);
|
||||
|
||||
// Parse multimedia from text (split by newlines, filter empty)
|
||||
const parseMultimedia = (text) => {
|
||||
if (!text || !text.trim()) return [];
|
||||
return text.split('\n')
|
||||
.map(line => line.trim())
|
||||
.filter(line => line.length > 0);
|
||||
};
|
||||
|
||||
const requestBody = {
|
||||
task_id: this.task.id,
|
||||
parameters: this.executionParameters,
|
||||
images: parseMultimedia(this.executionMultimedia.images),
|
||||
videos: parseMultimedia(this.executionMultimedia.videos),
|
||||
audios: parseMultimedia(this.executionMultimedia.audios),
|
||||
files: parseMultimedia(this.executionMultimedia.files)
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/agent/jobs/execute', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
task_id: this.task.id,
|
||||
parameters: this.executionParameters
|
||||
})
|
||||
body: JSON.stringify(requestBody)
|
||||
});
|
||||
if (response.ok) {
|
||||
this.showExecuteTaskModal = false;
|
||||
this.executionParameters = {};
|
||||
this.executionParametersText = '';
|
||||
this.executionMultimedia = {images: '', videos: '', audios: '', files: ''};
|
||||
this.executeModalTab = 'parameters';
|
||||
this.fetchJobs();
|
||||
} else {
|
||||
const error = await response.json();
|
||||
@@ -844,6 +1045,34 @@ Provide a detailed response that addresses their specific needs.</pre>
|
||||
alert('Failed to execute task: ' + error.message);
|
||||
}
|
||||
},
|
||||
|
||||
handleFileUpload(event, type) {
|
||||
const files = event.target.files;
|
||||
if (!files || files.length === 0) return;
|
||||
|
||||
const dataURIs = [];
|
||||
let processed = 0;
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
const reader = new FileReader();
|
||||
|
||||
reader.onload = (e) => {
|
||||
const dataURI = e.target.result;
|
||||
dataURIs.push(dataURI);
|
||||
processed++;
|
||||
|
||||
if (processed === files.length) {
|
||||
// Append to existing content
|
||||
const current = this.executionMultimedia[type + 's'] || '';
|
||||
const newContent = current ? current + '\n' + dataURIs.join('\n') : dataURIs.join('\n');
|
||||
this.executionMultimedia[type + 's'] = newContent;
|
||||
}
|
||||
};
|
||||
|
||||
reader.readAsDataURL(file);
|
||||
}
|
||||
},
|
||||
|
||||
async deleteTask() {
|
||||
if (!confirm('Are you sure you want to delete this task? This will also delete all associated jobs.')) return;
|
||||
|
||||
@@ -75,13 +75,14 @@
|
||||
Find Backend Components
|
||||
</h3>
|
||||
<div class="relative">
|
||||
<div class="absolute inset-y-0 start-0 flex items-center ps-4 pointer-events-none">
|
||||
<div class="absolute inset-y-0 start-0 flex items-center ps-4 pointer-events-none z-10">
|
||||
<i class="fas fa-search text-[#94A3B8]"></i>
|
||||
</div>
|
||||
<input
|
||||
x-model="searchTerm"
|
||||
@input.debounce.500ms="fetchBackends()"
|
||||
class="input w-full pl-12 pr-16 py-4"
|
||||
class="input w-full pr-16 py-4"
|
||||
style="padding-left: 3.5rem !important;"
|
||||
type="search"
|
||||
placeholder="Search backends by name, description or type...">
|
||||
<span class="absolute right-4 top-4" x-show="loading">
|
||||
|
||||
@@ -717,9 +717,10 @@ SOFTWARE.
|
||||
type="text"
|
||||
x-model="searchQuery"
|
||||
placeholder="Search conversations..."
|
||||
class="w-full bg-[var(--color-bg-primary)] text-[var(--color-text-primary)] border border-[var(--color-bg-secondary)] focus:border-[var(--color-primary-border)] focus:ring-1 focus:ring-[var(--color-primary)]/50 rounded px-2 py-1.5 pl-7 text-xs placeholder-[var(--color-text-secondary)]"
|
||||
class="w-full bg-[var(--color-bg-primary)] text-[var(--color-text-primary)] border border-[var(--color-bg-secondary)] focus:border-[var(--color-primary-border)] focus:ring-1 focus:ring-[var(--color-primary)]/50 rounded py-1.5 pr-2 text-xs placeholder-[var(--color-text-secondary)]"
|
||||
style="padding-left: 2rem !important;"
|
||||
/>
|
||||
<i class="fa-solid fa-search absolute left-2 top-1/2 transform -translate-y-1/2 text-[var(--color-text-secondary)] text-xs"></i>
|
||||
<i class="fa-solid fa-search absolute left-2.5 top-1/2 transform -translate-y-1/2 text-[var(--color-text-secondary)] text-xs pointer-events-none z-10"></i>
|
||||
<button
|
||||
x-show="searchQuery.length > 0"
|
||||
@click="searchQuery = ''"
|
||||
|
||||
@@ -80,13 +80,14 @@
|
||||
Find Your Perfect Model
|
||||
</h3>
|
||||
<div class="relative">
|
||||
<div class="absolute inset-y-0 start-0 flex items-center ps-4 pointer-events-none">
|
||||
<div class="absolute inset-y-0 start-0 flex items-center ps-4 pointer-events-none z-10">
|
||||
<i class="fas fa-search text-[var(--color-text-secondary)]"></i>
|
||||
</div>
|
||||
<input
|
||||
x-model="searchTerm"
|
||||
@input.debounce.500ms="fetchModels()"
|
||||
class="input w-full pl-12 pr-16 py-4"
|
||||
class="input w-full pr-16 py-4"
|
||||
style="padding-left: 3.5rem !important;"
|
||||
type="search"
|
||||
placeholder="Search models by name, tag, or description...">
|
||||
<span class="absolute right-4 top-4" x-show="loading">
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
<input id="image-model" type="hidden" value="{{.Model}}">
|
||||
<form id="genimage" action="text2image/{{.Model}}" method="get" class="mb-8">
|
||||
<div class="relative">
|
||||
<div class="absolute inset-y-0 left-0 flex items-center pl-4">
|
||||
<div class="absolute inset-y-0 left-0 flex items-center pl-4 pointer-events-none z-10">
|
||||
<i class="fas fa-magic text-[var(--color-primary)]"></i>
|
||||
</div>
|
||||
<input
|
||||
@@ -65,7 +65,8 @@
|
||||
name="input"
|
||||
placeholder="Describe the image you want to generate..."
|
||||
autocomplete="off"
|
||||
class="input w-full pl-12 pr-12 py-4 text-lg"
|
||||
class="input w-full pr-12 py-4 text-lg"
|
||||
style="padding-left: 3.5rem !important;"
|
||||
required
|
||||
/>
|
||||
<span id="loader" class="my-2 loader absolute right-4 top-4 hidden">
|
||||
|
||||
@@ -27,6 +27,10 @@ type Task struct {
|
||||
// - {{.Status}} - Job status string
|
||||
Webhooks []WebhookConfig `json:"webhooks,omitempty"` // Webhook configs for job completion notifications
|
||||
|
||||
// Multimedia sources (for cron jobs)
|
||||
// URLs to fetch multimedia content from when cron job executes
|
||||
// Each source can have custom headers for authentication/authorization
|
||||
MultimediaSources []MultimediaSourceConfig `json:"multimedia_sources,omitempty"` // Multimedia sources for cron jobs
|
||||
}
|
||||
|
||||
// WebhookConfig represents configuration for sending webhook notifications
|
||||
@@ -44,6 +48,21 @@ type WebhookConfig struct {
|
||||
// - {{.Status}} - Job status string
|
||||
}
|
||||
|
||||
// MultimediaSourceConfig represents configuration for fetching multimedia content
|
||||
// Used in cron jobs to periodically fetch multimedia from URLs with custom headers
|
||||
type MultimediaSourceConfig struct {
|
||||
Type string `json:"type"` // "image", "video", "audio", "file"
|
||||
URL string `json:"url"` // URL to fetch from
|
||||
Headers map[string]string `json:"headers,omitempty"` // Custom headers for HTTP request (e.g., Authorization)
|
||||
}
|
||||
|
||||
type MultimediaAttachment struct {
|
||||
Images []string `json:"images,omitempty"`
|
||||
Videos []string `json:"videos,omitempty"`
|
||||
Audios []string `json:"audios,omitempty"`
|
||||
Files []string `json:"files,omitempty"`
|
||||
}
|
||||
|
||||
// JobStatus represents the status of a job
|
||||
type JobStatus string
|
||||
|
||||
@@ -75,6 +94,13 @@ type Job struct {
|
||||
|
||||
// Execution traces (reasoning, tool calls, tool results)
|
||||
Traces []JobTrace `json:"traces,omitempty"`
|
||||
|
||||
// Multimedia content (for manual execution)
|
||||
// Can contain URLs or base64-encoded data URIs
|
||||
Images []string `json:"images,omitempty"` // List of image URLs or base64 strings
|
||||
Videos []string `json:"videos,omitempty"` // List of video URLs or base64 strings
|
||||
Audios []string `json:"audios,omitempty"` // List of audio URLs or base64 strings
|
||||
Files []string `json:"files,omitempty"` // List of file URLs or base64 strings
|
||||
}
|
||||
|
||||
// JobTrace represents a single execution trace entry
|
||||
@@ -90,6 +116,12 @@ type JobTrace struct {
|
||||
type JobExecutionRequest struct {
|
||||
TaskID string `json:"task_id"` // Required
|
||||
Parameters map[string]string `json:"parameters"` // Optional, for templating
|
||||
// Multimedia content (optional, for manual execution)
|
||||
// Can contain URLs or base64-encoded data URIs
|
||||
Images []string `json:"images,omitempty"` // List of image URLs or base64 strings
|
||||
Videos []string `json:"videos,omitempty"` // List of video URLs or base64 strings
|
||||
Audios []string `json:"audios,omitempty"` // List of audio URLs or base64 strings
|
||||
Files []string `json:"files,omitempty"` // List of file URLs or base64 strings
|
||||
}
|
||||
|
||||
// JobExecutionResponse represents the response after creating a job
|
||||
|
||||
@@ -3,6 +3,7 @@ package services
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -72,6 +73,13 @@ type JobExecution struct {
|
||||
Cancel context.CancelFunc
|
||||
}
|
||||
|
||||
const (
|
||||
JobImageType = "image"
|
||||
JobVideoType = "video"
|
||||
JobAudioType = "audio"
|
||||
JobFileType = "file"
|
||||
)
|
||||
|
||||
// NewAgentJobService creates a new AgentJobService instance
|
||||
func NewAgentJobService(
|
||||
appConfig *config.ApplicationConfig,
|
||||
@@ -366,7 +374,8 @@ func (s *AgentJobService) buildPrompt(templateStr string, params map[string]stri
|
||||
}
|
||||
|
||||
// ExecuteJob creates and queues a job for execution
|
||||
func (s *AgentJobService) ExecuteJob(taskID string, params map[string]string, triggeredBy string) (string, error) {
|
||||
// multimedia can be nil for backward compatibility
|
||||
func (s *AgentJobService) ExecuteJob(taskID string, params map[string]string, triggeredBy string, multimedia *schema.MultimediaAttachment) (string, error) {
|
||||
task := s.tasks.Get(taskID)
|
||||
if task.ID == "" {
|
||||
return "", fmt.Errorf("task not found: %s", taskID)
|
||||
@@ -388,6 +397,52 @@ func (s *AgentJobService) ExecuteJob(taskID string, params map[string]string, tr
|
||||
TriggeredBy: triggeredBy,
|
||||
}
|
||||
|
||||
// Handle multimedia: merge task-level (for cron) and job-level (for manual execution)
|
||||
if triggeredBy == "cron" && len(task.MultimediaSources) > 0 {
|
||||
// Fetch multimedia from task sources
|
||||
job.Images = []string{}
|
||||
job.Videos = []string{}
|
||||
job.Audios = []string{}
|
||||
job.Files = []string{}
|
||||
|
||||
for _, source := range task.MultimediaSources {
|
||||
// Fetch content from URL with custom headers
|
||||
dataURI, err := s.fetchMultimediaFromURL(source.URL, source.Headers, source.Type)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("url", source.URL).Str("type", source.Type).Msg("Failed to fetch multimedia from task source")
|
||||
continue
|
||||
}
|
||||
|
||||
// Add to appropriate slice based on type
|
||||
switch source.Type {
|
||||
case JobImageType:
|
||||
job.Images = append(job.Images, dataURI)
|
||||
case JobVideoType:
|
||||
job.Videos = append(job.Videos, dataURI)
|
||||
case JobAudioType:
|
||||
job.Audios = append(job.Audios, dataURI)
|
||||
case JobFileType:
|
||||
job.Files = append(job.Files, dataURI)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Override with job-level multimedia if provided (manual execution takes precedence)
|
||||
if multimedia != nil {
|
||||
if len(multimedia.Images) > 0 {
|
||||
job.Images = multimedia.Images
|
||||
}
|
||||
if len(multimedia.Videos) > 0 {
|
||||
job.Videos = multimedia.Videos
|
||||
}
|
||||
if len(multimedia.Audios) > 0 {
|
||||
job.Audios = multimedia.Audios
|
||||
}
|
||||
if len(multimedia.Files) > 0 {
|
||||
job.Files = multimedia.Files
|
||||
}
|
||||
}
|
||||
|
||||
// Store job
|
||||
s.jobs.Set(jobID, job)
|
||||
|
||||
@@ -512,6 +567,108 @@ func (s *AgentJobService) DeleteJob(id string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type multimediaContent struct {
|
||||
url string
|
||||
mediaType string
|
||||
}
|
||||
|
||||
func (mu multimediaContent) URL() string {
|
||||
return mu.url
|
||||
}
|
||||
|
||||
// fetchMultimediaFromURL fetches multimedia content from a URL with custom headers
|
||||
// and converts it to a data URI string
|
||||
func (s *AgentJobService) fetchMultimediaFromURL(url string, headers map[string]string, mediaType string) (string, error) {
|
||||
// Create HTTP request
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// Set custom headers
|
||||
for key, value := range headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
// Execute request
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to fetch URL: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return "", fmt.Errorf("HTTP error: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Read content
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
// Encode to base64
|
||||
encoded := base64.StdEncoding.EncodeToString(data)
|
||||
|
||||
// Determine MIME type
|
||||
mimeType := s.getMimeTypeForMediaType(mediaType)
|
||||
if contentType := resp.Header.Get("Content-Type"); contentType != "" {
|
||||
mimeType = contentType
|
||||
}
|
||||
|
||||
// Return as data URI
|
||||
return fmt.Sprintf("data:%s;base64,%s", mimeType, encoded), nil
|
||||
}
|
||||
|
||||
// getMimeTypeForMediaType returns the default MIME type for a media type
|
||||
func (s *AgentJobService) getMimeTypeForMediaType(mediaType string) string {
|
||||
switch mediaType {
|
||||
case JobImageType:
|
||||
return "image/png"
|
||||
case JobVideoType:
|
||||
return "video/mp4"
|
||||
case JobAudioType:
|
||||
return "audio/mpeg"
|
||||
case JobFileType:
|
||||
return "application/octet-stream"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
|
||||
// convertToMultimediaContent converts a slice of strings (URLs or base64) to multimediaContent objects
|
||||
func (s *AgentJobService) convertToMultimediaContent(items []string, mediaType string) ([]cogito.Multimedia, error) {
|
||||
result := make([]cogito.Multimedia, 0, len(items))
|
||||
|
||||
for _, item := range items {
|
||||
if item == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if it's already a data URI
|
||||
if strings.HasPrefix(item, "data:") {
|
||||
result = append(result, multimediaContent{url: item, mediaType: mediaType})
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if it's a URL
|
||||
if strings.HasPrefix(item, "http://") || strings.HasPrefix(item, "https://") {
|
||||
// Pass URL directly to cogito (it handles fetching)
|
||||
result = append(result, multimediaContent{url: item, mediaType: mediaType})
|
||||
continue
|
||||
}
|
||||
|
||||
// Assume it's base64 without data URI prefix
|
||||
// Add appropriate prefix based on media type
|
||||
mimeType := s.getMimeTypeForMediaType(mediaType)
|
||||
dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, item)
|
||||
result = append(result, multimediaContent{url: dataURI, mediaType: mediaType})
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// executeJobInternal executes a job using cogito
|
||||
func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, ctx context.Context) error {
|
||||
// Update job status to running
|
||||
@@ -585,7 +742,51 @@ func (s *AgentJobService) executeJobInternal(job schema.Job, task schema.Task, c
|
||||
|
||||
// Create cogito fragment
|
||||
fragment := cogito.NewEmptyFragment()
|
||||
fragment = fragment.AddMessage("user", prompt)
|
||||
|
||||
// Collect all multimedia content
|
||||
multimediaItems := []cogito.Multimedia{}
|
||||
|
||||
// Convert images
|
||||
if len(job.Images) > 0 {
|
||||
images, err := s.convertToMultimediaContent(job.Images, JobImageType)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert images")
|
||||
} else {
|
||||
multimediaItems = append(multimediaItems, images...)
|
||||
}
|
||||
}
|
||||
|
||||
// Convert videos
|
||||
if len(job.Videos) > 0 {
|
||||
videos, err := s.convertToMultimediaContent(job.Videos, JobVideoType)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert videos")
|
||||
} else {
|
||||
multimediaItems = append(multimediaItems, videos...)
|
||||
}
|
||||
}
|
||||
|
||||
// Convert audios
|
||||
if len(job.Audios) > 0 {
|
||||
audios, err := s.convertToMultimediaContent(job.Audios, JobAudioType)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert audios")
|
||||
} else {
|
||||
multimediaItems = append(multimediaItems, audios...)
|
||||
}
|
||||
}
|
||||
|
||||
// Convert files
|
||||
if len(job.Files) > 0 {
|
||||
files, err := s.convertToMultimediaContent(job.Files, JobFileType)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("job_id", job.ID).Msg("Failed to convert files")
|
||||
} else {
|
||||
multimediaItems = append(multimediaItems, files...)
|
||||
}
|
||||
}
|
||||
|
||||
fragment = fragment.AddMessage("user", prompt, multimediaItems...)
|
||||
|
||||
// Get API address and key
|
||||
_, port, err := net.SplitHostPort(s.appConfig.APIAddress)
|
||||
@@ -845,7 +1046,8 @@ func (s *AgentJobService) ScheduleCronTask(task schema.Task) error {
|
||||
}
|
||||
entryID, err := s.cronScheduler.AddFunc(cronExpr, func() {
|
||||
// Create job for cron execution with configured parameters
|
||||
_, err := s.ExecuteJob(task.ID, cronParams, "cron")
|
||||
// Multimedia will be fetched from task sources in ExecuteJob
|
||||
_, err := s.ExecuteJob(task.ID, cronParams, "cron", nil)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("task_id", task.ID).Msg("Failed to execute cron job")
|
||||
}
|
||||
|
||||
@@ -153,7 +153,7 @@ var _ = Describe("AgentJobService", func() {
|
||||
|
||||
It("should create and queue a job", func() {
|
||||
params := map[string]string{"name": "World"}
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test")
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test", nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(jobID).NotTo(BeEmpty())
|
||||
|
||||
@@ -166,12 +166,12 @@ var _ = Describe("AgentJobService", func() {
|
||||
|
||||
It("should list jobs with filters", func() {
|
||||
params := map[string]string{}
|
||||
jobID1, err := service.ExecuteJob(taskID, params, "test")
|
||||
jobID1, err := service.ExecuteJob(taskID, params, "test", nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
time.Sleep(10 * time.Millisecond) // Ensure different timestamps
|
||||
|
||||
jobID2, err := service.ExecuteJob(taskID, params, "test")
|
||||
jobID2, err := service.ExecuteJob(taskID, params, "test", nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
allJobs := service.ListJobs(nil, nil, 0)
|
||||
@@ -195,7 +195,7 @@ var _ = Describe("AgentJobService", func() {
|
||||
|
||||
It("should cancel a pending job", func() {
|
||||
params := map[string]string{}
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test")
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test", nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = service.CancelJob(jobID)
|
||||
@@ -208,7 +208,7 @@ var _ = Describe("AgentJobService", func() {
|
||||
|
||||
It("should delete a job", func() {
|
||||
params := map[string]string{}
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test")
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test", nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = service.DeleteJob(jobID)
|
||||
@@ -258,7 +258,7 @@ var _ = Describe("AgentJobService", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
params := map[string]string{}
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test")
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test", nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
service.SaveJobsToFile()
|
||||
@@ -311,7 +311,7 @@ var _ = Describe("AgentJobService", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
params := map[string]string{}
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test")
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test", nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Manually set job creation time to be old
|
||||
@@ -329,4 +329,270 @@ var _ = Describe("AgentJobService", func() {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Multimedia support", func() {
|
||||
Describe("Task multimedia sources", func() {
|
||||
It("should create a task with multimedia sources", func() {
|
||||
task := schema.Task{
|
||||
Name: "Multimedia Task",
|
||||
Model: "test-model",
|
||||
Prompt: "Analyze this image",
|
||||
MultimediaSources: []schema.MultimediaSourceConfig{
|
||||
{
|
||||
Type: "image",
|
||||
URL: "https://example.com/image.png",
|
||||
Headers: map[string]string{"Authorization": "Bearer token123"},
|
||||
},
|
||||
{
|
||||
Type: "video",
|
||||
URL: "https://example.com/video.mp4",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
id, err := service.CreateTask(task)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(id).NotTo(BeEmpty())
|
||||
|
||||
retrieved, err := service.GetTask(id)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(retrieved.MultimediaSources).To(HaveLen(2))
|
||||
Expect(retrieved.MultimediaSources[0].Type).To(Equal("image"))
|
||||
Expect(retrieved.MultimediaSources[0].URL).To(Equal("https://example.com/image.png"))
|
||||
Expect(retrieved.MultimediaSources[0].Headers["Authorization"]).To(Equal("Bearer token123"))
|
||||
Expect(retrieved.MultimediaSources[1].Type).To(Equal("video"))
|
||||
})
|
||||
|
||||
It("should save and load tasks with multimedia sources from file", func() {
|
||||
task := schema.Task{
|
||||
Name: "Persistent Multimedia Task",
|
||||
Model: "test-model",
|
||||
Prompt: "Test prompt",
|
||||
MultimediaSources: []schema.MultimediaSourceConfig{
|
||||
{
|
||||
Type: "audio",
|
||||
URL: "https://example.com/audio.mp3",
|
||||
Headers: map[string]string{
|
||||
"X-Custom-Header": "value",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
id, err := service.CreateTask(task)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Create a new service instance to test loading
|
||||
newService := services.NewAgentJobService(
|
||||
appConfig,
|
||||
modelLoader,
|
||||
configLoader,
|
||||
evaluator,
|
||||
)
|
||||
|
||||
err = newService.LoadTasksFromFile()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
retrieved, err := newService.GetTask(id)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(retrieved.Name).To(Equal("Persistent Multimedia Task"))
|
||||
Expect(retrieved.MultimediaSources).To(HaveLen(1))
|
||||
Expect(retrieved.MultimediaSources[0].Type).To(Equal("audio"))
|
||||
Expect(retrieved.MultimediaSources[0].URL).To(Equal("https://example.com/audio.mp3"))
|
||||
Expect(retrieved.MultimediaSources[0].Headers["X-Custom-Header"]).To(Equal("value"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Job multimedia", func() {
|
||||
var taskID string
|
||||
|
||||
BeforeEach(func() {
|
||||
task := schema.Task{
|
||||
Name: "Test Task",
|
||||
Model: "test-model",
|
||||
Prompt: "Hello {{.name}}",
|
||||
Enabled: true,
|
||||
}
|
||||
var err error
|
||||
taskID, err = service.CreateTask(task)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("should create a job with multimedia content", func() {
|
||||
params := map[string]string{"name": "World"}
|
||||
multimedia := &schema.MultimediaAttachment{
|
||||
Images: []string{"https://example.com/image1.png", "data:image/png;base64,iVBORw0KG"},
|
||||
Videos: []string{"https://example.com/video.mp4"},
|
||||
Audios: []string{"data:audio/mpeg;base64,SUQzBAAAAA"},
|
||||
Files: []string{"https://example.com/file.pdf"},
|
||||
}
|
||||
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test", multimedia)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(jobID).NotTo(BeEmpty())
|
||||
|
||||
job, err := service.GetJob(jobID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(job.TaskID).To(Equal(taskID))
|
||||
Expect(job.Images).To(HaveLen(2))
|
||||
Expect(job.Images[0]).To(Equal("https://example.com/image1.png"))
|
||||
Expect(job.Images[1]).To(Equal("data:image/png;base64,iVBORw0KG"))
|
||||
Expect(job.Videos).To(HaveLen(1))
|
||||
Expect(job.Videos[0]).To(Equal("https://example.com/video.mp4"))
|
||||
Expect(job.Audios).To(HaveLen(1))
|
||||
Expect(job.Audios[0]).To(Equal("data:audio/mpeg;base64,SUQzBAAAAA"))
|
||||
Expect(job.Files).To(HaveLen(1))
|
||||
Expect(job.Files[0]).To(Equal("https://example.com/file.pdf"))
|
||||
})
|
||||
|
||||
It("should create a job with partial multimedia (only images)", func() {
|
||||
params := map[string]string{}
|
||||
multimedia := &schema.MultimediaAttachment{
|
||||
Images: []string{"https://example.com/image.png"},
|
||||
}
|
||||
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test", multimedia)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
job, err := service.GetJob(jobID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(job.Images).To(HaveLen(1))
|
||||
Expect(job.Videos).To(BeEmpty())
|
||||
Expect(job.Audios).To(BeEmpty())
|
||||
Expect(job.Files).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("should create a job without multimedia (nil)", func() {
|
||||
params := map[string]string{"name": "Test"}
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test", nil)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
job, err := service.GetJob(jobID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(job.Images).To(BeEmpty())
|
||||
Expect(job.Videos).To(BeEmpty())
|
||||
Expect(job.Audios).To(BeEmpty())
|
||||
Expect(job.Files).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("should save and load jobs with multimedia from file", func() {
|
||||
params := map[string]string{}
|
||||
multimedia := &schema.MultimediaAttachment{
|
||||
Images: []string{"https://example.com/image.png"},
|
||||
Videos: []string{"https://example.com/video.mp4"},
|
||||
}
|
||||
|
||||
jobID, err := service.ExecuteJob(taskID, params, "test", multimedia)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Wait a bit for async save to complete
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
// Ensure directory exists before saving
|
||||
err = os.MkdirAll(tempDir, 0755)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
err = service.SaveJobsToFile()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// Create a new service instance to test loading
|
||||
newService := services.NewAgentJobService(
|
||||
appConfig,
|
||||
modelLoader,
|
||||
configLoader,
|
||||
evaluator,
|
||||
)
|
||||
|
||||
err = newService.LoadJobsFromFile()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
retrieved, err := newService.GetJob(jobID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(retrieved.TaskID).To(Equal(taskID))
|
||||
Expect(retrieved.Images).To(HaveLen(1))
|
||||
Expect(retrieved.Images[0]).To(Equal("https://example.com/image.png"))
|
||||
Expect(retrieved.Videos).To(HaveLen(1))
|
||||
Expect(retrieved.Videos[0]).To(Equal("https://example.com/video.mp4"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Multimedia format handling", func() {
|
||||
var taskID string
|
||||
|
||||
BeforeEach(func() {
|
||||
task := schema.Task{
|
||||
Name: "Test Task",
|
||||
Model: "test-model",
|
||||
Prompt: "Test prompt",
|
||||
Enabled: true,
|
||||
}
|
||||
var err error
|
||||
taskID, err = service.CreateTask(task)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
})
|
||||
|
||||
It("should handle URLs correctly", func() {
|
||||
multimedia := &schema.MultimediaAttachment{
|
||||
Images: []string{"https://example.com/image.png"},
|
||||
Videos: []string{"http://example.com/video.mp4"},
|
||||
}
|
||||
|
||||
jobID, err := service.ExecuteJob(taskID, map[string]string{}, "test", multimedia)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
job, err := service.GetJob(jobID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(job.Images[0]).To(Equal("https://example.com/image.png"))
|
||||
Expect(job.Videos[0]).To(Equal("http://example.com/video.mp4"))
|
||||
})
|
||||
|
||||
It("should handle data URIs correctly", func() {
|
||||
multimedia := &schema.MultimediaAttachment{
|
||||
Images: []string{"data:image/png;base64,iVBORw0KG"},
|
||||
Videos: []string{"data:video/mp4;base64,AAAAIGZ0eXBpc29t"},
|
||||
}
|
||||
|
||||
jobID, err := service.ExecuteJob(taskID, map[string]string{}, "test", multimedia)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
job, err := service.GetJob(jobID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(job.Images[0]).To(Equal("data:image/png;base64,iVBORw0KG"))
|
||||
Expect(job.Videos[0]).To(Equal("data:video/mp4;base64,AAAAIGZ0eXBpc29t"))
|
||||
})
|
||||
|
||||
It("should handle base64 strings (will be converted during execution)", func() {
|
||||
// Base64 strings without data URI prefix should be stored as-is
|
||||
// They will be converted to data URIs during execution
|
||||
multimedia := &schema.MultimediaAttachment{
|
||||
Images: []string{"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="},
|
||||
}
|
||||
|
||||
jobID, err := service.ExecuteJob(taskID, map[string]string{}, "test", multimedia)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
job, err := service.GetJob(jobID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
// The base64 string is stored as-is in the job
|
||||
Expect(job.Images[0]).To(Equal("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="))
|
||||
})
|
||||
|
||||
It("should handle empty multimedia arrays", func() {
|
||||
multimedia := &schema.MultimediaAttachment{
|
||||
Images: []string{""},
|
||||
}
|
||||
|
||||
jobID, err := service.ExecuteJob(taskID, map[string]string{}, "test", multimedia)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
job, err := service.GetJob(jobID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
// Empty strings are stored in the job but will be filtered during execution
|
||||
// The job stores what was provided, filtering happens in convertToMultimediaContent
|
||||
Expect(job.Images).To(HaveLen(1))
|
||||
Expect(job.Images[0]).To(Equal(""))
|
||||
Expect(job.Videos).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user