mirror of
https://github.com/mudler/LocalAI.git
synced 2026-05-19 00:59:58 -05:00
fix(videogen): drop incomplete endpoint, add GGUF support for LTX-2 (#8160)
* Debug Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Drop openai video endpoint (is not complete) Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add download button Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
committed by
GitHub
GitHub
parent
be7ed85838
commit
0fa0ac4797
@@ -167,6 +167,16 @@ func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
|
||||
|
||||
baseURL := middleware.BaseURL(c)
|
||||
|
||||
xlog.Debug("VideoEndpoint: Calling VideoGeneration",
|
||||
"num_frames", input.NumFrames,
|
||||
"fps", input.FPS,
|
||||
"cfg_scale", input.CFGScale,
|
||||
"step", input.Step,
|
||||
"seed", input.Seed,
|
||||
"width", width,
|
||||
"height", height,
|
||||
"negative_prompt", input.NegativePrompt)
|
||||
|
||||
fn, err := backend.VideoGeneration(
|
||||
height,
|
||||
width,
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
package openai
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/http/endpoints/localai"
|
||||
"github.com/mudler/LocalAI/core/http/middleware"
|
||||
"github.com/mudler/LocalAI/core/schema"
|
||||
model "github.com/mudler/LocalAI/pkg/model"
|
||||
)
|
||||
|
||||
func VideoEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.OpenAIRequest)
|
||||
if !ok || input == nil {
|
||||
return echo.ErrBadRequest
|
||||
}
|
||||
var raw map[string]interface{}
|
||||
body := make([]byte, 0)
|
||||
if c.Request().Body != nil {
|
||||
c.Request().Body.Read(body)
|
||||
}
|
||||
if len(body) > 0 {
|
||||
_ = json.Unmarshal(body, &raw)
|
||||
}
|
||||
// Build VideoRequest using shared mapper
|
||||
vr := MapOpenAIToVideo(input, raw)
|
||||
// Place VideoRequest into context so localai.VideoEndpoint can consume it
|
||||
c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, vr)
|
||||
// Delegate to existing localai handler
|
||||
return localai.VideoEndpoint(cl, ml, appConfig)(c)
|
||||
}
|
||||
}
|
||||
|
||||
// VideoEndpoint godoc
|
||||
// @Summary Generate a video from an OpenAI-compatible request
|
||||
// @Description Accepts an OpenAI-style request and delegates to the LocalAI video generator
|
||||
// @Tags openai
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body schema.OpenAIRequest true "OpenAI-style request"
|
||||
// @Success 200 {object} map[string]interface{}
|
||||
// @Failure 400 {object} map[string]interface{}
|
||||
// @Router /v1/videos [post]
|
||||
|
||||
func MapOpenAIToVideo(input *schema.OpenAIRequest, raw map[string]interface{}) *schema.VideoRequest {
|
||||
vr := &schema.VideoRequest{}
|
||||
if input == nil {
|
||||
return vr
|
||||
}
|
||||
|
||||
if input.Model != "" {
|
||||
vr.Model = input.Model
|
||||
}
|
||||
|
||||
// Prompt mapping
|
||||
switch p := input.Prompt.(type) {
|
||||
case string:
|
||||
vr.Prompt = p
|
||||
case []interface{}:
|
||||
if len(p) > 0 {
|
||||
if s, ok := p[0].(string); ok {
|
||||
vr.Prompt = s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Size
|
||||
size := input.Size
|
||||
if size == "" && raw != nil {
|
||||
if v, ok := raw["size"].(string); ok {
|
||||
size = v
|
||||
}
|
||||
}
|
||||
if size != "" {
|
||||
parts := strings.SplitN(size, "x", 2)
|
||||
if len(parts) == 2 {
|
||||
if wi, err := strconv.Atoi(parts[0]); err == nil {
|
||||
vr.Width = int32(wi)
|
||||
}
|
||||
if hi, err := strconv.Atoi(parts[1]); err == nil {
|
||||
vr.Height = int32(hi)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seconds -> num frames
|
||||
secondsStr := ""
|
||||
if raw != nil {
|
||||
if v, ok := raw["seconds"].(string); ok {
|
||||
secondsStr = v
|
||||
} else if v, ok := raw["seconds"].(float64); ok {
|
||||
secondsStr = fmt.Sprintf("%v", int(v))
|
||||
}
|
||||
}
|
||||
fps := int32(30)
|
||||
if raw != nil {
|
||||
if rawFPS, ok := raw["fps"]; ok {
|
||||
switch rf := rawFPS.(type) {
|
||||
case float64:
|
||||
fps = int32(rf)
|
||||
case string:
|
||||
if fi, err := strconv.Atoi(rf); err == nil {
|
||||
fps = int32(fi)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if secondsStr != "" {
|
||||
if secF, err := strconv.Atoi(secondsStr); err == nil {
|
||||
vr.FPS = fps
|
||||
vr.NumFrames = int32(secF) * fps
|
||||
}
|
||||
}
|
||||
|
||||
// input_reference
|
||||
if raw != nil {
|
||||
if v, ok := raw["input_reference"].(string); ok {
|
||||
vr.StartImage = v
|
||||
}
|
||||
}
|
||||
|
||||
// response format
|
||||
if input.ResponseFormat != nil {
|
||||
if rf, ok := input.ResponseFormat.(string); ok {
|
||||
vr.ResponseFormat = rf
|
||||
}
|
||||
}
|
||||
|
||||
if input.Step != 0 {
|
||||
vr.Step = int32(input.Step)
|
||||
}
|
||||
|
||||
return vr
|
||||
}
|
||||
@@ -152,27 +152,6 @@ func RegisterOpenAIRoutes(app *echo.Echo,
|
||||
app.POST("/v1/images/inpainting", inpaintingHandler, imageMiddleware...)
|
||||
app.POST("/images/inpainting", inpaintingHandler, imageMiddleware...)
|
||||
|
||||
// videos (OpenAI-compatible endpoints mapped to LocalAI video handler)
|
||||
videoHandler := openai.VideoEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig())
|
||||
videoMiddleware := []echo.MiddlewareFunc{
|
||||
traceMiddleware,
|
||||
re.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_VIDEO)),
|
||||
re.SetModelAndConfig(func() schema.LocalAIRequest { return new(schema.OpenAIRequest) }),
|
||||
func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
if err := re.SetOpenAIRequest(c); err != nil {
|
||||
return err
|
||||
}
|
||||
return next(c)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
// OpenAI-style create video endpoint
|
||||
app.POST("/v1/videos", videoHandler, videoMiddleware...)
|
||||
app.POST("/v1/videos/generations", videoHandler, videoMiddleware...)
|
||||
app.POST("/videos", videoHandler, videoMiddleware...)
|
||||
|
||||
// List models
|
||||
app.GET("/v1/models", openai.ListModelsEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig()))
|
||||
app.GET("/models", openai.ListModelsEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig()))
|
||||
|
||||
@@ -135,9 +135,9 @@ async function promptVideo() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Make API request
|
||||
// Make API request to LocalAI endpoint
|
||||
try {
|
||||
const response = await fetch("v1/videos/generations", {
|
||||
const response = await fetch("video", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -219,9 +219,13 @@ async function promptVideo() {
|
||||
`;
|
||||
captionDiv.appendChild(detailsDiv);
|
||||
|
||||
// Button container
|
||||
const buttonContainer = document.createElement("div");
|
||||
buttonContainer.className = "mt-1.5 flex gap-2";
|
||||
|
||||
// Copy prompt button
|
||||
const copyBtn = document.createElement("button");
|
||||
copyBtn.className = "mt-1.5 px-2 py-0.5 text-[10px] bg-[var(--color-primary)] text-white rounded hover:opacity-80";
|
||||
copyBtn.className = "px-2 py-0.5 text-[10px] bg-[var(--color-primary)] text-white rounded hover:opacity-80";
|
||||
copyBtn.innerHTML = '<i class="fas fa-copy mr-1"></i>Copy Prompt';
|
||||
copyBtn.onclick = () => {
|
||||
navigator.clipboard.writeText(prompt).then(() => {
|
||||
@@ -231,7 +235,18 @@ async function promptVideo() {
|
||||
}, 2000);
|
||||
});
|
||||
};
|
||||
captionDiv.appendChild(copyBtn);
|
||||
buttonContainer.appendChild(copyBtn);
|
||||
|
||||
// Download video button
|
||||
const downloadBtn = document.createElement("button");
|
||||
downloadBtn.className = "px-2 py-0.5 text-[10px] bg-[var(--color-primary)] text-white rounded hover:opacity-80";
|
||||
downloadBtn.innerHTML = '<i class="fas fa-download mr-1"></i>Download Video';
|
||||
downloadBtn.onclick = () => {
|
||||
downloadVideo(item, downloadBtn);
|
||||
};
|
||||
buttonContainer.appendChild(downloadBtn);
|
||||
|
||||
captionDiv.appendChild(buttonContainer);
|
||||
|
||||
videoContainer.appendChild(captionDiv);
|
||||
resultDiv.appendChild(videoContainer);
|
||||
@@ -269,6 +284,67 @@ function escapeHtml(text) {
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
// Helper function to download video
|
||||
function downloadVideo(item, button) {
|
||||
try {
|
||||
let videoUrl;
|
||||
let filename = "generated-video.mp4";
|
||||
|
||||
if (item.url) {
|
||||
// If we have a URL, use it directly
|
||||
videoUrl = item.url;
|
||||
// Extract filename from URL if possible
|
||||
const urlParts = item.url.split("/");
|
||||
if (urlParts.length > 0) {
|
||||
const lastPart = urlParts[urlParts.length - 1];
|
||||
if (lastPart && lastPart.includes(".")) {
|
||||
filename = lastPart;
|
||||
}
|
||||
}
|
||||
} else if (item.b64_json) {
|
||||
// Convert base64 to blob
|
||||
const byteCharacters = atob(item.b64_json);
|
||||
const byteNumbers = new Array(byteCharacters.length);
|
||||
for (let i = 0; i < byteCharacters.length; i++) {
|
||||
byteNumbers[i] = byteCharacters.charCodeAt(i);
|
||||
}
|
||||
const byteArray = new Uint8Array(byteNumbers);
|
||||
const blob = new Blob([byteArray], { type: "video/mp4" });
|
||||
videoUrl = URL.createObjectURL(blob);
|
||||
} else {
|
||||
console.error("No video data available for download");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary anchor element to trigger download
|
||||
const link = document.createElement("a");
|
||||
link.href = videoUrl;
|
||||
link.download = filename;
|
||||
link.style.display = "none";
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
|
||||
// Clean up object URL if we created one
|
||||
if (item.b64_json && videoUrl.startsWith("blob:")) {
|
||||
setTimeout(() => URL.revokeObjectURL(videoUrl), 100);
|
||||
}
|
||||
|
||||
// Show feedback
|
||||
const originalHTML = button.innerHTML;
|
||||
button.innerHTML = '<i class="fas fa-check mr-1"></i>Downloaded!';
|
||||
setTimeout(() => {
|
||||
button.innerHTML = originalHTML;
|
||||
}, 2000);
|
||||
} catch (error) {
|
||||
console.error("Error downloading video:", error);
|
||||
button.innerHTML = '<i class="fas fa-exclamation-triangle mr-1"></i>Error';
|
||||
setTimeout(() => {
|
||||
button.innerHTML = '<i class="fas fa-download mr-1"></i>Download Video';
|
||||
}, 2000);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize
|
||||
document.addEventListener("DOMContentLoaded", function() {
|
||||
const input = document.getElementById("input");
|
||||
|
||||
Reference in New Issue
Block a user