mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-05-24 12:49:52 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(importer): unify importing code with CLI (#7299)

Browse Source 
* feat(importer): support ollama and OCI, unify code

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat: support importing from local file

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* support also yaml config files

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Correctly handle local files

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Extract importing errors

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add importer tests

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add integration tests

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chore(UX): improve and specify supported URI formats

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fail if backend does not have a runfile

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Adapt tests

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat(gallery): add cache for galleries

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix(ui): remove handler duplicate

File input handlers are now handled by Alpine.js @change handlers in chat.html.
Removed duplicate listeners to prevent files from being processed twice

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix(ui): be consistent in attachments in the chat

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fail if no importer matches

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix: propagate ops correctly

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fixups

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-11-19 20:52:11 +01:00
committed by GitHub
parent 3152611184
commit 77bbeed57e
26 changed files with 884 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/downloader/uri.go
+47 -30
View File
@@ -19,6 +19,7 @@ import (
"github.com/mudler/LocalAI/pkg/oci"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/mudler/LocalAI/pkg/xio"
"github.com/rs/zerolog/log"
)
@@ -49,17 +50,16 @@ func loadConfig() string {
return HF_ENDPOINT
}
func (uri URI) DownloadWithCallback(basePath string, f func(url string, i []byte) error) error {
return uri.DownloadWithAuthorizationAndCallback(context.Background(), basePath, "", f)
func (uri URI) ReadWithCallback(basePath string, f func(url string, i []byte) error) error {
return uri.ReadWithAuthorizationAndCallback(context.Background(), basePath, "", f)
}
func (uri URI) DownloadWithAuthorizationAndCallback(ctx context.Context, basePath string, authorization string, f func(url string, i []byte) error) error {
func (uri URI) ReadWithAuthorizationAndCallback(ctx context.Context, basePath string, authorization string, f func(url string, i []byte) error) error {
url := uri.ResolveURL()
if strings.HasPrefix(url, LocalPrefix) {
rawURL := strings.TrimPrefix(url, LocalPrefix)
if strings.HasPrefix(string(uri), LocalPrefix) {
// checks if the file is symbolic, and resolve if so - otherwise, this function returns the path unmodified.
resolvedFile, err := filepath.EvalSymlinks(rawURL)
resolvedFile, err := filepath.EvalSymlinks(url)
if err != nil {
return err
}
@@ -175,6 +175,8 @@ func (s URI) LooksLikeOCIFile() bool {
func (s URI) ResolveURL() string {
switch {
case strings.HasPrefix(string(s), LocalPrefix):
return strings.TrimPrefix(string(s), LocalPrefix)
case strings.HasPrefix(string(s), GithubURI2):
repository := strings.Replace(string(s), GithubURI2, "", 1)
@@ -311,11 +313,6 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
return oci.ExtractOCIImage(ctx, img, url, filePath, downloadStatus)
}
// We need to check if url looks like an URL or bail out
if !URI(url).LooksLikeHTTPURL() {
return fmt.Errorf("url %q does not look like an HTTP URL", url)
}
// Check for cancellation before starting
select {
case <-ctx.Done():
@@ -326,6 +323,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
// Check if the file already exists
_, err := os.Stat(filePath)
if err == nil {
log.Debug().Str("filePath", filePath).Msg("[downloader] File already exists")
// File exists, check SHA
if sha != "" {
// Verify SHA
@@ -350,12 +348,12 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
log.Debug().Msgf("File %q already exists. Skipping download", filePath)
return nil
}
} else if !os.IsNotExist(err) {
} else if !os.IsNotExist(err) || !URI(url).LooksLikeHTTPURL() {
// Error occurred while checking file existence
return fmt.Errorf("failed to check file %q existence: %v", filePath, err)
return fmt.Errorf("file %s does not exist (%v) and %s does not look like an HTTP URL", filePath, err, url)
}
log.Info().Msgf("Downloading %q", url)
log.Info().Msgf("Downloading %s", url)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
@@ -365,7 +363,7 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
// save partial download to dedicated file
tmpFilePath := filePath + ".partial"
tmpFileInfo, err := os.Stat(tmpFilePath)
if err == nil {
if err == nil && uri.LooksLikeHTTPURL() {
support, err := uri.checkSeverSupportsRangeHeader()
if err != nil {
return fmt.Errorf("failed to check if uri server supports range header: %v", err)
@@ -383,22 +381,40 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
return fmt.Errorf("failed to check file %q existence: %v", filePath, err)
}
// Start the request
resp, err := http.DefaultClient.Do(req)
if err != nil {
// Check if error is due to context cancellation
if errors.Is(err, context.Canceled) {
// Clean up partial file on cancellation
removePartialFile(tmpFilePath)
return err
var source io.ReadCloser
var contentLength int64
if _, e := os.Stat(uri.ResolveURL()); strings.HasPrefix(string(uri), LocalPrefix) || e == nil {
file, err := os.Open(uri.ResolveURL())
if err != nil {
return fmt.Errorf("failed to open file %q: %v", uri.ResolveURL(), err)
}
return fmt.Errorf("failed to download file %q: %v", filePath, err)
}
defer resp.Body.Close()
l, err := file.Stat()
if err != nil {
return fmt.Errorf("failed to get file size %q: %v", uri.ResolveURL(), err)
}
source = file
contentLength = l.Size()
} else {
// Start the request
resp, err := http.DefaultClient.Do(req)
if err != nil {
// Check if error is due to context cancellation
if errors.Is(err, context.Canceled) {
// Clean up partial file on cancellation
removePartialFile(tmpFilePath)
return err
}
return fmt.Errorf("failed to download file %q: %v", filePath, err)
}
//defer resp.Body.Close()
if resp.StatusCode >= 400 {
return fmt.Errorf("failed to download url %q, invalid status code %d", url, resp.StatusCode)
if resp.StatusCode >= 400 {
return fmt.Errorf("failed to download url %q, invalid status code %d", url, resp.StatusCode)
}
source = resp.Body
contentLength = resp.ContentLength
}
defer source.Close()
// Create parent directory
err = os.MkdirAll(filepath.Dir(filePath), 0750)
@@ -418,14 +434,15 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string
}
progress := &progressWriter{
fileName: tmpFilePath,
total: resp.ContentLength,
total: contentLength,
hash: hash,
fileNo: fileN,
totalFiles: total,
downloadStatus: downloadStatus,
ctx: ctx,
}
_, err = io.Copy(io.MultiWriter(outFile, progress), resp.Body)
_, err = xio.Copy(ctx, io.MultiWriter(outFile, progress), source)
if err != nil {
// Check if error is due to context cancellation
if errors.Is(err, context.Canceled) {
pkg/downloader/uri_test.go
+3 -3
View File
@@ -20,7 +20,7 @@ var _ = Describe("Gallery API tests", func() {
It("parses github with a branch", func() {
uri := URI("github:go-skynet/model-gallery/gpt4all-j.yaml")
Expect(
uri.DownloadWithCallback("", func(url string, i []byte) error {
uri.ReadWithCallback("", func(url string, i []byte) error {
Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml"))
return nil
}),
@@ -30,7 +30,7 @@ var _ = Describe("Gallery API tests", func() {
uri := URI("github:go-skynet/model-gallery/gpt4all-j.yaml@main")
Expect(
uri.DownloadWithCallback("", func(url string, i []byte) error {
uri.ReadWithCallback("", func(url string, i []byte) error {
Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml"))
return nil
}),
@@ -39,7 +39,7 @@ var _ = Describe("Gallery API tests", func() {
It("parses github with urls", func() {
uri := URI("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml")
Expect(
uri.DownloadWithCallback("", func(url string, i []byte) error {
uri.ReadWithCallback("", func(url string, i []byte) error {
Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml"))
return nil
}),
pkg/huggingface-api/client.go
+1 -1
View File
@@ -185,7 +185,7 @@ func (c *Client) ListFiles(repoID string) ([]FileInfo, error) {
func (c *Client) GetFileSHA(repoID, fileName string) (string, error) {
files, err := c.ListFiles(repoID)
if err != nil {
return "", fmt.Errorf("failed to list files: %w", err)
return "", fmt.Errorf("failed to list files while getting SHA: %w", err)
}
for _, file := range files {