From 79a41a5e0729c446a5c50be4b68e49cf71589c69 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 28 Aug 2025 19:11:02 +0200 Subject: [PATCH] fix: register backends to model-loader during installation (#6159) Signed-off-by: Ettore Di Giacinto --- core/application/startup.go | 8 +++---- core/backend/llm.go | 2 +- core/cli/backends.go | 4 +++- core/cli/models.go | 4 +++- core/gallery/backends.go | 10 ++++----- core/gallery/backends_test.go | 36 ++++++++++++++---------------- core/gallery/models.go | 4 +++- core/gallery/models_test.go | 2 +- core/services/backends.go | 5 +---- core/services/models.go | 26 +++++++++++---------- core/startup/backend_preload.go | 12 +++++----- core/startup/model_preload.go | 11 ++++----- core/startup/model_preload_test.go | 27 ++++++++++++---------- pkg/model/initializers.go | 2 ++ 14 files changed, 82 insertions(+), 71 deletions(-) diff --git a/core/application/startup.go b/core/application/startup.go index 69238d54f..a497e0e60 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -56,12 +56,12 @@ func New(opts ...config.AppOption) (*Application, error) { } } - if err := coreStartup.InstallModels(options.Galleries, options.BackendGalleries, options.SystemState, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil { + if err := coreStartup.InstallModels(options.Galleries, options.BackendGalleries, options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, nil, options.ModelsURL...); err != nil { log.Error().Err(err).Msg("error installing models") } for _, backend := range options.ExternalBackends { - if err := coreStartup.InstallExternalBackends(options.BackendGalleries, options.SystemState, nil, backend, "", ""); err != nil { + if err := coreStartup.InstallExternalBackends(options.BackendGalleries, options.SystemState, application.ModelLoader(), nil, backend, "", ""); err != nil { log.Error().Err(err).Msg("error installing external backend") } } @@ -87,13 +87,13 @@ func New(opts ...config.AppOption) (*Application, error) { } if options.PreloadJSONModels != "" { - if err := services.ApplyGalleryFromString(options.SystemState, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels); err != nil { + if err := services.ApplyGalleryFromString(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadJSONModels); err != nil { return nil, err } } if options.PreloadModelsFromPath != "" { - if err := services.ApplyGalleryFromFile(options.SystemState, options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath); err != nil { + if err := services.ApplyGalleryFromFile(options.SystemState, application.ModelLoader(), options.EnforcePredownloadScans, options.AutoloadBackendGalleries, options.Galleries, options.BackendGalleries, options.PreloadModelsFromPath); err != nil { return nil, err } } diff --git a/core/backend/llm.go b/core/backend/llm.go index a74141fe6..ffc714975 100644 --- a/core/backend/llm.go +++ b/core/backend/llm.go @@ -47,7 +47,7 @@ func ModelInference(ctx context.Context, s string, messages []schema.Message, im if !slices.Contains(modelNames, c.Name) { utils.ResetDownloadTimers() // if we failed to load the model, we try to download it - err := gallery.InstallModelFromGallery(o.Galleries, o.BackendGalleries, o.SystemState, c.Name, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries) + err := gallery.InstallModelFromGallery(o.Galleries, o.BackendGalleries, o.SystemState, loader, c.Name, gallery.GalleryModel{}, utils.DisplayDownloadFunction, o.EnforcePredownloadScans, o.AutoloadBackendGalleries) if err != nil { log.Error().Err(err).Msgf("failed to install model %q from gallery", modelFile) //return nil, err diff --git a/core/cli/backends.go b/core/cli/backends.go index 37d719c14..666f1eb0b 100644 --- a/core/cli/backends.go +++ b/core/cli/backends.go @@ -6,6 +6,7 @@ import ( cliContext "github.com/mudler/LocalAI/core/cli/context" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/core/gallery" @@ -100,7 +101,8 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error { } } - err = startup.InstallExternalBackends(galleries, systemState, progressCallback, bi.BackendArgs, bi.Name, bi.Alias) + modelLoader := model.NewModelLoader(systemState, true) + err = startup.InstallExternalBackends(galleries, systemState, modelLoader, progressCallback, bi.BackendArgs, bi.Name, bi.Alias) if err != nil { return err } diff --git a/core/cli/models.go b/core/cli/models.go index 9fe916383..2b8da49c4 100644 --- a/core/cli/models.go +++ b/core/cli/models.go @@ -11,6 +11,7 @@ import ( "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/startup" "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/rs/zerolog/log" "github.com/schollz/progressbar/v3" @@ -125,7 +126,8 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error { log.Info().Str("model", modelName).Str("license", model.License).Msg("installing model") } - err = startup.InstallModels(galleries, backendGalleries, systemState, !mi.DisablePredownloadScan, mi.AutoloadBackendGalleries, progressCallback, modelName) + modelLoader := model.NewModelLoader(systemState, true) + err = startup.InstallModels(galleries, backendGalleries, systemState, modelLoader, !mi.DisablePredownloadScan, mi.AutoloadBackendGalleries, progressCallback, modelName) if err != nil { return err } diff --git a/core/gallery/backends.go b/core/gallery/backends.go index f7af7bbae..b198a7f80 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -59,7 +59,7 @@ func writeBackendMetadata(backendPath string, metadata *BackendMetadata) error { } // Installs a model from the gallery -func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.SystemState, name string, downloadStatus func(string, string, string, float64), force bool) error { +func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.SystemState, modelLoader *model.ModelLoader, name string, downloadStatus func(string, string, string, float64), force bool) error { if !force { // check if we already have the backend installed backends, err := ListSystemBackends(systemState) @@ -99,7 +99,7 @@ func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.S log.Debug().Str("name", name).Str("bestBackend", bestBackend.Name).Msg("Installing backend from meta backend") // Then, let's install the best backend - if err := InstallBackend(systemState, bestBackend, downloadStatus); err != nil { + if err := InstallBackend(systemState, modelLoader, bestBackend, downloadStatus); err != nil { return err } @@ -124,10 +124,10 @@ func InstallBackendFromGallery(galleries []config.Gallery, systemState *system.S return nil } - return InstallBackend(systemState, backend, downloadStatus) + return InstallBackend(systemState, modelLoader, backend, downloadStatus) } -func InstallBackend(systemState *system.SystemState, config *GalleryBackend, downloadStatus func(string, string, string, float64)) error { +func InstallBackend(systemState *system.SystemState, modelLoader *model.ModelLoader, config *GalleryBackend, downloadStatus func(string, string, string, float64)) error { // Create base path if it doesn't exist err := os.MkdirAll(systemState.Backend.BackendsPath, 0750) if err != nil { @@ -185,7 +185,7 @@ func InstallBackend(systemState *system.SystemState, config *GalleryBackend, dow return fmt.Errorf("failed to write metadata for backend %q: %v", name, err) } - return nil + return RegisterBackends(systemState, modelLoader) } func DeleteBackendFromSystem(systemState *system.SystemState, name string) error { diff --git a/core/gallery/backends_test.go b/core/gallery/backends_test.go index ea3462011..d9cec0841 100644 --- a/core/gallery/backends_test.go +++ b/core/gallery/backends_test.go @@ -7,6 +7,7 @@ import ( "runtime" "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -19,8 +20,10 @@ const ( var _ = Describe("Gallery Backends", func() { var ( - tempDir string - galleries []config.Gallery + tempDir string + galleries []config.Gallery + ml *model.ModelLoader + systemState *system.SystemState ) BeforeEach(func() { @@ -35,6 +38,9 @@ var _ = Describe("Gallery Backends", func() { URL: "https://gist.githubusercontent.com/mudler/71d5376bc2aa168873fa519fa9f4bd56/raw/0557f9c640c159fa8e4eab29e8d98df6a3d6e80f/backend-gallery.yaml", }, } + systemState, err = system.GetSystemState(system.WithBackendPath(tempDir)) + Expect(err).NotTo(HaveOccurred()) + ml = model.NewModelLoader(systemState, true) }) AfterEach(func() { @@ -43,21 +49,13 @@ var _ = Describe("Gallery Backends", func() { Describe("InstallBackendFromGallery", func() { It("should return error when backend is not found", func() { - systemState, err := system.GetSystemState( - system.WithBackendPath(tempDir), - ) - Expect(err).NotTo(HaveOccurred()) - err = InstallBackendFromGallery(galleries, systemState, "non-existent", nil, true) + err := InstallBackendFromGallery(galleries, systemState, ml, "non-existent", nil, true) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no backend found with name \"non-existent\"")) }) It("should install backend from gallery", func() { - systemState, err := system.GetSystemState( - system.WithBackendPath(tempDir), - ) - Expect(err).NotTo(HaveOccurred()) - err = InstallBackendFromGallery(galleries, systemState, "test-backend", nil, true) + err := InstallBackendFromGallery(galleries, systemState, ml, "test-backend", nil, true) Expect(err).ToNot(HaveOccurred()) Expect(filepath.Join(tempDir, "test-backend", "run.sh")).To(BeARegularFile()) }) @@ -233,7 +231,7 @@ var _ = Describe("Gallery Backends", func() { VRAM: 1000000000000, Backend: system.Backend{BackendsPath: tempDir}, } - err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, "meta-backend", nil, true) + err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true) Expect(err).NotTo(HaveOccurred()) metaBackendPath := filepath.Join(tempDir, "meta-backend") @@ -313,7 +311,7 @@ var _ = Describe("Gallery Backends", func() { VRAM: 1000000000000, Backend: system.Backend{BackendsPath: tempDir}, } - err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, "meta-backend", nil, true) + err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true) Expect(err).NotTo(HaveOccurred()) metaBackendPath := filepath.Join(tempDir, "meta-backend") @@ -397,7 +395,7 @@ var _ = Describe("Gallery Backends", func() { VRAM: 1000000000000, Backend: system.Backend{BackendsPath: tempDir}, } - err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, "meta-backend", nil, true) + err = InstallBackendFromGallery([]config.Gallery{gallery}, nvidiaSystemState, ml, "meta-backend", nil, true) Expect(err).NotTo(HaveOccurred()) metaBackendPath := filepath.Join(tempDir, "meta-backend") @@ -496,7 +494,7 @@ var _ = Describe("Gallery Backends", func() { system.WithBackendPath(newPath), ) Expect(err).NotTo(HaveOccurred()) - err = InstallBackend(systemState, &backend, nil) + err = InstallBackend(systemState, ml, &backend, nil) Expect(err).To(HaveOccurred()) // Will fail due to invalid URI, but path should be created Expect(newPath).To(BeADirectory()) }) @@ -528,7 +526,7 @@ var _ = Describe("Gallery Backends", func() { system.WithBackendPath(tempDir), ) Expect(err).NotTo(HaveOccurred()) - err = InstallBackend(systemState, &backend, nil) + err = InstallBackend(systemState, ml, &backend, nil) Expect(err).ToNot(HaveOccurred()) Expect(filepath.Join(tempDir, "test-backend", "metadata.json")).To(BeARegularFile()) dat, err := os.ReadFile(filepath.Join(tempDir, "test-backend", "metadata.json")) @@ -561,7 +559,7 @@ var _ = Describe("Gallery Backends", func() { Expect(filepath.Join(tempDir, "test-backend", "metadata.json")).ToNot(BeARegularFile()) - err = InstallBackend(systemState, &backend, nil) + err = InstallBackend(systemState, ml, &backend, nil) Expect(err).ToNot(HaveOccurred()) Expect(filepath.Join(tempDir, "test-backend", "metadata.json")).To(BeARegularFile()) }) @@ -582,7 +580,7 @@ var _ = Describe("Gallery Backends", func() { system.WithBackendPath(tempDir), ) Expect(err).NotTo(HaveOccurred()) - err = InstallBackend(systemState, &backend, nil) + err = InstallBackend(systemState, ml, &backend, nil) Expect(err).ToNot(HaveOccurred()) Expect(filepath.Join(tempDir, "test-backend", "metadata.json")).To(BeARegularFile()) diff --git a/core/gallery/models.go b/core/gallery/models.go index 57d07be57..a1abe0ee1 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -11,6 +11,7 @@ import ( "github.com/mudler/LocalAI/core/config" lconfig "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" @@ -73,6 +74,7 @@ type PromptTemplate struct { func InstallModelFromGallery( modelGalleries, backendGalleries []config.Gallery, systemState *system.SystemState, + modelLoader *model.ModelLoader, name string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool) error { applyModel := func(model *GalleryModel) error { @@ -131,7 +133,7 @@ func InstallModelFromGallery( if automaticallyInstallBackend && installedModel.Backend != "" { log.Debug().Msgf("Installing backend %q", installedModel.Backend) - if err := InstallBackendFromGallery(backendGalleries, systemState, installedModel.Backend, downloadStatus, false); err != nil { + if err := InstallBackendFromGallery(backendGalleries, systemState, modelLoader, installedModel.Backend, downloadStatus, false); err != nil { return err } } diff --git a/core/gallery/models_test.go b/core/gallery/models_test.go index f259f0743..3ae76c203 100644 --- a/core/gallery/models_test.go +++ b/core/gallery/models_test.go @@ -88,7 +88,7 @@ var _ = Describe("Model test", func() { Expect(models[0].URL).To(Equal(bertEmbeddingsURL)) Expect(models[0].Installed).To(BeFalse()) - err = InstallModelFromGallery(galleries, []config.Gallery{}, systemState, "test@bert", GalleryModel{}, func(s1, s2, s3 string, f float64) {}, true, true) + err = InstallModelFromGallery(galleries, []config.Gallery{}, systemState, nil, "test@bert", GalleryModel{}, func(s1, s2, s3 string, f float64) {}, true, true) Expect(err).ToNot(HaveOccurred()) dat, err := os.ReadFile(filepath.Join(tempdir, "bert.yaml")) diff --git a/core/services/backends.go b/core/services/backends.go index 73001f97d..7295734df 100644 --- a/core/services/backends.go +++ b/core/services/backends.go @@ -25,10 +25,7 @@ func (g *GalleryService) backendHandler(op *GalleryOp[gallery.GalleryBackend], s } else { log.Warn().Msgf("installing backend %s", op.GalleryElementName) log.Debug().Msgf("backend galleries: %v", g.appConfig.BackendGalleries) - err = gallery.InstallBackendFromGallery(g.appConfig.BackendGalleries, systemState, op.GalleryElementName, progressCallback, true) - if err == nil { - err = gallery.RegisterBackends(systemState, g.modelLoader) - } + err = gallery.InstallBackendFromGallery(g.appConfig.BackendGalleries, systemState, g.modelLoader, op.GalleryElementName, progressCallback, true) } if err != nil { log.Error().Err(err).Msgf("error installing backend %s", op.GalleryElementName) diff --git a/core/services/models.go b/core/services/models.go index 032857a5b..43696f455 100644 --- a/core/services/models.go +++ b/core/services/models.go @@ -6,6 +6,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" "gopkg.in/yaml.v2" @@ -22,7 +23,7 @@ func (g *GalleryService) modelHandler(op *GalleryOp[gallery.GalleryModel], cl *c utils.DisplayDownloadFunction(fileName, current, total, percentage) } - err := processModelOperation(op, systemState, g.appConfig.EnforcePredownloadScans, g.appConfig.AutoloadBackendGalleries, progressCallback) + err := processModelOperation(op, systemState, g.modelLoader, g.appConfig.EnforcePredownloadScans, g.appConfig.AutoloadBackendGalleries, progressCallback) if err != nil { return err } @@ -49,7 +50,7 @@ func (g *GalleryService) modelHandler(op *GalleryOp[gallery.GalleryModel], cl *c return nil } -func installModelFromRemoteConfig(systemState *system.SystemState, req gallery.GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool, backendGalleries []config.Gallery) error { +func installModelFromRemoteConfig(systemState *system.SystemState, modelLoader *model.ModelLoader, req gallery.GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend bool, backendGalleries []config.Gallery) error { config, err := gallery.GetGalleryConfigFromURL[gallery.ModelConfig](req.URL, systemState.Model.ModelsPath) if err != nil { return err @@ -63,7 +64,7 @@ func installModelFromRemoteConfig(systemState *system.SystemState, req gallery.G } if automaticallyInstallBackend && installedModel.Backend != "" { - if err := gallery.InstallBackendFromGallery(backendGalleries, systemState, installedModel.Backend, downloadStatus, false); err != nil { + if err := gallery.InstallBackendFromGallery(backendGalleries, systemState, modelLoader, installedModel.Backend, downloadStatus, false); err != nil { return err } } @@ -76,22 +77,22 @@ type galleryModel struct { ID string `json:"id"` } -func processRequests(systemState *system.SystemState, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, requests []galleryModel) error { +func processRequests(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, requests []galleryModel) error { var err error for _, r := range requests { utils.ResetDownloadTimers() if r.ID == "" { - err = installModelFromRemoteConfig(systemState, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, backendGalleries) + err = installModelFromRemoteConfig(systemState, modelLoader, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, backendGalleries) } else { err = gallery.InstallModelFromGallery( - galleries, backendGalleries, systemState, r.ID, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend) + galleries, backendGalleries, systemState, modelLoader, r.ID, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend) } } return err } -func ApplyGalleryFromFile(systemState *system.SystemState, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string) error { +func ApplyGalleryFromFile(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string) error { dat, err := os.ReadFile(s) if err != nil { return err @@ -102,23 +103,24 @@ func ApplyGalleryFromFile(systemState *system.SystemState, enforceScan, automati return err } - return processRequests(systemState, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests) + return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests) } -func ApplyGalleryFromString(systemState *system.SystemState, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string) error { +func ApplyGalleryFromString(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, s string) error { var requests []galleryModel err := json.Unmarshal([]byte(s), &requests) if err != nil { return err } - return processRequests(systemState, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests) + return processRequests(systemState, modelLoader, enforceScan, automaticallyInstallBackend, galleries, backendGalleries, requests) } // processModelOperation handles the installation or deletion of a model func processModelOperation( op *GalleryOp[gallery.GalleryModel], systemState *system.SystemState, + modelLoader *model.ModelLoader, enforcePredownloadScans bool, automaticallyInstallBackend bool, progressCallback func(string, string, string, float64), @@ -130,7 +132,7 @@ func processModelOperation( // if the request contains a gallery name, we apply the gallery from the gallery list if op.GalleryElementName != "" { - return gallery.InstallModelFromGallery(op.Galleries, op.BackendGalleries, systemState, op.GalleryElementName, op.Req, progressCallback, enforcePredownloadScans, automaticallyInstallBackend) + return gallery.InstallModelFromGallery(op.Galleries, op.BackendGalleries, systemState, modelLoader, op.GalleryElementName, op.Req, progressCallback, enforcePredownloadScans, automaticallyInstallBackend) // } else if op.ConfigURL != "" { // err := startup.InstallModels(op.Galleries, modelPath, enforcePredownloadScans, progressCallback, op.ConfigURL) // if err != nil { @@ -138,6 +140,6 @@ func processModelOperation( // } // return cl.Preload(modelPath) } else { - return installModelFromRemoteConfig(systemState, op.Req, progressCallback, enforcePredownloadScans, automaticallyInstallBackend, op.BackendGalleries) + return installModelFromRemoteConfig(systemState, modelLoader, op.Req, progressCallback, enforcePredownloadScans, automaticallyInstallBackend, op.BackendGalleries) } } diff --git a/core/startup/backend_preload.go b/core/startup/backend_preload.go index 04296b406..bf9c4ec90 100644 --- a/core/startup/backend_preload.go +++ b/core/startup/backend_preload.go @@ -8,11 +8,12 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/rs/zerolog/log" ) -func InstallExternalBackends(galleries []config.Gallery, systemState *system.SystemState, downloadStatus func(string, string, string, float64), backend, name, alias string) error { +func InstallExternalBackends(galleries []config.Gallery, systemState *system.SystemState, modelLoader *model.ModelLoader, downloadStatus func(string, string, string, float64), backend, name, alias string) error { uri := downloader.URI(backend) switch { case uri.LooksLikeDir(): @@ -20,7 +21,7 @@ func InstallExternalBackends(galleries []config.Gallery, systemState *system.Sys name = filepath.Base(backend) } log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from path") - if err := gallery.InstallBackend(systemState, &gallery.GalleryBackend{ + if err := gallery.InstallBackend(systemState, modelLoader, &gallery.GalleryBackend{ Metadata: gallery.Metadata{ Name: name, }, @@ -34,7 +35,7 @@ func InstallExternalBackends(galleries []config.Gallery, systemState *system.Sys return fmt.Errorf("specifying a name is required for OCI images") } log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from OCI image") - if err := gallery.InstallBackend(systemState, &gallery.GalleryBackend{ + if err := gallery.InstallBackend(systemState, modelLoader, &gallery.GalleryBackend{ Metadata: gallery.Metadata{ Name: name, }, @@ -52,7 +53,7 @@ func InstallExternalBackends(galleries []config.Gallery, systemState *system.Sys name = strings.TrimSuffix(name, filepath.Ext(name)) log.Info().Str("backend", backend).Str("name", name).Msg("Installing backend from OCI image") - if err := gallery.InstallBackend(systemState, &gallery.GalleryBackend{ + if err := gallery.InstallBackend(systemState, modelLoader, &gallery.GalleryBackend{ Metadata: gallery.Metadata{ Name: name, }, @@ -65,10 +66,11 @@ func InstallExternalBackends(galleries []config.Gallery, systemState *system.Sys if name != "" || alias != "" { return fmt.Errorf("specifying a name or alias is not supported for this backend") } - err := gallery.InstallBackendFromGallery(galleries, systemState, backend, downloadStatus, true) + err := gallery.InstallBackendFromGallery(galleries, systemState, modelLoader, backend, downloadStatus, true) if err != nil { return fmt.Errorf("error installing backend %s: %w", backend, err) } + } return nil diff --git a/core/startup/model_preload.go b/core/startup/model_preload.go index aa4ef4d15..a98e0592b 100644 --- a/core/startup/model_preload.go +++ b/core/startup/model_preload.go @@ -11,6 +11,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" "github.com/rs/zerolog/log" @@ -24,7 +25,7 @@ const ( // InstallModels will preload models from the given list of URLs and galleries // It will download the model if it is not already present in the model path // It will also try to resolve if the model is an embedded model YAML configuration -func InstallModels(galleries, backendGalleries []config.Gallery, systemState *system.SystemState, enforceScan, autoloadBackendGalleries bool, downloadStatus func(string, string, string, float64), models ...string) error { +func InstallModels(galleries, backendGalleries []config.Gallery, systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, autoloadBackendGalleries bool, downloadStatus func(string, string, string, float64), models ...string) error { // create an error that groups all errors var err error @@ -47,7 +48,7 @@ func InstallModels(galleries, backendGalleries []config.Gallery, systemState *sy return nil } - if err := gallery.InstallBackendFromGallery(backendGalleries, systemState, model.Backend, downloadStatus, false); err != nil { + if err := gallery.InstallBackendFromGallery(backendGalleries, systemState, modelLoader, model.Backend, downloadStatus, false); err != nil { log.Error().Err(err).Str("backend", model.Backend).Msg("error installing backend") return err } @@ -147,7 +148,7 @@ func InstallModels(galleries, backendGalleries []config.Gallery, systemState *sy } } else { // Check if it's a model gallery, or print a warning - e, found := installModel(galleries, backendGalleries, url, systemState, downloadStatus, enforceScan, autoloadBackendGalleries) + e, found := installModel(galleries, backendGalleries, url, systemState, modelLoader, downloadStatus, enforceScan, autoloadBackendGalleries) if e != nil && found { log.Error().Err(err).Msgf("[startup] failed installing model '%s'", url) err = errors.Join(err, e) @@ -161,7 +162,7 @@ func InstallModels(galleries, backendGalleries []config.Gallery, systemState *sy return err } -func installModel(galleries, backendGalleries []config.Gallery, modelName string, systemState *system.SystemState, downloadStatus func(string, string, string, float64), enforceScan, autoloadBackendGalleries bool) (error, bool) { +func installModel(galleries, backendGalleries []config.Gallery, modelName string, systemState *system.SystemState, modelLoader *model.ModelLoader, downloadStatus func(string, string, string, float64), enforceScan, autoloadBackendGalleries bool) (error, bool) { models, err := gallery.AvailableGalleryModels(galleries, systemState) if err != nil { return err, false @@ -177,7 +178,7 @@ func installModel(galleries, backendGalleries []config.Gallery, modelName string } log.Info().Str("model", modelName).Str("license", model.License).Msg("installing model") - err = gallery.InstallModelFromGallery(galleries, backendGalleries, systemState, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries) + err = gallery.InstallModelFromGallery(galleries, backendGalleries, systemState, modelLoader, modelName, gallery.GalleryModel{}, downloadStatus, enforceScan, autoloadBackendGalleries) if err != nil { return err, true } diff --git a/core/startup/model_preload_test.go b/core/startup/model_preload_test.go index a15e13bec..16096f41b 100644 --- a/core/startup/model_preload_test.go +++ b/core/startup/model_preload_test.go @@ -7,6 +7,7 @@ import ( "github.com/mudler/LocalAI/core/config" . "github.com/mudler/LocalAI/core/startup" + "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" . "github.com/onsi/ginkgo/v2" @@ -14,18 +15,25 @@ import ( ) var _ = Describe("Preload test", func() { + var tmpdir string + var systemState *system.SystemState + var ml *model.ModelLoader + + BeforeEach(func() { + var err error + tmpdir, err = os.MkdirTemp("", "") + Expect(err).ToNot(HaveOccurred()) + systemState, err = system.GetSystemState(system.WithModelPath(tmpdir)) + Expect(err).ToNot(HaveOccurred()) + ml = model.NewModelLoader(systemState, true) + }) Context("Preloading from strings", func() { It("loads from embedded full-urls", func() { - tmpdir, err := os.MkdirTemp("", "") - Expect(err).ToNot(HaveOccurred()) url := "https://raw.githubusercontent.com/mudler/LocalAI-examples/main/configurations/phi-2.yaml" fileName := fmt.Sprintf("%s.yaml", "phi-2") - systemState, err := system.GetSystemState(system.WithModelPath(tmpdir)) - Expect(err).ToNot(HaveOccurred()) - - InstallModels([]config.Gallery{}, []config.Gallery{}, systemState, true, true, nil, url) + InstallModels([]config.Gallery{}, []config.Gallery{}, systemState, ml, true, true, nil, url) resultFile := filepath.Join(tmpdir, fileName) @@ -35,15 +43,10 @@ var _ = Describe("Preload test", func() { Expect(string(content)).To(ContainSubstring("name: phi-2")) }) It("downloads from urls", func() { - tmpdir, err := os.MkdirTemp("", "") - Expect(err).ToNot(HaveOccurred()) url := "huggingface://TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/tinyllama-1.1b-chat-v0.3.Q2_K.gguf" fileName := fmt.Sprintf("%s.gguf", "tinyllama-1.1b-chat-v0.3.Q2_K") - systemState, err := system.GetSystemState(system.WithModelPath(tmpdir)) - Expect(err).ToNot(HaveOccurred()) - - err = InstallModels([]config.Gallery{}, []config.Gallery{}, systemState, true, true, nil, url) + err := InstallModels([]config.Gallery{}, []config.Gallery{}, systemState, ml, true, true, nil, url) Expect(err).ToNot(HaveOccurred()) resultFile := filepath.Join(tmpdir, fileName) diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index dc60f98d2..98e424885 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -98,6 +98,7 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string client = NewModel(modelID, uri, nil) } } else { + log.Error().Msgf("Backend not found: %s", backend) return nil, fmt.Errorf("backend not found: %s", backend) } @@ -172,6 +173,7 @@ func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err e model, err := ml.LoadModel(o.modelID, o.model, ml.grpcModel(backend, o)) if err != nil { + log.Error().Str("modelID", o.modelID).Err(err).Msgf("Failed to load model %s with backend %s", o.modelID, o.backendString) return nil, err }