mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-02-22 03:09:49 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat: refactor build process, drop embedded backends (#5875)

Browse Source 
* feat: split remaining backends and drop embedded backends

- Drop silero-vad, huggingface, and stores backend from embedded
  binaries
- Refactor Makefile and Dockerfile to avoid building grpc backends
- Drop golang code that was used to embed backends
- Simplify building by using goreleaser

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chore(gallery): be specific with llama-cpp backend templates

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chore(docs): update

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chore(ci): minor fixes

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* chore: drop all ffmpeg references

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix: run protogen-go

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Always enable p2p mode

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Update gorelease file

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix(stores): do not always load

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Fix linting issues

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Simplify

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Mac OS fixup

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-07-22 16:31:04 +02:00
committed by GitHub
parent e29b2c3aff
commit 98e5291afc
118 changed files with 631 additions and 1339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
pkg/assets/extract.go
View File

@@ -1,64 +0,0 @@
package assets
import (
"fmt"
"os"
"path/filepath"
rice "github.com/GeertJohan/go.rice"
"github.com/mudler/LocalAI/pkg/library"
)
const backendAssetsDir = "backend-assets"
func ResolvePath(dir string, paths ...string) string {
return filepath.Join(append([]string{dir, backendAssetsDir}, paths...)...)
}
func ExtractFiles(content *rice.Box, extractDir string) error {
// Create the target directory with backend-assets subdirectory
backendAssetsDir := filepath.Join(extractDir, backendAssetsDir)
err := os.MkdirAll(backendAssetsDir, 0750)
if err != nil {
return fmt.Errorf("failed to create directory: %v", err)
}
// Walk through the rice box and extract files
err = content.Walk("", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Reconstruct the directory structure in the target directory
targetFile := filepath.Join(backendAssetsDir, path)
if info.IsDir() {
// Create the directory in the target directory
err := os.MkdirAll(targetFile, 0750)
if err != nil {
return fmt.Errorf("failed to create directory: %v", err)
}
return nil
}
// Read the file from the rice box
fileData, err := content.Bytes(path)
if err != nil {
return fmt.Errorf("failed to read file: %v", err)
}
// Create the file in the target directory
err = os.WriteFile(targetFile, fileData, 0700)
if err != nil {
return fmt.Errorf("failed to write file: %v", err)
}
return nil
})
// If there is a lib directory, set LD_LIBRARY_PATH to include it
// we might use this mechanism to carry over e.g. Nvidia CUDA libraries
// from the embedded FS to the target directory
library.LoadExtractedLibs(backendAssetsDir)
return err
}

27
pkg/assets/list.go
View File

@@ -1,27 +0,0 @@
package assets
import (
"os"
rice "github.com/GeertJohan/go.rice"
"github.com/rs/zerolog/log"
)
func ListFiles(content *rice.Box) (files []string) {
err := content.Walk("", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
files = append(files, path)
return nil
})
if err != nil {
log.Error().Err(err).Msg("error walking the rice box")
}
return
}

86
pkg/library/dynaload.go
View File

@@ -1,86 +0,0 @@
package library
import (
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"github.com/rs/zerolog/log"
)
/*
This file contains functions to load libraries from the asset directory to keep the business logic clean.
*/
// skipLibraryPath checks if LOCALAI_SKIP_LIBRARY_PATH is set
var skipLibraryPath = os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != ""
// LoadExtractedLibs loads the extracted libraries from the asset dir
func LoadExtractedLibs(dir string) error {
// Skip this if LOCALAI_SKIP_LIBRARY_PATH is set
if skipLibraryPath {
return nil
}
var err error = nil
for _, libDir := range []string{filepath.Join(dir, "lib"), filepath.Join(dir, "lib")} {
err = errors.Join(err, LoadExternal(libDir))
}
return err
}
// LoadLDSO checks if there is a ld.so in the asset dir and if so, prefixes the grpc process with it.
// In linux, if we find a ld.so in the asset dir we prefix it to run with the libs exposed in
// LD_LIBRARY_PATH for more compatibility
// If we don't do this, we might run into stack smash
// See also: https://stackoverflow.com/questions/847179/multiple-glibc-libraries-on-a-single-host/851229#851229
// In this case, we expect a ld.so in the lib asset dir.
// If that's present, we use it to run the grpc backends as supposedly built against
// that specific version of ld.so
func LoadLDSO(assetDir string, args []string, grpcProcess string) ([]string, string) {
if skipLibraryPath {
return args, grpcProcess
}
if runtime.GOOS != "linux" {
return args, grpcProcess
}
// Check if there is a ld.so file in the assetDir, if it does, we need to run the grpc process with it
ldPath := filepath.Join(assetDir, "backend-assets", "lib", "ld.so")
if _, err := os.Stat(ldPath); err == nil {
log.Debug().Msgf("ld.so found")
// We need to run the grpc process with the ld.so
args = append([]string{grpcProcess}, args...)
grpcProcess = ldPath
}
return args, grpcProcess
}
// LoadExternal sets the LD_LIBRARY_PATH to include the given directory
func LoadExternal(dir string) error {
// Skip this if LOCALAI_SKIP_LIBRARY_PATH is set
if skipLibraryPath {
return nil
}
lpathVar := "LD_LIBRARY_PATH"
if runtime.GOOS == "darwin" {
lpathVar = "DYLD_FALLBACK_LIBRARY_PATH" // should it be DYLD_LIBRARY_PATH ?
}
var setErr error = nil
if _, err := os.Stat(dir); err == nil {
ldLibraryPath := os.Getenv(lpathVar)
if ldLibraryPath == "" {
ldLibraryPath = dir
} else {
ldLibraryPath = fmt.Sprintf("%s:%s", ldLibraryPath, dir)
}
setErr = errors.Join(setErr, os.Setenv(lpathVar, ldLibraryPath))
}
return setErr
}

128
pkg/model/initializers.go
View File

@@ -5,18 +5,12 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
grpc "github.com/mudler/LocalAI/pkg/grpc"
"github.com/mudler/LocalAI/pkg/library"
"github.com/mudler/LocalAI/pkg/utils"
"github.com/phayes/freeport"
"github.com/rs/zerolog/log"
"github.com/elliotchance/orderedmap/v2"
)
const (
@@ -51,79 +45,6 @@ const (
LocalStoreBackend = "local-store"
)
func backendPath(assetDir, backend string) string {
return filepath.Join(assetDir, "backend-assets", "grpc", backend)
}
// backendsInAssetDir returns the list of backends in the asset directory
// that should be loaded
func backendsInAssetDir(assetDir string) (map[string][]string, error) {
// Exclude backends from automatic loading
excludeBackends := []string{LocalStoreBackend}
entry, err := os.ReadDir(backendPath(assetDir, ""))
if err != nil {
return nil, err
}
backends := make(map[string][]string)
ENTRY:
for _, e := range entry {
for _, exclude := range excludeBackends {
if e.Name() == exclude {
continue ENTRY
}
}
if e.IsDir() {
continue
}
if strings.HasSuffix(e.Name(), ".log") {
continue
}
backends[e.Name()] = []string{}
}
return backends, nil
}
func orderBackends(backends map[string][]string) ([]string, error) {
// order backends from the asset directory.
// as we scan for backends, we want to keep some order which backends are tried of.
// for example, llama.cpp should be tried first, and we want to keep the huggingface backend at the last.
// sets a priority list - first has more priority
priorityList := []string{}
toTheEnd := []string{
// last has to be huggingface
LCHuggingFaceBackend,
}
// create an ordered map
orderedBackends := orderedmap.NewOrderedMap[string, any]()
// add priorityList first
for _, p := range priorityList {
if _, ok := backends[p]; ok {
orderedBackends.Set(p, backends[p])
}
}
for k, v := range backends {
if !slices.Contains(toTheEnd, k) {
if _, ok := orderedBackends.Get(k); !ok {
orderedBackends.Set(k, v)
}
}
}
for _, t := range toTheEnd {
if _, ok := backends[t]; ok {
orderedBackends.Set(t, backends[t])
}
}
return orderedBackends.Keys(), nil
}
// starts the grpcModelProcess for the backend, and returns a grpc client
// It also loads the model
func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string, string) (*Model, error) {
@@ -177,35 +98,7 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
client = NewModel(modelID, uri, nil)
}
} else {
grpcProcess := backendPath(o.assetDir, backend)
if err := utils.VerifyPath(grpcProcess, o.assetDir); err != nil {
return nil, fmt.Errorf("referring to a backend not in asset dir: %s", err.Error())
}
// Check if the file exists
if _, err := os.Stat(grpcProcess); os.IsNotExist(err) {
return nil, fmt.Errorf("backend not found: %s", grpcProcess)
}
serverAddress, err := getFreeAddress()
if err != nil {
return nil, fmt.Errorf("failed allocating free ports: %s", err.Error())
}
args := []string{}
// Load the ld.so if it exists
args, grpcProcess = library.LoadLDSO(o.assetDir, args, grpcProcess)
// Make sure the process is executable in any circumstance
process, err := ml.startProcess(grpcProcess, modelID, serverAddress, args...)
if err != nil {
return nil, err
}
log.Debug().Msgf("GRPC Service Started")
client = NewModel(modelID, serverAddress, process)
return nil, fmt.Errorf("backend not found: %s", backend)
}
log.Debug().Msgf("Wait for the service to start up")
@@ -259,14 +152,6 @@ func (ml *ModelLoader) grpcModel(backend string, o *Options) func(string, string
}
}
func (ml *ModelLoader) ListAvailableBackends(assetdir string) ([]string, error) {
backends, err := backendsInAssetDir(assetdir)
if err != nil {
return nil, err
}
return orderBackends(backends)
}
func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err error) {
o := NewOptions(opts...)
@@ -346,17 +231,18 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
var err error
// get backends embedded in the binary
autoLoadBackends, err := ml.ListAvailableBackends(o.assetDir)
if err != nil {
ml.Close() // we failed, release the lock
return nil, err
}
autoLoadBackends := []string{}
// append externalBackends supplied by the user via the CLI
for b := range ml.GetAllExternalBackends(o) {
autoLoadBackends = append(autoLoadBackends, b)
}
if len(autoLoadBackends) == 0 {
log.Error().Msg("No backends found")
return nil, fmt.Errorf("no backends found")
}
log.Debug().Msgf("Loading from the following backends (in order): %+v", autoLoadBackends)
log.Info().Msgf("Trying to load the model '%s' with the backend '%s'", o.modelID, autoLoadBackends)

7
pkg/model/loader_options.go
View File

@@ -10,7 +10,6 @@ type Options struct {
backendString string
model string
modelID string
assetDir string
context context.Context
gRPCOptions *pb.ModelOptions
@@ -75,12 +74,6 @@ func WithLoadGRPCLoadModelOpts(opts *pb.ModelOptions) Option {
}
}
func WithAssetDir(assetDir string) Option {
return func(o *Options) {
o.assetDir = assetDir
}
}
func WithContext(ctx context.Context) Option {
return func(o *Options) {
o.context = ctx