mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-01-06 10:39:55 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(p2p): automatically install llama-cpp for p2p workers (#6199)

Browse Source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Signed-off-by: Ettore Di Giacinto <mudler@users.noreply.github.com>
This commit is contained in:
Ettore Di Giacinto
2025-09-04 21:57:39 +02:00
committed by GitHub
parent 3dc86b247d
commit 302958efd6
3 changed files with 20 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
core/cli/worker/worker.go
View File

@@ -2,6 +2,7 @@ package worker
type WorkerFlags struct {
BackendsPath string `env:"LOCALAI_BACKENDS_PATH,BACKENDS_PATH" type:"path" default:"${basepath}/backends" help:"Path containing backends used for inferencing" group:"backends"`
BackendGalleries string `env:"LOCALAI_BACKEND_GALLERIES,BACKEND_GALLERIES" help:"JSON list of backend galleries" group:"backends" default:"${backends}"`
BackendsSystemPath string `env:"LOCALAI_BACKENDS_SYSTEM_PATH,BACKEND_SYSTEM_PATH" type:"path" default:"/usr/share/localai/backends" help:"Path containing system backends used for inferencing" group:"backends"`
ExtraLLamaCPPArgs string `name:"llama-cpp-args" env:"LOCALAI_EXTRA_LLAMA_CPP_ARGS,EXTRA_LLAMA_CPP_ARGS" help:"Extra arguments to pass to llama-cpp-rpc-server"`
}

22
core/cli/worker/worker_llamacpp.go
View File

@@ -1,6 +1,7 @@
package worker
import (
"encoding/json"
"errors"
"fmt"
"os"
@@ -9,8 +10,10 @@ import (
"syscall"
cliContext "github.com/mudler/LocalAI/core/cli/context"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/cli/signals"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/model"
"github.com/mudler/LocalAI/pkg/system"
"github.com/rs/zerolog/log"
)
@@ -21,9 +24,10 @@ type LLamaCPP struct {
const (
llamaCPPRPCBinaryName = "llama-cpp-rpc-server"
llamaCPPGalleryName = "llama-cpp"
)
func findLLamaCPPBackend(systemState *system.SystemState) (string, error) {
func findLLamaCPPBackend(galleries string, systemState *system.SystemState) (string, error) {
backends, err := gallery.ListSystemBackends(systemState)
if err != nil {
log.Warn().Msgf("Failed listing system backends: %s", err)
@@ -31,9 +35,19 @@ func findLLamaCPPBackend(systemState *system.SystemState) (string, error) {
}
log.Debug().Msgf("System backends: %v", backends)
backend, ok := backends.Get("llama-cpp")
backend, ok := backends.Get(llamaCPPGalleryName)
if !ok {
return "", errors.New("llama-cpp backend not found, install it first")
ml := model.NewModelLoader(systemState, true)
var gals []config.Gallery
if err := json.Unmarshal([]byte(galleries), &gals); err != nil {
log.Error().Err(err).Msg("failed loading galleries")
return "", err
}
err := gallery.InstallBackendFromGallery(gals, systemState, ml, llamaCPPGalleryName, nil, true)
if err != nil {
log.Error().Err(err).Msg("llama-cpp backend not found, failed to install it")
return "", err
}
}
backendPath := filepath.Dir(backend.RunFile)
@@ -62,7 +76,7 @@ func (r *LLamaCPP) Run(ctx *cliContext.Context) error {
if err != nil {
return err
}
grpcProcess, err := findLLamaCPPBackend(systemState)
grpcProcess, err := findLLamaCPPBackend(r.BackendGalleries, systemState)
if err != nil {
return err
}

2
core/cli/worker/worker_p2p.go
View File

@@ -70,7 +70,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
for {
log.Info().Msgf("Starting llama-cpp-rpc-server on '%s:%d'", address, port)
grpcProcess, err := findLLamaCPPBackend(systemState)
grpcProcess, err := findLLamaCPPBackend(r.BackendGalleries, systemState)
if err != nil {
log.Error().Err(err).Msg("Failed to find llama-cpp-rpc-server")
return