feat(ui): runtime settings (#7320)

* feat(ui): add watchdog settings

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Do not re-read env

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Some refactor, move other settings to runtime (p2p)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add API Keys handling

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Allow to disable runtime settings

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Documentation

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Small fixups

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* show MCP toggle in index

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Drop context default

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Dockerfile

@@ -332,6 +332,6 @@ RUN mkdir -p /models /backends
 HEALTHCHECK --interval=1m --timeout=10m --retries=10 \
   CMD curl -f ${HEALTHCHECK_ENDPOINT} || exit 1
 
-VOLUME /models /backends
+VOLUME /models /backends /configuration
 EXPOSE 8080
 ENTRYPOINT [ "/entrypoint.sh" ]

core/application/application.go

@@ -1,6 +1,9 @@
 package application
 
 import (
+	"context"
+	"sync"
+
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/services"
 	"github.com/mudler/LocalAI/core/templates"
@@ -11,8 +14,14 @@ type Application struct {
 	backendLoader      *config.ModelConfigLoader
 	modelLoader        *model.ModelLoader
 	applicationConfig  *config.ApplicationConfig
+	startupConfig      *config.ApplicationConfig // Stores original config from env vars (before file loading)
 	templatesEvaluator *templates.Evaluator
 	galleryService     *services.GalleryService
+	watchdogMutex      sync.Mutex
+	watchdogStop       chan bool
+	p2pMutex           sync.Mutex
+	p2pCtx             context.Context
+	p2pCancel          context.CancelFunc
 }
 
 func newApplication(appConfig *config.ApplicationConfig) *Application {
@@ -44,6 +53,11 @@ func (a *Application) GalleryService() *services.GalleryService {
 	return a.galleryService
 }
 
+// StartupConfig returns the original startup configuration (from env vars, before file loading)
+func (a *Application) StartupConfig() *config.ApplicationConfig {
+	return a.startupConfig
+}
+
 func (a *Application) start() error {
 	galleryService := services.NewGalleryService(a.ApplicationConfig(), a.ModelLoader())
 	err := galleryService.Start(a.ApplicationConfig().Context, a.ModelConfigLoader(), a.ApplicationConfig().SystemState)

core/application/config_file_watcher.go

@@ -1,180 +1,343 @@
-package application
+package application
-
+
-import (
+import (
-	"encoding/json"
+	"encoding/json"
-	"fmt"
+	"fmt"
-	"os"
+	"os"
-	"path"
+	"path"
-	"path/filepath"
+	"path/filepath"
-	"time"
+	"time"
-
+
-	"dario.cat/mergo"
+	"dario.cat/mergo"
-	"github.com/fsnotify/fsnotify"
+	"github.com/fsnotify/fsnotify"
-	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/config"
-	"github.com/rs/zerolog/log"
+	"github.com/rs/zerolog/log"
-)
+)
-
+
-type fileHandler func(fileContent []byte, appConfig *config.ApplicationConfig) error
+type fileHandler func(fileContent []byte, appConfig *config.ApplicationConfig) error
-
+
-type configFileHandler struct {
+type configFileHandler struct {
-	handlers map[string]fileHandler
+	handlers map[string]fileHandler
-
+
-	watcher *fsnotify.Watcher
+	watcher *fsnotify.Watcher
-
+
-	appConfig *config.ApplicationConfig
+	appConfig *config.ApplicationConfig
-}
+}
-
+
-// TODO: This should be a singleton eventually so other parts of the code can register config file handlers,
+// TODO: This should be a singleton eventually so other parts of the code can register config file handlers,
-// then we can export it to other packages
+// then we can export it to other packages
-func newConfigFileHandler(appConfig *config.ApplicationConfig) configFileHandler {
+func newConfigFileHandler(appConfig *config.ApplicationConfig) configFileHandler {
-	c := configFileHandler{
+	c := configFileHandler{
-		handlers:  make(map[string]fileHandler),
+		handlers:  make(map[string]fileHandler),
-		appConfig: appConfig,
+		appConfig: appConfig,
-	}
+	}
-	err := c.Register("api_keys.json", readApiKeysJson(*appConfig), true)
+	err := c.Register("api_keys.json", readApiKeysJson(*appConfig), true)
-	if err != nil {
+	if err != nil {
-		log.Error().Err(err).Str("file", "api_keys.json").Msg("unable to register config file handler")
+		log.Error().Err(err).Str("file", "api_keys.json").Msg("unable to register config file handler")
-	}
+	}
-	err = c.Register("external_backends.json", readExternalBackendsJson(*appConfig), true)
+	err = c.Register("external_backends.json", readExternalBackendsJson(*appConfig), true)
-	if err != nil {
+	if err != nil {
-		log.Error().Err(err).Str("file", "external_backends.json").Msg("unable to register config file handler")
+		log.Error().Err(err).Str("file", "external_backends.json").Msg("unable to register config file handler")
-	}
+	}
-	return c
+	err = c.Register("runtime_settings.json", readRuntimeSettingsJson(*appConfig), true)
-}
+	if err != nil {
-
+		log.Error().Err(err).Str("file", "runtime_settings.json").Msg("unable to register config file handler")
-func (c *configFileHandler) Register(filename string, handler fileHandler, runNow bool) error {
+	}
-	_, ok := c.handlers[filename]
+	return c
-	if ok {
+}
-		return fmt.Errorf("handler already registered for file %s", filename)
+
-	}
+func (c *configFileHandler) Register(filename string, handler fileHandler, runNow bool) error {
-	c.handlers[filename] = handler
+	_, ok := c.handlers[filename]
-	if runNow {
+	if ok {
-		c.callHandler(filename, handler)
+		return fmt.Errorf("handler already registered for file %s", filename)
-	}
+	}
-	return nil
+	c.handlers[filename] = handler
-}
+	if runNow {
-
+		c.callHandler(filename, handler)
-func (c *configFileHandler) callHandler(filename string, handler fileHandler) {
+	}
-	rootedFilePath := filepath.Join(c.appConfig.DynamicConfigsDir, filepath.Clean(filename))
+	return nil
-	log.Trace().Str("filename", rootedFilePath).Msg("reading file for dynamic config update")
+}
-	fileContent, err := os.ReadFile(rootedFilePath)
+
-	if err != nil && !os.IsNotExist(err) {
+func (c *configFileHandler) callHandler(filename string, handler fileHandler) {
-		log.Error().Err(err).Str("filename", rootedFilePath).Msg("could not read file")
+	rootedFilePath := filepath.Join(c.appConfig.DynamicConfigsDir, filepath.Clean(filename))
-	}
+	log.Trace().Str("filename", rootedFilePath).Msg("reading file for dynamic config update")
-
+	fileContent, err := os.ReadFile(rootedFilePath)
-	if err = handler(fileContent, c.appConfig); err != nil {
+	if err != nil && !os.IsNotExist(err) {
-		log.Error().Err(err).Msg("WatchConfigDirectory goroutine failed to update options")
+		log.Error().Err(err).Str("filename", rootedFilePath).Msg("could not read file")
-	}
+	}
-}
+
-
+	if err = handler(fileContent, c.appConfig); err != nil {
-func (c *configFileHandler) Watch() error {
+		log.Error().Err(err).Msg("WatchConfigDirectory goroutine failed to update options")
-	configWatcher, err := fsnotify.NewWatcher()
+	}
-	c.watcher = configWatcher
+}
-	if err != nil {
+
-		return err
+func (c *configFileHandler) Watch() error {
-	}
+	configWatcher, err := fsnotify.NewWatcher()
-
+	c.watcher = configWatcher
-	if c.appConfig.DynamicConfigsDirPollInterval > 0 {
+	if err != nil {
-		log.Debug().Msg("Poll interval set, falling back to polling for configuration changes")
+		return err
-		ticker := time.NewTicker(c.appConfig.DynamicConfigsDirPollInterval)
+	}
-		go func() {
+
-			for {
+	if c.appConfig.DynamicConfigsDirPollInterval > 0 {
-				<-ticker.C
+		log.Debug().Msg("Poll interval set, falling back to polling for configuration changes")
-				for file, handler := range c.handlers {
+		ticker := time.NewTicker(c.appConfig.DynamicConfigsDirPollInterval)
-					log.Debug().Str("file", file).Msg("polling config file")
+		go func() {
-					c.callHandler(file, handler)
+			for {
-				}
+				<-ticker.C
-			}
+				for file, handler := range c.handlers {
-		}()
+					log.Debug().Str("file", file).Msg("polling config file")
-	}
+					c.callHandler(file, handler)
-
+				}
-	// Start listening for events.
+			}
-	go func() {
+		}()
-		for {
+	}
-			select {
+
-			case event, ok := <-c.watcher.Events:
+	// Start listening for events.
-				if !ok {
+	go func() {
-					return
+		for {
-				}
+			select {
-				if event.Has(fsnotify.Write | fsnotify.Create | fsnotify.Remove) {
+			case event, ok := <-c.watcher.Events:
-					handler, ok := c.handlers[path.Base(event.Name)]
+				if !ok {
-					if !ok {
+					return
-						continue
+				}
-					}
+				if event.Has(fsnotify.Write | fsnotify.Create | fsnotify.Remove) {
-
+					handler, ok := c.handlers[path.Base(event.Name)]
-					c.callHandler(filepath.Base(event.Name), handler)
+					if !ok {
-				}
+						continue
-			case err, ok := <-c.watcher.Errors:
+					}
-				log.Error().Err(err).Msg("config watcher error received")
+
-				if !ok {
+					c.callHandler(filepath.Base(event.Name), handler)
-					return
+				}
-				}
+			case err, ok := <-c.watcher.Errors:
-			}
+				log.Error().Err(err).Msg("config watcher error received")
-		}
+				if !ok {
-	}()
+					return
-
+				}
-	// Add a path.
+			}
-	err = c.watcher.Add(c.appConfig.DynamicConfigsDir)
+		}
-	if err != nil {
+	}()
-		return fmt.Errorf("unable to create a watcher on the configuration directory: %+v", err)
+
-	}
+	// Add a path.
-
+	err = c.watcher.Add(c.appConfig.DynamicConfigsDir)
-	return nil
+	if err != nil {
-}
+		return fmt.Errorf("unable to create a watcher on the configuration directory: %+v", err)
-
+	}
-// TODO: When we institute graceful shutdown, this should be called
+
-func (c *configFileHandler) Stop() error {
+	return nil
-	return c.watcher.Close()
+}
-}
+
-
+// TODO: When we institute graceful shutdown, this should be called
-func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
+func (c *configFileHandler) Stop() error {
-	handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
+	return c.watcher.Close()
-		log.Debug().Msg("processing api keys runtime update")
+}
-		log.Trace().Int("numKeys", len(startupAppConfig.ApiKeys)).Msg("api keys provided at startup")
+
-
+func readApiKeysJson(startupAppConfig config.ApplicationConfig) fileHandler {
-		if len(fileContent) > 0 {
+	handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
-			// Parse JSON content from the file
+		log.Debug().Msg("processing api keys runtime update")
-			var fileKeys []string
+		log.Trace().Int("numKeys", len(startupAppConfig.ApiKeys)).Msg("api keys provided at startup")
-			err := json.Unmarshal(fileContent, &fileKeys)
+
-			if err != nil {
+		if len(fileContent) > 0 {
-				return err
+			// Parse JSON content from the file
-			}
+			var fileKeys []string
-
+			err := json.Unmarshal(fileContent, &fileKeys)
-			log.Trace().Int("numKeys", len(fileKeys)).Msg("discovered API keys from api keys dynamic config dile")
+			if err != nil {
-
+				return err
-			appConfig.ApiKeys = append(startupAppConfig.ApiKeys, fileKeys...)
+			}
-		} else {
+
-			log.Trace().Msg("no API keys discovered from dynamic config file")
+			log.Trace().Int("numKeys", len(fileKeys)).Msg("discovered API keys from api keys dynamic config dile")
-			appConfig.ApiKeys = startupAppConfig.ApiKeys
+
-		}
+			appConfig.ApiKeys = append(startupAppConfig.ApiKeys, fileKeys...)
-		log.Trace().Int("numKeys", len(appConfig.ApiKeys)).Msg("total api keys after processing")
+		} else {
-		return nil
+			log.Trace().Msg("no API keys discovered from dynamic config file")
-	}
+			appConfig.ApiKeys = startupAppConfig.ApiKeys
-
+		}
-	return handler
+		log.Trace().Int("numKeys", len(appConfig.ApiKeys)).Msg("total api keys after processing")
-}
+		return nil
-
+	}
-func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHandler {
+
-	handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
+	return handler
-		log.Debug().Msg("processing external_backends.json")
+}
-
+
-		if len(fileContent) > 0 {
+func readExternalBackendsJson(startupAppConfig config.ApplicationConfig) fileHandler {
-			// Parse JSON content from the file
+	handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
-			var fileBackends map[string]string
+		log.Debug().Msg("processing external_backends.json")
-			err := json.Unmarshal(fileContent, &fileBackends)
+
-			if err != nil {
+		if len(fileContent) > 0 {
-				return err
+			// Parse JSON content from the file
-			}
+			var fileBackends map[string]string
-			appConfig.ExternalGRPCBackends = startupAppConfig.ExternalGRPCBackends
+			err := json.Unmarshal(fileContent, &fileBackends)
-			err = mergo.Merge(&appConfig.ExternalGRPCBackends, &fileBackends)
+			if err != nil {
-			if err != nil {
+				return err
-				return err
+			}
-			}
+			appConfig.ExternalGRPCBackends = startupAppConfig.ExternalGRPCBackends
-		} else {
+			err = mergo.Merge(&appConfig.ExternalGRPCBackends, &fileBackends)
-			appConfig.ExternalGRPCBackends = startupAppConfig.ExternalGRPCBackends
+			if err != nil {
-		}
+				return err
-		log.Debug().Msg("external backends loaded from external_backends.json")
+			}
-		return nil
+		} else {
-	}
+			appConfig.ExternalGRPCBackends = startupAppConfig.ExternalGRPCBackends
-	return handler
+		}
-}
+		log.Debug().Msg("external backends loaded from external_backends.json")
+		return nil
+	}
+	return handler
+}
+
+type runtimeSettings struct {
+	WatchdogEnabled          *bool             `json:"watchdog_enabled,omitempty"`
+	WatchdogIdleEnabled      *bool             `json:"watchdog_idle_enabled,omitempty"`
+	WatchdogBusyEnabled      *bool             `json:"watchdog_busy_enabled,omitempty"`
+	WatchdogIdleTimeout      *string           `json:"watchdog_idle_timeout,omitempty"`
+	WatchdogBusyTimeout      *string           `json:"watchdog_busy_timeout,omitempty"`
+	SingleBackend            *bool             `json:"single_backend,omitempty"`
+	ParallelBackendRequests  *bool             `json:"parallel_backend_requests,omitempty"`
+	Threads                  *int              `json:"threads,omitempty"`
+	ContextSize              *int              `json:"context_size,omitempty"`
+	F16                      *bool             `json:"f16,omitempty"`
+	Debug                    *bool             `json:"debug,omitempty"`
+	CORS                     *bool             `json:"cors,omitempty"`
+	CSRF                     *bool             `json:"csrf,omitempty"`
+	CORSAllowOrigins         *string           `json:"cors_allow_origins,omitempty"`
+	P2PToken                 *string           `json:"p2p_token,omitempty"`
+	P2PNetworkID             *string           `json:"p2p_network_id,omitempty"`
+	Federated                *bool             `json:"federated,omitempty"`
+	Galleries                *[]config.Gallery `json:"galleries,omitempty"`
+	BackendGalleries         *[]config.Gallery `json:"backend_galleries,omitempty"`
+	AutoloadGalleries        *bool             `json:"autoload_galleries,omitempty"`
+	AutoloadBackendGalleries *bool             `json:"autoload_backend_galleries,omitempty"`
+	ApiKeys                  *[]string         `json:"api_keys,omitempty"`
+}
+
+func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHandler {
+	handler := func(fileContent []byte, appConfig *config.ApplicationConfig) error {
+		log.Debug().Msg("processing runtime_settings.json")
+
+		// Determine if settings came from env vars by comparing with startup config
+		// startupAppConfig contains the original values set from env vars at startup.
+		// If current values match startup values, they came from env vars (or defaults).
+		// We apply file settings only if current values match startup values (meaning not from env vars).
+		envWatchdogIdle := appConfig.WatchDogIdle == startupAppConfig.WatchDogIdle
+		envWatchdogBusy := appConfig.WatchDogBusy == startupAppConfig.WatchDogBusy
+		envWatchdogIdleTimeout := appConfig.WatchDogIdleTimeout == startupAppConfig.WatchDogIdleTimeout
+		envWatchdogBusyTimeout := appConfig.WatchDogBusyTimeout == startupAppConfig.WatchDogBusyTimeout
+		envSingleBackend := appConfig.SingleBackend == startupAppConfig.SingleBackend
+		envParallelRequests := appConfig.ParallelBackendRequests == startupAppConfig.ParallelBackendRequests
+		envThreads := appConfig.Threads == startupAppConfig.Threads
+		envContextSize := appConfig.ContextSize == startupAppConfig.ContextSize
+		envF16 := appConfig.F16 == startupAppConfig.F16
+		envDebug := appConfig.Debug == startupAppConfig.Debug
+		envCORS := appConfig.CORS == startupAppConfig.CORS
+		envCSRF := appConfig.CSRF == startupAppConfig.CSRF
+		envCORSAllowOrigins := appConfig.CORSAllowOrigins == startupAppConfig.CORSAllowOrigins
+		envP2PToken := appConfig.P2PToken == startupAppConfig.P2PToken
+		envP2PNetworkID := appConfig.P2PNetworkID == startupAppConfig.P2PNetworkID
+		envFederated := appConfig.Federated == startupAppConfig.Federated
+		envAutoloadGalleries := appConfig.AutoloadGalleries == startupAppConfig.AutoloadGalleries
+		envAutoloadBackendGalleries := appConfig.AutoloadBackendGalleries == startupAppConfig.AutoloadBackendGalleries
+
+		if len(fileContent) > 0 {
+			var settings runtimeSettings
+			err := json.Unmarshal(fileContent, &settings)
+			if err != nil {
+				return err
+			}
+
+			// Apply file settings only if they don't match startup values (i.e., not from env vars)
+			if settings.WatchdogIdleEnabled != nil && !envWatchdogIdle {
+				appConfig.WatchDogIdle = *settings.WatchdogIdleEnabled
+				if appConfig.WatchDogIdle {
+					appConfig.WatchDog = true
+				}
+			}
+			if settings.WatchdogBusyEnabled != nil && !envWatchdogBusy {
+				appConfig.WatchDogBusy = *settings.WatchdogBusyEnabled
+				if appConfig.WatchDogBusy {
+					appConfig.WatchDog = true
+				}
+			}
+			if settings.WatchdogIdleTimeout != nil && !envWatchdogIdleTimeout {
+				dur, err := time.ParseDuration(*settings.WatchdogIdleTimeout)
+				if err == nil {
+					appConfig.WatchDogIdleTimeout = dur
+				} else {
+					log.Warn().Err(err).Str("timeout", *settings.WatchdogIdleTimeout).Msg("invalid watchdog idle timeout in runtime_settings.json")
+				}
+			}
+			if settings.WatchdogBusyTimeout != nil && !envWatchdogBusyTimeout {
+				dur, err := time.ParseDuration(*settings.WatchdogBusyTimeout)
+				if err == nil {
+					appConfig.WatchDogBusyTimeout = dur
+				} else {
+					log.Warn().Err(err).Str("timeout", *settings.WatchdogBusyTimeout).Msg("invalid watchdog busy timeout in runtime_settings.json")
+				}
+			}
+			if settings.SingleBackend != nil && !envSingleBackend {
+				appConfig.SingleBackend = *settings.SingleBackend
+			}
+			if settings.ParallelBackendRequests != nil && !envParallelRequests {
+				appConfig.ParallelBackendRequests = *settings.ParallelBackendRequests
+			}
+			if settings.Threads != nil && !envThreads {
+				appConfig.Threads = *settings.Threads
+			}
+			if settings.ContextSize != nil && !envContextSize {
+				appConfig.ContextSize = *settings.ContextSize
+			}
+			if settings.F16 != nil && !envF16 {
+				appConfig.F16 = *settings.F16
+			}
+			if settings.Debug != nil && !envDebug {
+				appConfig.Debug = *settings.Debug
+			}
+			if settings.CORS != nil && !envCORS {
+				appConfig.CORS = *settings.CORS
+			}
+			if settings.CSRF != nil && !envCSRF {
+				appConfig.CSRF = *settings.CSRF
+			}
+			if settings.CORSAllowOrigins != nil && !envCORSAllowOrigins {
+				appConfig.CORSAllowOrigins = *settings.CORSAllowOrigins
+			}
+			if settings.P2PToken != nil && !envP2PToken {
+				appConfig.P2PToken = *settings.P2PToken
+			}
+			if settings.P2PNetworkID != nil && !envP2PNetworkID {
+				appConfig.P2PNetworkID = *settings.P2PNetworkID
+			}
+			if settings.Federated != nil && !envFederated {
+				appConfig.Federated = *settings.Federated
+			}
+			if settings.Galleries != nil {
+				appConfig.Galleries = *settings.Galleries
+			}
+			if settings.BackendGalleries != nil {
+				appConfig.BackendGalleries = *settings.BackendGalleries
+			}
+			if settings.AutoloadGalleries != nil && !envAutoloadGalleries {
+				appConfig.AutoloadGalleries = *settings.AutoloadGalleries
+			}
+			if settings.AutoloadBackendGalleries != nil && !envAutoloadBackendGalleries {
+				appConfig.AutoloadBackendGalleries = *settings.AutoloadBackendGalleries
+			}
+			if settings.ApiKeys != nil {
+				// API keys from env vars (startup) should be kept, runtime settings keys replace all runtime keys
+				// If runtime_settings.json specifies ApiKeys (even if empty), it replaces all runtime keys
+				// Start with env keys, then add runtime_settings.json keys (which may be empty to clear them)
+				envKeys := startupAppConfig.ApiKeys
+				runtimeKeys := *settings.ApiKeys
+				// Replace all runtime keys with what's in runtime_settings.json
+				appConfig.ApiKeys = append(envKeys, runtimeKeys...)
+			}
+
+			// If watchdog is enabled via file but not via env, ensure WatchDog flag is set
+			if !envWatchdogIdle && !envWatchdogBusy {
+				if settings.WatchdogEnabled != nil && *settings.WatchdogEnabled {
+					appConfig.WatchDog = true
+				}
+			}
+		}
+		log.Debug().Msg("runtime settings loaded from runtime_settings.json")
+		return nil
+	}
+	return handler
+}

core/application/p2p.go

@@ -0,0 +1,240 @@
+package application
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"slices"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/core/p2p"
+	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/core/services"
+
+	"github.com/mudler/edgevpn/pkg/node"
+	"github.com/rs/zerolog/log"
+	zlog "github.com/rs/zerolog/log"
+)
+
+func (a *Application) StopP2P() error {
+	if a.p2pCancel != nil {
+		a.p2pCancel()
+		a.p2pCancel = nil
+		a.p2pCtx = nil
+		// Wait a bit for shutdown to complete
+		time.Sleep(200 * time.Millisecond)
+	}
+	return nil
+}
+
+func (a *Application) StartP2P() error {
+	// we need a p2p token
+	if a.applicationConfig.P2PToken == "" {
+		return fmt.Errorf("P2P token is not set")
+	}
+
+	networkID := a.applicationConfig.P2PNetworkID
+
+	ctx, cancel := context.WithCancel(a.ApplicationConfig().Context)
+	a.p2pCtx = ctx
+	a.p2pCancel = cancel
+
+	var n *node.Node
+	// Here we are avoiding creating multiple nodes:
+	// - if the federated mode is enabled, we create a federated node and expose a service
+	// - exposing a service creates a node with specific options, and we don't want to create another node
+
+	// If the federated mode is enabled, we expose a service to the local instance running
+	// at r.Address
+	if a.applicationConfig.Federated {
+		_, port, err := net.SplitHostPort(a.applicationConfig.APIAddress)
+		if err != nil {
+			return err
+		}
+
+		// Here a new node is created and started
+		// and a service is exposed by the node
+		node, err := p2p.ExposeService(ctx, "localhost", port, a.applicationConfig.P2PToken, p2p.NetworkID(networkID, p2p.FederatedID))
+		if err != nil {
+			return err
+		}
+
+		if err := p2p.ServiceDiscoverer(ctx, node, a.applicationConfig.P2PToken, p2p.NetworkID(networkID, p2p.FederatedID), nil, false); err != nil {
+			return err
+		}
+
+		n = node
+		// start node sync in the background
+		if err := a.p2pSync(ctx, node); err != nil {
+			return err
+		}
+	}
+
+	// If a node wasn't created previously, create it
+	if n == nil {
+		node, err := p2p.NewNode(a.applicationConfig.P2PToken)
+		if err != nil {
+			return err
+		}
+		err = node.Start(ctx)
+		if err != nil {
+			return fmt.Errorf("starting new node: %w", err)
+		}
+		n = node
+	}
+
+	// Attach a ServiceDiscoverer to the p2p node
+	log.Info().Msg("Starting P2P server discovery...")
+	if err := p2p.ServiceDiscoverer(ctx, n, a.applicationConfig.P2PToken, p2p.NetworkID(networkID, p2p.WorkerID), func(serviceID string, node schema.NodeData) {
+		var tunnelAddresses []string
+		for _, v := range p2p.GetAvailableNodes(p2p.NetworkID(networkID, p2p.WorkerID)) {
+			if v.IsOnline() {
+				tunnelAddresses = append(tunnelAddresses, v.TunnelAddress)
+			} else {
+				log.Info().Msgf("Node %s is offline", v.ID)
+			}
+		}
+		if a.applicationConfig.TunnelCallback != nil {
+			a.applicationConfig.TunnelCallback(tunnelAddresses)
+		}
+	}, true); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RestartP2P restarts the P2P stack with current ApplicationConfig settings
+// Note: This method signals that P2P should be restarted, but the actual restart
+// is handled by the caller to avoid import cycles
+func (a *Application) RestartP2P() error {
+	a.p2pMutex.Lock()
+	defer a.p2pMutex.Unlock()
+
+	// Stop existing P2P if running
+	if a.p2pCancel != nil {
+		a.p2pCancel()
+		a.p2pCancel = nil
+		a.p2pCtx = nil
+		// Wait a bit for shutdown to complete
+		time.Sleep(200 * time.Millisecond)
+	}
+
+	appConfig := a.ApplicationConfig()
+
+	// Start P2P if token is set
+	if appConfig.P2PToken == "" {
+		return fmt.Errorf("P2P token is not set")
+	}
+
+	// Create new context for P2P
+	ctx, cancel := context.WithCancel(appConfig.Context)
+	a.p2pCtx = ctx
+	a.p2pCancel = cancel
+
+	// Get API address from config
+	address := appConfig.APIAddress
+	if address == "" {
+		address = "127.0.0.1:8080" // default
+	}
+
+	// Start P2P stack in a goroutine
+	go func() {
+		if err := a.StartP2P(); err != nil {
+			log.Error().Err(err).Msg("Failed to start P2P stack")
+			cancel() // Cancel context on error
+		}
+	}()
+	log.Info().Msg("P2P stack restarted with new settings")
+
+	return nil
+}
+
+func syncState(ctx context.Context, n *node.Node, app *Application) error {
+	zlog.Debug().Msg("[p2p-sync] Syncing state")
+
+	whatWeHave := []string{}
+	for _, model := range app.ModelConfigLoader().GetAllModelsConfigs() {
+		whatWeHave = append(whatWeHave, model.Name)
+	}
+
+	ledger, _ := n.Ledger()
+	currentData := ledger.CurrentData()
+	zlog.Debug().Msgf("[p2p-sync] Current data: %v", currentData)
+	data, exists := ledger.GetKey("shared_state", "models")
+	if !exists {
+		ledger.AnnounceUpdate(ctx, time.Minute, "shared_state", "models", whatWeHave)
+		zlog.Debug().Msgf("No models found in the ledger, announced our models: %v", whatWeHave)
+	}
+
+	models := []string{}
+	if err := data.Unmarshal(&models); err != nil {
+		zlog.Warn().Err(err).Msg("error unmarshalling models")
+		return nil
+	}
+
+	zlog.Debug().Msgf("[p2p-sync] Models that are present in this instance: %v\nModels that are in the ledger: %v", whatWeHave, models)
+
+	// Sync with our state
+	whatIsNotThere := []string{}
+	for _, model := range whatWeHave {
+		if !slices.Contains(models, model) {
+			whatIsNotThere = append(whatIsNotThere, model)
+		}
+	}
+	if len(whatIsNotThere) > 0 {
+		zlog.Debug().Msgf("[p2p-sync] Announcing our models: %v", append(models, whatIsNotThere...))
+		ledger.AnnounceUpdate(
+			ctx,
+			1*time.Minute,
+			"shared_state",
+			"models",
+			append(models, whatIsNotThere...),
+		)
+	}
+
+	// Check if we have a model that is not in our state, otherwise install it
+	for _, model := range models {
+		if slices.Contains(whatWeHave, model) {
+			zlog.Debug().Msgf("[p2p-sync] Model %s is already present in this instance", model)
+			continue
+		}
+
+		// we install model
+		zlog.Info().Msgf("[p2p-sync] Installing model which is not present in this instance: %s", model)
+
+		uuid, err := uuid.NewUUID()
+		if err != nil {
+			zlog.Error().Err(err).Msg("error generating UUID")
+			continue
+		}
+
+		app.GalleryService().ModelGalleryChannel <- services.GalleryOp[gallery.GalleryModel, gallery.ModelConfig]{
+			ID:                 uuid.String(),
+			GalleryElementName: model,
+			Galleries:          app.ApplicationConfig().Galleries,
+			BackendGalleries:   app.ApplicationConfig().BackendGalleries,
+		}
+	}
+
+	return nil
+}
+
+func (a *Application) p2pSync(ctx context.Context, n *node.Node) error {
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(1 * time.Minute):
+				if err := syncState(ctx, n, a); err != nil {
+					zlog.Error().Err(err).Msg("error syncing state")
+				}
+			}
+
+		}
+	}()
+	return nil
+}

core/application/startup.go

@@ -1,8 +1,11 @@
 package application
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
+	"time"
 
 	"github.com/mudler/LocalAI/core/backend"
 	"github.com/mudler/LocalAI/core/config"
@@ -18,7 +21,12 @@ import (
 
 func New(opts ...config.AppOption) (*Application, error) {
 	options := config.NewApplicationConfig(opts...)
+
+	// Store a copy of the startup config (from env vars, before file loading)
+	// This is used to determine if settings came from env vars vs file
+	startupConfigCopy := *options
 	application := newApplication(options)
+	application.startupConfig = &startupConfigCopy
 
 	log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.SystemState.Model.ModelsPath)
 	log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion())
@@ -110,6 +118,13 @@ func New(opts ...config.AppOption) (*Application, error) {
 		}
 	}
 
+	// Load runtime settings from file if DynamicConfigsDir is set
+	// This applies file settings with env var precedence (env vars take priority)
+	// Note: startupConfigCopy was already created above, so it has the original env var values
+	if options.DynamicConfigsDir != "" {
+		loadRuntimeSettingsFromFile(options)
+	}
+
 	// turn off any process that was started by GRPC if the context is canceled
 	go func() {
 		<-options.Context.Done()
@@ -120,21 +135,8 @@ func New(opts ...config.AppOption) (*Application, error) {
 		}
 	}()
 
-	if options.WatchDog {
+	// Initialize watchdog with current settings (after loading from file)
-		wd := model.NewWatchDog(
+	initializeWatchdog(application, options)
-			application.ModelLoader(),
-			options.WatchDogBusyTimeout,
-			options.WatchDogIdleTimeout,
-			options.WatchDogBusy,
-			options.WatchDogIdle)
-		application.ModelLoader().SetWatchDog(wd)
-		go wd.Run()
-		go func() {
-			<-options.Context.Done()
-			log.Debug().Msgf("Context canceled, shutting down")
-			wd.Shutdown()
-		}()
-	}
 
 	if options.LoadToMemory != nil && !options.SingleBackend {
 		for _, m := range options.LoadToMemory {
@@ -186,3 +188,131 @@ func startWatcher(options *config.ApplicationConfig) {
 		log.Error().Err(err).Msg("failed creating watcher")
 	}
 }
+
+// loadRuntimeSettingsFromFile loads settings from runtime_settings.json with env var precedence
+// This function is called at startup, before env vars are applied via AppOptions.
+// Since env vars are applied via AppOptions in run.go, we need to check if they're set.
+// We do this by checking if the current options values differ from defaults, which would
+// indicate they were set from env vars. However, a simpler approach is to just apply
+// file settings here, and let the AppOptions (which are applied after this) override them.
+// But actually, this is called AFTER AppOptions are applied in New(), so we need to check env vars.
+// The cleanest solution: Store original values before applying file, or check if values match
+// what would be set from env vars. For now, we'll apply file settings and they'll be
+// overridden by AppOptions if env vars were set (but AppOptions are already applied).
+// Actually, this function is called in New() before AppOptions are fully processed for watchdog.
+// Let's check the call order: New() -> loadRuntimeSettingsFromFile() -> initializeWatchdog()
+// But AppOptions are applied in NewApplicationConfig() which is called first.
+// So at this point, options already has values from env vars. We should compare against
+// defaults to see if env vars were set. But we don't have defaults stored.
+// Simplest: Just apply file settings. If env vars were set, they're already in options.
+// The file watcher handler will handle runtime changes properly by comparing with startupAppConfig.
+func loadRuntimeSettingsFromFile(options *config.ApplicationConfig) {
+	settingsFile := filepath.Join(options.DynamicConfigsDir, "runtime_settings.json")
+	fileContent, err := os.ReadFile(settingsFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Debug().Msg("runtime_settings.json not found, using defaults")
+			return
+		}
+		log.Warn().Err(err).Msg("failed to read runtime_settings.json")
+		return
+	}
+
+	var settings struct {
+		WatchdogEnabled         *bool   `json:"watchdog_enabled,omitempty"`
+		WatchdogIdleEnabled     *bool   `json:"watchdog_idle_enabled,omitempty"`
+		WatchdogBusyEnabled     *bool   `json:"watchdog_busy_enabled,omitempty"`
+		WatchdogIdleTimeout     *string `json:"watchdog_idle_timeout,omitempty"`
+		WatchdogBusyTimeout     *string `json:"watchdog_busy_timeout,omitempty"`
+		SingleBackend           *bool   `json:"single_backend,omitempty"`
+		ParallelBackendRequests *bool   `json:"parallel_backend_requests,omitempty"`
+	}
+
+	if err := json.Unmarshal(fileContent, &settings); err != nil {
+		log.Warn().Err(err).Msg("failed to parse runtime_settings.json")
+		return
+	}
+
+	// At this point, options already has values from env vars (via AppOptions in run.go).
+	// To avoid env var duplication, we determine if env vars were set by checking if
+	// current values differ from defaults. Defaults are: false for bools, 0 for durations.
+	// If current value is at default, it likely wasn't set from env var, so we can apply file.
+	// If current value is non-default, it was likely set from env var, so we preserve it.
+	// Note: This means env vars explicitly setting to false/0 won't be distinguishable from defaults,
+	// but that's an acceptable limitation to avoid env var duplication.
+
+	if settings.WatchdogIdleEnabled != nil {
+		// Only apply if current value is default (false), suggesting it wasn't set from env var
+		if !options.WatchDogIdle {
+			options.WatchDogIdle = *settings.WatchdogIdleEnabled
+			if options.WatchDogIdle {
+				options.WatchDog = true
+			}
+		}
+	}
+	if settings.WatchdogBusyEnabled != nil {
+		if !options.WatchDogBusy {
+			options.WatchDogBusy = *settings.WatchdogBusyEnabled
+			if options.WatchDogBusy {
+				options.WatchDog = true
+			}
+		}
+	}
+	if settings.WatchdogIdleTimeout != nil {
+		// Only apply if current value is default (0), suggesting it wasn't set from env var
+		if options.WatchDogIdleTimeout == 0 {
+			dur, err := time.ParseDuration(*settings.WatchdogIdleTimeout)
+			if err == nil {
+				options.WatchDogIdleTimeout = dur
+			} else {
+				log.Warn().Err(err).Str("timeout", *settings.WatchdogIdleTimeout).Msg("invalid watchdog idle timeout in runtime_settings.json")
+			}
+		}
+	}
+	if settings.WatchdogBusyTimeout != nil {
+		if options.WatchDogBusyTimeout == 0 {
+			dur, err := time.ParseDuration(*settings.WatchdogBusyTimeout)
+			if err == nil {
+				options.WatchDogBusyTimeout = dur
+			} else {
+				log.Warn().Err(err).Str("timeout", *settings.WatchdogBusyTimeout).Msg("invalid watchdog busy timeout in runtime_settings.json")
+			}
+		}
+	}
+	if settings.SingleBackend != nil {
+		if !options.SingleBackend {
+			options.SingleBackend = *settings.SingleBackend
+		}
+	}
+	if settings.ParallelBackendRequests != nil {
+		if !options.ParallelBackendRequests {
+			options.ParallelBackendRequests = *settings.ParallelBackendRequests
+		}
+	}
+	if !options.WatchDogIdle && !options.WatchDogBusy {
+		if settings.WatchdogEnabled != nil && *settings.WatchdogEnabled {
+			options.WatchDog = true
+		}
+	}
+
+	log.Debug().Msg("Runtime settings loaded from runtime_settings.json")
+}
+
+// initializeWatchdog initializes the watchdog with current ApplicationConfig settings
+func initializeWatchdog(application *Application, options *config.ApplicationConfig) {
+	if options.WatchDog {
+		wd := model.NewWatchDog(
+			application.ModelLoader(),
+			options.WatchDogBusyTimeout,
+			options.WatchDogIdleTimeout,
+			options.WatchDogBusy,
+			options.WatchDogIdle)
+		application.ModelLoader().SetWatchDog(wd)
+		go wd.Run()
+		go func() {
+			<-options.Context.Done()
+			log.Debug().Msgf("Context canceled, shutting down")
+			wd.Shutdown()
+		}()
+	}
+}

core/application/watchdog.go

@@ -0,0 +1,88 @@
+package application
+
+import (
+	"time"
+
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/rs/zerolog/log"
+)
+
+func (a *Application) StopWatchdog() error {
+	if a.watchdogStop != nil {
+		close(a.watchdogStop)
+		a.watchdogStop = nil
+	}
+	return nil
+}
+
+// startWatchdog starts the watchdog with current ApplicationConfig settings
+// This is an internal method that assumes the caller holds the watchdogMutex
+func (a *Application) startWatchdog() error {
+	appConfig := a.ApplicationConfig()
+
+	// Create new watchdog if enabled
+	if appConfig.WatchDog {
+		wd := model.NewWatchDog(
+			a.modelLoader,
+			appConfig.WatchDogBusyTimeout,
+			appConfig.WatchDogIdleTimeout,
+			appConfig.WatchDogBusy,
+			appConfig.WatchDogIdle)
+		a.modelLoader.SetWatchDog(wd)
+
+		// Create new stop channel
+		a.watchdogStop = make(chan bool, 1)
+
+		// Start watchdog goroutine
+		go wd.Run()
+
+		// Setup shutdown handler
+		go func() {
+			select {
+			case <-a.watchdogStop:
+				log.Debug().Msg("Watchdog stop signal received")
+				wd.Shutdown()
+			case <-appConfig.Context.Done():
+				log.Debug().Msg("Context canceled, shutting down watchdog")
+				wd.Shutdown()
+			}
+		}()
+
+		log.Info().Msg("Watchdog started with new settings")
+	} else {
+		log.Info().Msg("Watchdog disabled")
+	}
+
+	return nil
+}
+
+// StartWatchdog starts the watchdog with current ApplicationConfig settings
+func (a *Application) StartWatchdog() error {
+	a.watchdogMutex.Lock()
+	defer a.watchdogMutex.Unlock()
+
+	return a.startWatchdog()
+}
+
+// RestartWatchdog restarts the watchdog with current ApplicationConfig settings
+func (a *Application) RestartWatchdog() error {
+	a.watchdogMutex.Lock()
+	defer a.watchdogMutex.Unlock()
+
+	// Shutdown existing watchdog if running
+	if a.watchdogStop != nil {
+		close(a.watchdogStop)
+		a.watchdogStop = nil
+	}
+
+	// Shutdown existing watchdog if running
+	currentWD := a.modelLoader.GetWatchDog()
+	if currentWD != nil {
+		currentWD.Shutdown()
+		// Wait a bit for shutdown to complete
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	// Start watchdog with new settings
+	return a.startWatchdog()
+}

core/cli/api/p2p.go

@@ -1,87 +0,0 @@
-package cli_api
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"os"
-	"strings"
-
-	"github.com/mudler/LocalAI/core/application"
-	"github.com/mudler/LocalAI/core/p2p"
-	"github.com/mudler/LocalAI/core/schema"
-	"github.com/mudler/edgevpn/pkg/node"
-
-	"github.com/rs/zerolog/log"
-)
-
-func StartP2PStack(ctx context.Context, address, token, networkID string, federated bool, app *application.Application) error {
-	var n *node.Node
-	// Here we are avoiding creating multiple nodes:
-	// - if the federated mode is enabled, we create a federated node and expose a service
-	// - exposing a service creates a node with specific options, and we don't want to create another node
-
-	// If the federated mode is enabled, we expose a service to the local instance running
-	// at r.Address
-	if federated {
-		_, port, err := net.SplitHostPort(address)
-		if err != nil {
-			return err
-		}
-
-		// Here a new node is created and started
-		// and a service is exposed by the node
-		node, err := p2p.ExposeService(ctx, "localhost", port, token, p2p.NetworkID(networkID, p2p.FederatedID))
-		if err != nil {
-			return err
-		}
-
-		if err := p2p.ServiceDiscoverer(ctx, node, token, p2p.NetworkID(networkID, p2p.FederatedID), nil, false); err != nil {
-			return err
-		}
-
-		n = node
-
-		// start node sync in the background
-		if err := p2p.Sync(ctx, node, app); err != nil {
-			return err
-		}
-	}
-
-	// If the p2p mode is enabled, we start the service discovery
-	if token != "" {
-		// If a node wasn't created previously, create it
-		if n == nil {
-			node, err := p2p.NewNode(token)
-			if err != nil {
-				return err
-			}
-			err = node.Start(ctx)
-			if err != nil {
-				return fmt.Errorf("starting new node: %w", err)
-			}
-			n = node
-		}
-
-		// Attach a ServiceDiscoverer to the p2p node
-		log.Info().Msg("Starting P2P server discovery...")
-		if err := p2p.ServiceDiscoverer(ctx, n, token, p2p.NetworkID(networkID, p2p.WorkerID), func(serviceID string, node schema.NodeData) {
-			var tunnelAddresses []string
-			for _, v := range p2p.GetAvailableNodes(p2p.NetworkID(networkID, p2p.WorkerID)) {
-				if v.IsOnline() {
-					tunnelAddresses = append(tunnelAddresses, v.TunnelAddress)
-				} else {
-					log.Info().Msgf("Node %s is offline", v.ID)
-				}
-			}
-			tunnelEnvVar := strings.Join(tunnelAddresses, ",")
-
-			os.Setenv("LLAMACPP_GRPC_SERVERS", tunnelEnvVar)
-			log.Debug().Msgf("setting LLAMACPP_GRPC_SERVERS to %s", tunnelEnvVar)
-		}, true); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

core/cli/run.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/mudler/LocalAI/core/application"
-	cli_api "github.com/mudler/LocalAI/core/cli/api"
 	cliContext "github.com/mudler/LocalAI/core/cli/context"
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/http"
@@ -52,6 +51,7 @@ type RunCMD struct {
 	UploadLimit                        int      `env:"LOCALAI_UPLOAD_LIMIT,UPLOAD_LIMIT" default:"15" help:"Default upload-limit in MB" group:"api"`
 	APIKeys                            []string `env:"LOCALAI_API_KEY,API_KEY" help:"List of API Keys to enable API authentication. When this is set, all the requests must be authenticated with one of these API keys" group:"api"`
 	DisableWebUI                       bool     `env:"LOCALAI_DISABLE_WEBUI,DISABLE_WEBUI" default:"false" help:"Disables the web user interface. When set to true, the server will only expose API endpoints without serving the web interface" group:"api"`
+	DisableRuntimeSettings             bool     `env:"LOCALAI_DISABLE_RUNTIME_SETTINGS,DISABLE_RUNTIME_SETTINGS" default:"false" help:"Disables the runtime settings. When set to true, the server will not load the runtime settings from the runtime_settings.json file" group:"api"`
 	DisablePredownloadScan             bool     `env:"LOCALAI_DISABLE_PREDOWNLOAD_SCAN" help:"If true, disables the best-effort security scanner before downloading any files." group:"hardening" default:"false"`
 	OpaqueErrors                       bool     `env:"LOCALAI_OPAQUE_ERRORS" default:"false" help:"If true, all error responses are replaced with blank 500 errors. This is intended only for hardening against information leaks and is normally not recommended." group:"hardening"`
 	UseSubtleKeyComparison             bool     `env:"LOCALAI_SUBTLE_KEY_COMPARISON" default:"false" help:"If true, API Key validation comparisons will be performed using constant-time comparisons rather than simple equality. This trades off performance on each request for resiliancy against timing attacks." group:"hardening"`
@@ -98,6 +98,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 	}
 
 	opts := []config.AppOption{
+		config.WithContext(context.Background()),
 		config.WithConfigFile(r.ModelsConfigFile),
 		config.WithJSONStringPreload(r.PreloadModels),
 		config.WithYAMLConfigPreload(r.PreloadModelsConfig),
@@ -128,12 +129,22 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		config.WithLoadToMemory(r.LoadToMemory),
 		config.WithMachineTag(r.MachineTag),
 		config.WithAPIAddress(r.Address),
+		config.WithTunnelCallback(func(tunnels []string) {
+			tunnelEnvVar := strings.Join(tunnels, ",")
+			// TODO: this is very specific to llama.cpp, we should have a more generic way to set the environment variable
+			os.Setenv("LLAMACPP_GRPC_SERVERS", tunnelEnvVar)
+			log.Debug().Msgf("setting LLAMACPP_GRPC_SERVERS to %s", tunnelEnvVar)
+		}),
 	}
 
 	if r.DisableMetricsEndpoint {
 		opts = append(opts, config.DisableMetricsEndpoint)
 	}
 
+	if r.DisableRuntimeSettings {
+		opts = append(opts, config.DisableRuntimeSettings)
+	}
+
 	token := ""
 	if r.Peer2Peer || r.Peer2PeerToken != "" {
 		log.Info().Msg("P2P mode enabled")
@@ -152,7 +163,9 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		opts = append(opts, config.WithP2PToken(token))
 	}
 
-	backgroundCtx := context.Background()
+	if r.Federated {
+		opts = append(opts, config.EnableFederated)
+	}
 
 	idleWatchDog := r.EnableWatchdogIdle
 	busyWatchDog := r.EnableWatchdogBusy
@@ -222,8 +235,10 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		return err
 	}
 
-	if err := cli_api.StartP2PStack(backgroundCtx, r.Address, token, r.Peer2PeerNetworkID, r.Federated, app); err != nil {
+	if token != "" {
-		return err
+		if err := app.StartP2P(); err != nil {
+			return err
+		}
 	}
 
 	signals.RegisterGracefulTerminationHandler(func() {

core/config/application_config.go

@@ -33,6 +33,7 @@ type ApplicationConfig struct {
 	ApiKeys                       []string
 	P2PToken                      string
 	P2PNetworkID                  string
+	Federated                     bool
 
 	DisableWebUI                       bool
 	EnforcePredownloadScans            bool
@@ -65,6 +66,10 @@ type ApplicationConfig struct {
 	MachineTag string
 
 	APIAddress string
+
+	TunnelCallback func(tunnels []string)
+
+	DisableRuntimeSettings bool
 }
 
 type AppOption func(*ApplicationConfig)
@@ -73,7 +78,6 @@ func NewApplicationConfig(o ...AppOption) *ApplicationConfig {
 	opt := &ApplicationConfig{
 		Context:       context.Background(),
 		UploadLimitMB: 15,
-		ContextSize:   512,
 		Debug:         true,
 	}
 	for _, oo := range o {
@@ -152,6 +156,10 @@ var DisableWebUI = func(o *ApplicationConfig) {
 	o.DisableWebUI = true
 }
 
+var DisableRuntimeSettings = func(o *ApplicationConfig) {
+	o.DisableRuntimeSettings = true
+}
+
 func SetWatchDogBusyTimeout(t time.Duration) AppOption {
 	return func(o *ApplicationConfig) {
 		o.WatchDogBusyTimeout = t
@@ -180,6 +188,10 @@ var EnableBackendGalleriesAutoload = func(o *ApplicationConfig) {
 	o.AutoloadBackendGalleries = true
 }
 
+var EnableFederated = func(o *ApplicationConfig) {
+	o.Federated = true
+}
+
 func WithExternalBackend(name string, uri string) AppOption {
 	return func(o *ApplicationConfig) {
 		if o.ExternalGRPCBackends == nil {
@@ -273,6 +285,12 @@ func WithContextSize(ctxSize int) AppOption {
 	}
 }
 
+func WithTunnelCallback(callback func(tunnels []string)) AppOption {
+	return func(o *ApplicationConfig) {
+		o.TunnelCallback = callback
+	}
+}
+
 func WithF16(f16 bool) AppOption {
 	return func(o *ApplicationConfig) {
 		o.F16 = f16

core/http/app.go

@@ -208,7 +208,7 @@ func API(application *application.Application) (*echo.Echo, error) {
 	routes.RegisterLocalAIRoutes(e, requestExtractor, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService(), opcache, application.TemplatesEvaluator())
 	routes.RegisterOpenAIRoutes(e, requestExtractor, application)
 	if !application.ApplicationConfig().DisableWebUI {
-		routes.RegisterUIAPIRoutes(e, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService(), opcache)
+		routes.RegisterUIAPIRoutes(e, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService(), opcache, application)
 		routes.RegisterUIRoutes(e, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.GalleryService())
 	}
 	routes.RegisterJINARoutes(e, requestExtractor, application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig())

core/http/endpoints/localai/import_model.go

@@ -145,7 +145,7 @@ func ImportModelEndpoint(cl *config.ModelConfigLoader, appConfig *config.Applica
 		}
 
 		// Set defaults
-		modelConfig.SetDefaults()
+		modelConfig.SetDefaults(appConfig.ToConfigLoaderOptions()...)
 
 		// Validate the configuration
 		if valid, _ := modelConfig.Validate(); !valid {

core/http/endpoints/localai/mcp.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"strings"
+	"net"
 	"time"
 
 	"github.com/labstack/echo/v4"
@@ -105,7 +105,10 @@ func MCPStreamEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, eval
 			fragment = fragment.AddMessage(message.Role, message.StringContent)
 		}
 
-		port := appConfig.APIAddress[strings.LastIndex(appConfig.APIAddress, ":")+1:]
+		_, port, err := net.SplitHostPort(appConfig.APIAddress)
+		if err != nil {
+			return err
+		}
 		apiKey := ""
 		if len(appConfig.ApiKeys) > 0 {
 			apiKey = appConfig.ApiKeys[0]

core/http/endpoints/localai/settings.go

@@ -0,0 +1,340 @@
+package localai
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/application"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/p2p"
+	"github.com/rs/zerolog/log"
+)
+
+type SettingsResponse struct {
+	Success bool   `json:"success"`
+	Error   string `json:"error,omitempty"`
+	Message string `json:"message,omitempty"`
+}
+
+type RuntimeSettings struct {
+	WatchdogEnabled          *bool             `json:"watchdog_enabled,omitempty"`
+	WatchdogIdleEnabled      *bool             `json:"watchdog_idle_enabled,omitempty"`
+	WatchdogBusyEnabled      *bool             `json:"watchdog_busy_enabled,omitempty"`
+	WatchdogIdleTimeout      *string           `json:"watchdog_idle_timeout,omitempty"`
+	WatchdogBusyTimeout      *string           `json:"watchdog_busy_timeout,omitempty"`
+	SingleBackend            *bool             `json:"single_backend,omitempty"`
+	ParallelBackendRequests  *bool             `json:"parallel_backend_requests,omitempty"`
+	Threads                  *int              `json:"threads,omitempty"`
+	ContextSize              *int              `json:"context_size,omitempty"`
+	F16                      *bool             `json:"f16,omitempty"`
+	Debug                    *bool             `json:"debug,omitempty"`
+	CORS                     *bool             `json:"cors,omitempty"`
+	CSRF                     *bool             `json:"csrf,omitempty"`
+	CORSAllowOrigins         *string           `json:"cors_allow_origins,omitempty"`
+	P2PToken                 *string           `json:"p2p_token,omitempty"`
+	P2PNetworkID             *string           `json:"p2p_network_id,omitempty"`
+	Federated                *bool             `json:"federated,omitempty"`
+	Galleries                *[]config.Gallery `json:"galleries,omitempty"`
+	BackendGalleries         *[]config.Gallery `json:"backend_galleries,omitempty"`
+	AutoloadGalleries        *bool             `json:"autoload_galleries,omitempty"`
+	AutoloadBackendGalleries *bool             `json:"autoload_backend_galleries,omitempty"`
+	ApiKeys                  *[]string         `json:"api_keys"` // No omitempty - we need to save empty arrays to clear keys
+}
+
+// GetSettingsEndpoint returns current settings with precedence (env > file > defaults)
+func GetSettingsEndpoint(app *application.Application) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		appConfig := app.ApplicationConfig()
+		startupConfig := app.StartupConfig()
+
+		if startupConfig == nil {
+			// Fallback if startup config not available
+			startupConfig = appConfig
+		}
+
+		settings := RuntimeSettings{}
+
+		// Set all current values (using pointers for RuntimeSettings)
+		watchdogIdle := appConfig.WatchDogIdle
+		watchdogBusy := appConfig.WatchDogBusy
+		watchdogEnabled := appConfig.WatchDog
+		singleBackend := appConfig.SingleBackend
+		parallelBackendRequests := appConfig.ParallelBackendRequests
+		threads := appConfig.Threads
+		contextSize := appConfig.ContextSize
+		f16 := appConfig.F16
+		debug := appConfig.Debug
+		cors := appConfig.CORS
+		csrf := appConfig.CSRF
+		corsAllowOrigins := appConfig.CORSAllowOrigins
+		p2pToken := appConfig.P2PToken
+		p2pNetworkID := appConfig.P2PNetworkID
+		federated := appConfig.Federated
+		galleries := appConfig.Galleries
+		backendGalleries := appConfig.BackendGalleries
+		autoloadGalleries := appConfig.AutoloadGalleries
+		autoloadBackendGalleries := appConfig.AutoloadBackendGalleries
+		apiKeys := appConfig.ApiKeys
+
+		settings.WatchdogIdleEnabled = &watchdogIdle
+		settings.WatchdogBusyEnabled = &watchdogBusy
+		settings.WatchdogEnabled = &watchdogEnabled
+		settings.SingleBackend = &singleBackend
+		settings.ParallelBackendRequests = &parallelBackendRequests
+		settings.Threads = &threads
+		settings.ContextSize = &contextSize
+		settings.F16 = &f16
+		settings.Debug = &debug
+		settings.CORS = &cors
+		settings.CSRF = &csrf
+		settings.CORSAllowOrigins = &corsAllowOrigins
+		settings.P2PToken = &p2pToken
+		settings.P2PNetworkID = &p2pNetworkID
+		settings.Federated = &federated
+		settings.Galleries = &galleries
+		settings.BackendGalleries = &backendGalleries
+		settings.AutoloadGalleries = &autoloadGalleries
+		settings.AutoloadBackendGalleries = &autoloadBackendGalleries
+		settings.ApiKeys = &apiKeys
+
+		var idleTimeout, busyTimeout string
+		if appConfig.WatchDogIdleTimeout > 0 {
+			idleTimeout = appConfig.WatchDogIdleTimeout.String()
+		} else {
+			idleTimeout = "15m" // default
+		}
+		if appConfig.WatchDogBusyTimeout > 0 {
+			busyTimeout = appConfig.WatchDogBusyTimeout.String()
+		} else {
+			busyTimeout = "5m" // default
+		}
+		settings.WatchdogIdleTimeout = &idleTimeout
+		settings.WatchdogBusyTimeout = &busyTimeout
+		return c.JSON(http.StatusOK, settings)
+	}
+}
+
+// UpdateSettingsEndpoint updates settings, saves to file, and applies immediately
+func UpdateSettingsEndpoint(app *application.Application) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		appConfig := app.ApplicationConfig()
+		startupConfig := app.StartupConfig()
+
+		if startupConfig == nil {
+			// Fallback if startup config not available
+			startupConfig = appConfig
+		}
+
+		body, err := io.ReadAll(c.Request().Body)
+		if err != nil {
+			return c.JSON(http.StatusBadRequest, SettingsResponse{
+				Success: false,
+				Error:   "Failed to read request body: " + err.Error(),
+			})
+		}
+
+		var settings RuntimeSettings
+		if err := json.Unmarshal(body, &settings); err != nil {
+			return c.JSON(http.StatusBadRequest, SettingsResponse{
+				Success: false,
+				Error:   "Failed to parse JSON: " + err.Error(),
+			})
+		}
+
+		// Validate timeouts if provided
+		if settings.WatchdogIdleTimeout != nil {
+			_, err := time.ParseDuration(*settings.WatchdogIdleTimeout)
+			if err != nil {
+				return c.JSON(http.StatusBadRequest, SettingsResponse{
+					Success: false,
+					Error:   "Invalid watchdog_idle_timeout format: " + err.Error(),
+				})
+			}
+		}
+		if settings.WatchdogBusyTimeout != nil {
+			_, err := time.ParseDuration(*settings.WatchdogBusyTimeout)
+			if err != nil {
+				return c.JSON(http.StatusBadRequest, SettingsResponse{
+					Success: false,
+					Error:   "Invalid watchdog_busy_timeout format: " + err.Error(),
+				})
+			}
+		}
+
+		// Save to file
+		if appConfig.DynamicConfigsDir == "" {
+			return c.JSON(http.StatusBadRequest, SettingsResponse{
+				Success: false,
+				Error:   "DynamicConfigsDir is not set",
+			})
+		}
+
+		settingsFile := filepath.Join(appConfig.DynamicConfigsDir, "runtime_settings.json")
+		settingsJSON, err := json.MarshalIndent(settings, "", "  ")
+		if err != nil {
+			return c.JSON(http.StatusInternalServerError, SettingsResponse{
+				Success: false,
+				Error:   "Failed to marshal settings: " + err.Error(),
+			})
+		}
+
+		if err := os.WriteFile(settingsFile, settingsJSON, 0600); err != nil {
+			return c.JSON(http.StatusInternalServerError, SettingsResponse{
+				Success: false,
+				Error:   "Failed to write settings file: " + err.Error(),
+			})
+		}
+
+		// Apply settings immediately, checking env var overrides per field
+		watchdogChanged := false
+		if settings.WatchdogEnabled != nil {
+			appConfig.WatchDog = *settings.WatchdogEnabled
+			watchdogChanged = true
+		}
+		if settings.WatchdogIdleEnabled != nil {
+			appConfig.WatchDogIdle = *settings.WatchdogIdleEnabled
+			if appConfig.WatchDogIdle {
+				appConfig.WatchDog = true
+			}
+			watchdogChanged = true
+		}
+		if settings.WatchdogBusyEnabled != nil {
+			appConfig.WatchDogBusy = *settings.WatchdogBusyEnabled
+			if appConfig.WatchDogBusy {
+				appConfig.WatchDog = true
+			}
+			watchdogChanged = true
+		}
+		if settings.WatchdogIdleTimeout != nil {
+			dur, _ := time.ParseDuration(*settings.WatchdogIdleTimeout)
+			appConfig.WatchDogIdleTimeout = dur
+			watchdogChanged = true
+		}
+		if settings.WatchdogBusyTimeout != nil {
+			dur, _ := time.ParseDuration(*settings.WatchdogBusyTimeout)
+			appConfig.WatchDogBusyTimeout = dur
+			watchdogChanged = true
+		}
+		if settings.SingleBackend != nil {
+			appConfig.SingleBackend = *settings.SingleBackend
+		}
+		if settings.ParallelBackendRequests != nil {
+			appConfig.ParallelBackendRequests = *settings.ParallelBackendRequests
+		}
+		if settings.Threads != nil {
+			appConfig.Threads = *settings.Threads
+		}
+		if settings.ContextSize != nil {
+			appConfig.ContextSize = *settings.ContextSize
+		}
+		if settings.F16 != nil {
+			appConfig.F16 = *settings.F16
+		}
+		if settings.Debug != nil {
+			appConfig.Debug = *settings.Debug
+		}
+		if settings.CORS != nil {
+			appConfig.CORS = *settings.CORS
+		}
+		if settings.CSRF != nil {
+			appConfig.CSRF = *settings.CSRF
+		}
+		if settings.CORSAllowOrigins != nil {
+			appConfig.CORSAllowOrigins = *settings.CORSAllowOrigins
+		}
+		if settings.P2PToken != nil {
+			appConfig.P2PToken = *settings.P2PToken
+		}
+		if settings.P2PNetworkID != nil {
+			appConfig.P2PNetworkID = *settings.P2PNetworkID
+		}
+		if settings.Federated != nil {
+			appConfig.Federated = *settings.Federated
+		}
+		if settings.Galleries != nil {
+			appConfig.Galleries = *settings.Galleries
+		}
+		if settings.BackendGalleries != nil {
+			appConfig.BackendGalleries = *settings.BackendGalleries
+		}
+		if settings.AutoloadGalleries != nil {
+			appConfig.AutoloadGalleries = *settings.AutoloadGalleries
+		}
+		if settings.AutoloadBackendGalleries != nil {
+			appConfig.AutoloadBackendGalleries = *settings.AutoloadBackendGalleries
+		}
+		if settings.ApiKeys != nil {
+			// API keys from env vars (startup) should be kept, runtime settings keys are added
+			// Combine startup keys (env vars) with runtime settings keys
+			envKeys := startupConfig.ApiKeys
+			runtimeKeys := *settings.ApiKeys
+			// Merge: env keys first (they take precedence), then runtime keys
+			appConfig.ApiKeys = append(envKeys, runtimeKeys...)
+
+			// Note: We only save to runtime_settings.json (not api_keys.json) to avoid duplication
+			// The runtime_settings.json is the unified config file. If api_keys.json exists,
+			// it will be loaded first, but runtime_settings.json takes precedence and deduplicates.
+		}
+
+		// Restart watchdog if settings changed
+		if watchdogChanged {
+			if settings.WatchdogEnabled != nil && !*settings.WatchdogEnabled || settings.WatchdogEnabled == nil {
+				if err := app.StopWatchdog(); err != nil {
+					log.Error().Err(err).Msg("Failed to stop watchdog")
+					return c.JSON(http.StatusInternalServerError, SettingsResponse{
+						Success: false,
+						Error:   "Settings saved but failed to stop watchdog: " + err.Error(),
+					})
+				}
+			} else {
+				if err := app.RestartWatchdog(); err != nil {
+					log.Error().Err(err).Msg("Failed to restart watchdog")
+					return c.JSON(http.StatusInternalServerError, SettingsResponse{
+						Success: false,
+						Error:   "Settings saved but failed to restart watchdog: " + err.Error(),
+					})
+				}
+			}
+		}
+
+		// Restart P2P if P2P settings changed
+		p2pChanged := settings.P2PToken != nil || settings.P2PNetworkID != nil || settings.Federated != nil
+		if p2pChanged {
+			if settings.P2PToken != nil && *settings.P2PToken == "" {
+				// stop P2P
+				if err := app.StopP2P(); err != nil {
+					log.Error().Err(err).Msg("Failed to stop P2P")
+					return c.JSON(http.StatusInternalServerError, SettingsResponse{
+						Success: false,
+						Error:   "Settings saved but failed to stop P2P: " + err.Error(),
+					})
+				}
+			} else {
+				if settings.P2PToken != nil && *settings.P2PToken == "0" {
+					// generate a token if users sets 0 (disabled)
+					token := p2p.GenerateToken(60, 60)
+					settings.P2PToken = &token
+					appConfig.P2PToken = token
+				}
+				// Stop existing P2P
+				if err := app.RestartP2P(); err != nil {
+					log.Error().Err(err).Msg("Failed to stop P2P")
+					return c.JSON(http.StatusInternalServerError, SettingsResponse{
+						Success: false,
+						Error:   "Settings saved but failed to stop P2P: " + err.Error(),
+					})
+				}
+			}
+		}
+
+		return c.JSON(http.StatusOK, SettingsResponse{
+			Success: true,
+			Message: "Settings updated successfully",
+		})
+	}
+}

core/http/endpoints/localai/welcome.go

@@ -43,17 +43,18 @@ func WelcomeEndpoint(appConfig *config.ApplicationConfig,
 		processingModels, taskTypes := opcache.GetStatus()
 
 		summary := map[string]interface{}{
-			"Title":             "LocalAI API - " + internal.PrintableVersion(),
+			"Title":                  "LocalAI API - " + internal.PrintableVersion(),
-			"Version":           internal.PrintableVersion(),
+			"Version":                internal.PrintableVersion(),
-			"BaseURL":           middleware.BaseURL(c),
+			"BaseURL":                middleware.BaseURL(c),
-			"Models":            modelsWithoutConfig,
+			"Models":                 modelsWithoutConfig,
-			"ModelsConfig":      modelConfigs,
+			"ModelsConfig":           modelConfigs,
-			"GalleryConfig":     galleryConfigs,
+			"GalleryConfig":          galleryConfigs,
-			"ApplicationConfig": appConfig,
+			"ApplicationConfig":      appConfig,
-			"ProcessingModels":  processingModels,
+			"ProcessingModels":       processingModels,
-			"TaskTypes":         taskTypes,
+			"TaskTypes":              taskTypes,
-			"LoadedModels":      loadedModelsMap,
+			"LoadedModels":           loadedModelsMap,
-			"InstalledBackends": installedBackends,
+			"InstalledBackends":      installedBackends,
+			"DisableRuntimeSettings": appConfig.DisableRuntimeSettings,
 		}
 
 		contentType := c.Request().Header.Get("Content-Type")

core/http/endpoints/openai/mcp.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"strings"
+	"net"
 	"time"
 
 	"github.com/labstack/echo/v4"
@@ -75,7 +75,11 @@ func MCPCompletionEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader,
 			fragment = fragment.AddMessage(message.Role, message.StringContent)
 		}
 
-		port := appConfig.APIAddress[strings.LastIndex(appConfig.APIAddress, ":")+1:]
+		_, port, err := net.SplitHostPort(appConfig.APIAddress)
+		if err != nil {
+			return err
+		}
+
 		apiKey := ""
 		if appConfig.ApiKeys != nil {
 			apiKey = appConfig.ApiKeys[0]

core/http/routes/ui.go

@@ -23,6 +23,17 @@ func RegisterUIRoutes(app *echo.Echo,
 	app.GET("/", localai.WelcomeEndpoint(appConfig, cl, ml, processingOps))
 	app.GET("/manage", localai.WelcomeEndpoint(appConfig, cl, ml, processingOps))
 
+	if !appConfig.DisableRuntimeSettings {
+		// Settings page
+		app.GET("/settings", func(c echo.Context) error {
+			summary := map[string]interface{}{
+				"Title":   "LocalAI - Settings",
+				"BaseURL": middleware.BaseURL(c),
+			}
+			return c.Render(200, "views/settings", summary)
+		})
+	}
+
 	// P2P
 	app.GET("/p2p/", func(c echo.Context) error {
 		summary := map[string]interface{}{

core/http/routes/ui_api.go

@@ -12,8 +12,10 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/application"
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/core/http/endpoints/localai"
 	"github.com/mudler/LocalAI/core/p2p"
 	"github.com/mudler/LocalAI/core/services"
 	"github.com/mudler/LocalAI/pkg/model"
@@ -21,7 +23,7 @@ import (
 )
 
 // RegisterUIAPIRoutes registers JSON API routes for the web UI
-func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, galleryService *services.GalleryService, opcache *services.OpCache) {
+func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, galleryService *services.GalleryService, opcache *services.OpCache, applicationInstance *application.Application) {
 
 	// Operations API - Get all current operations (models + backends)
 	app.GET("/api/operations", func(c echo.Context) error {
@@ -264,17 +266,17 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 		installedModelsCount := len(modelConfigs) + len(modelsWithoutConfig)
 
 		return c.JSON(200, map[string]interface{}{
-			"models":            modelsJSON,
+			"models":           modelsJSON,
-			"repositories":      appConfig.Galleries,
+			"repositories":     appConfig.Galleries,
-			"allTags":           tags,
+			"allTags":          tags,
-			"processingModels":  processingModelsData,
+			"processingModels": processingModelsData,
-			"taskTypes":         taskTypes,
+			"taskTypes":        taskTypes,
-			"availableModels":   totalModels,
+			"availableModels":  totalModels,
-			"installedModels":   installedModelsCount,
+			"installedModels":  installedModelsCount,
-			"currentPage":       pageNum,
+			"currentPage":      pageNum,
-			"totalPages":        totalPages,
+			"totalPages":       totalPages,
-			"prevPage":          prevPage,
+			"prevPage":         prevPage,
-			"nextPage":          nextPage,
+			"nextPage":         nextPage,
 		})
 	})
 
@@ -802,4 +804,10 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
 			},
 		})
 	})
+
+	if !appConfig.DisableRuntimeSettings {
+		// Settings API
+		app.GET("/api/settings", localai.GetSettingsEndpoint(applicationInstance))
+		app.POST("/api/settings", localai.UpdateSettingsEndpoint(applicationInstance))
+	}
 }

core/http/static/chat.js

@@ -1382,6 +1382,12 @@ document.addEventListener('DOMContentLoaded', function() {
     if (chatData) {
       try {
         const data = JSON.parse(chatData);
+        
+        // Set MCP mode if provided
+        if (data.mcpMode === true && Alpine.store("chat")) {
+          Alpine.store("chat").mcpMode = true;
+        }
+        
         const input = document.getElementById('input');
         
         if (input && data.message) {
@@ -1417,6 +1423,9 @@ document.addEventListener('DOMContentLoaded', function() {
               processAndSendMessage(input.value);
             }
           }, 500);
+        } else {
+          // No message, but might have mcpMode - clear localStorage
+          localStorage.removeItem('localai_index_chat_data');
         }
       } catch (error) {
         console.error('Error processing chat data from index:', error);

core/http/views/chat.html

@@ -44,8 +44,25 @@ SOFTWARE.
     // Function to initialize store
     function __initChatStore() {
       if (!window.Alpine) return;
+      
+      // Check for MCP mode from localStorage (set by index page)
+      // Note: We don't clear localStorage here - chat.js will handle that after reading all data
+      let initialMcpMode = false;
+      try {
+        const chatData = localStorage.getItem('localai_index_chat_data');
+        if (chatData) {
+          const parsed = JSON.parse(chatData);
+          if (parsed.mcpMode === true) {
+            initialMcpMode = true;
+          }
+        }
+      } catch (e) {
+        console.error('Error reading MCP mode from localStorage:', e);
+      }
+      
       if (Alpine.store("chat")) {
         Alpine.store("chat").contextSize = __chatContextSize;
+        Alpine.store("chat").mcpMode = initialMcpMode;
         return;
       }
 
@@ -53,7 +70,7 @@ SOFTWARE.
         history: [],
         languages: [undefined],
         systemPrompt: "",
-        mcpMode: false,
+        mcpMode: initialMcpMode,
         contextSize: __chatContextSize,
         tokenUsage: {
           promptTokens: 0,

core/http/views/index.html

@@ -128,6 +128,9 @@
                 audioFiles: [], 
                 textFiles: [],
                 attachedFiles: [],
+                mcpMode: false,
+                mcpAvailable: false,
+                mcpModels: {},
                 currentPlaceholder: 'Send a message...',
                 placeholderIndex: 0,
                 charIndex: 0,
@@ -163,6 +166,8 @@
                 init() {
                     window.currentPlaceholderText = this.currentPlaceholder;
                     this.startTypingAnimation();
+                    // Build MCP models map from data attributes
+                    this.buildMCPModelsMap();
                     // Select first model by default
                     this.$nextTick(() => {
                         const select = this.$el.querySelector('select');
@@ -171,9 +176,43 @@
                             const firstModelOption = select.options[1];
                             if (firstModelOption && firstModelOption.value) {
                                 this.selectedModel = firstModelOption.value;
+                                this.checkMCPAvailability();
                             }
                         }
                     });
+                    // Watch for changes to selectedModel to update MCP availability
+                    this.$watch('selectedModel', () => {
+                        this.checkMCPAvailability();
+                    });
+                },
+                buildMCPModelsMap() {
+                    const select = this.$el.querySelector('select');
+                    if (!select) return;
+                    this.mcpModels = {};
+                    for (let i = 0; i < select.options.length; i++) {
+                        const option = select.options[i];
+                        if (option.value) {
+                            const hasMcpAttr = option.getAttribute('data-has-mcp');
+                            this.mcpModels[option.value] = hasMcpAttr === 'true';
+                        }
+                    }
+                    // Debug: uncomment to see the MCP models map
+                    // console.log('MCP Models Map:', this.mcpModels);
+                },
+                checkMCPAvailability() {
+                    if (!this.selectedModel) {
+                        this.mcpAvailable = false;
+                        this.mcpMode = false;
+                        return;
+                    }
+                    // Check MCP availability from the map
+                    const hasMCP = this.mcpModels[this.selectedModel] === true;
+                    this.mcpAvailable = hasMCP;
+                    // Debug: uncomment to see what's happening
+                    // console.log('MCP Check:', { model: this.selectedModel, hasMCP, mcpAvailable: this.mcpAvailable, map: this.mcpModels });
+                    if (!hasMCP) {
+                        this.mcpMode = false;
+                    }
                 },
                 startTypingAnimation() {
                     if (this.isTyping) return;
@@ -268,24 +307,50 @@
                     }
                 }
             }">
-                <!-- Model Selector -->
+                <!-- Model Selector with MCP Toggle -->
                 <div class="mb-4">
                     <label class="block text-sm font-medium text-[#94A3B8] mb-2">Select Model</label>
-                    <select
+                    <div class="flex items-center gap-3">
-                        x-model="selectedModel"
+                        <select
-                        class="w-full bg-[#1E293B] text-[#E5E7EB] border border-[#38BDF8]/20 focus:border-[#38BDF8] focus:ring-2 focus:ring-[#38BDF8]/50 rounded-lg p-3 appearance-none"
+                            x-model="selectedModel"
-                        required
+                            @change="$nextTick(() => checkMCPAvailability())"
-                    >
+                            class="flex-1 bg-[#1E293B] text-[#E5E7EB] border border-[#38BDF8]/20 focus:border-[#38BDF8] focus:ring-2 focus:ring-[#38BDF8]/50 rounded-lg p-3 appearance-none"
-                        <option value="" disabled class="text-[#94A3B8]">Select a model to chat with...</option>
+                            required
-                        {{ range .ModelsConfig }}
+                        >
-                        {{ $cfg := . }}
+                            <option value="" disabled class="text-[#94A3B8]">Select a model to chat with...</option>
-                        {{ range .KnownUsecaseStrings }}
+                            {{ range .ModelsConfig }}
-                            {{ if eq . "FLAG_CHAT" }}
+                            {{ $cfg := . }}
-                            <option value="{{$cfg.Name}}" class="bg-[#1E293B] text-[#E5E7EB]">{{$cfg.Name}}</option>
+                            {{ $hasMCP := or (ne $cfg.MCP.Servers "") (ne $cfg.MCP.Stdio "") }}
+                            {{ range .KnownUsecaseStrings }}
+                                {{ if eq . "FLAG_CHAT" }}
+                                <option value="{{$cfg.Name}}" data-has-mcp="{{if $hasMCP}}true{{else}}false{{end}}" class="bg-[#1E293B] text-[#E5E7EB]">{{$cfg.Name}}</option>
+                                {{ end }}
                             {{ end }}
-                        {{ end }}
+                            {{ end }}
-                        {{ end }}
+                        </select>
-                    </select>
+                        
+                        <!-- Compact MCP Toggle - Show only if MCP is available for selected model -->
+                        <div 
+                            x-show="mcpAvailable"
+                            class="flex items-center gap-2 px-3 py-2 text-xs rounded text-[#E5E7EB] bg-[#1E293B] border border-[#38BDF8]/20 whitespace-nowrap">
+                            <i class="fa-solid fa-plug text-[#38BDF8] text-sm"></i>
+                            <span class="text-[#94A3B8]">MCP</span>
+                            <label class="relative inline-flex items-center cursor-pointer ml-1">
+                                <input type="checkbox" id="index_mcp_toggle" class="sr-only peer" x-model="mcpMode">
+                                <div class="w-9 h-5 bg-[#101827] peer-focus:outline-none peer-focus:ring-2 peer-focus:ring-[#38BDF8]/30 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-[#1E293B] after:border after:rounded-full after:h-4 after:w-4 after:transition-all peer-checked:bg-[#38BDF8]"></div>
+                            </label>
+                        </div>
+                    </div>
+                    
+                    <!-- MCP Mode Notification - Compact tooltip style -->
+                    <div 
+                        x-show="mcpMode && mcpAvailable"
+                        class="mt-2 p-2 bg-[#38BDF8]/10 border border-[#38BDF8]/30 rounded text-[#94A3B8] text-xs">
+                        <div class="flex items-start space-x-2">
+                            <i class="fa-solid fa-info-circle text-[#38BDF8] mt-0.5 text-xs"></i>
+                            <p class="text-[#94A3B8]">Non-streaming mode active. Responses may take longer to process.</p>
+                        </div>
+                    </div>
                 </div>
 
                 <!-- Input Bar -->
@@ -476,12 +541,20 @@ function startChat(event) {
         return;
     }
 
+    // Get MCP mode from checkbox (if available)
+    let mcpMode = false;
+    const mcpToggle = document.getElementById('index_mcp_toggle');
+    if (mcpToggle && mcpToggle.checked) {
+        mcpMode = true;
+    }
+
     // Store message and files in localStorage for chat page to pick up
     const chatData = {
         message: message,
         imageFiles: [],
         audioFiles: [],
-        textFiles: []
+        textFiles: [],
+        mcpMode: mcpMode
     };
 
     // Convert files to base64 for storage

core/http/views/manage.html

@@ -66,6 +66,14 @@
                 <i class="fas fa-cogs mr-1.5 text-[10px]"></i>
                 <span>Backend Gallery</span>
             </a>
+
+            {{ if not .DisableRuntimeSettings }}
+            <a href="/settings"
+               class="inline-flex items-center bg-[#1E293B] hover:bg-[#1E293B]/80 border border-[#38BDF8]/20 text-[#E5E7EB] py-1.5 px-3 rounded text-xs font-medium transition-colors">
+                <i class="fas fa-cog mr-1.5 text-[10px]"></i>
+                <span>Settings</span>
+            </a>
+            {{ end }}
         </div>
 
         <!-- Models Section -->

core/http/views/settings.html

@@ -0,0 +1,653 @@
+<!DOCTYPE html>
+<html lang="en">
+{{template "views/partials/head" .}}
+
+<body class="bg-[#101827] text-[#E5E7EB]">
+<div class="flex flex-col min-h-screen" x-data="settingsDashboard()">
+
+    {{template "views/partials/navbar" .}}
+
+    <!-- Notifications -->
+    <div class="fixed top-20 right-4 z-50 space-y-2" style="max-width: 400px;">
+        <template x-for="notification in notifications" :key="notification.id">
+            <div x-show="true" 
+                 x-transition:enter="transition ease-out duration-200"
+                 x-transition:enter-start="opacity-0"
+                 x-transition:enter-end="opacity-100"
+                 x-transition:leave="transition ease-in duration-150"
+                 x-transition:leave-start="opacity-100"
+                 x-transition:leave-end="opacity-0"
+                 :class="notification.type === 'error' ? 'bg-red-500' : 'bg-green-500'"
+                 class="rounded-lg p-4 text-white flex items-start space-x-3">
+                <div class="flex-shrink-0">
+                    <i :class="notification.type === 'error' ? 'fas fa-exclamation-circle' : 'fas fa-check-circle'" class="text-xl"></i>
+                </div>
+                <div class="flex-1 min-w-0">
+                    <p class="text-sm font-medium break-words" x-text="notification.message"></p>
+                </div>
+                <button @click="dismissNotification(notification.id)" class="flex-shrink-0 text-white hover:opacity-80 transition-opacity">
+                    <i class="fas fa-times"></i>
+                </button>
+            </div>
+        </template>
+    </div>
+
+    <div class="container mx-auto px-4 py-6 flex-grow max-w-4xl">
+        <!-- Header -->
+        <div class="mb-6">
+            <div class="flex items-center justify-between mb-2">
+                <h1 class="text-2xl font-semibold text-[#E5E7EB]">
+                    Application Settings
+                </h1>
+                <a href="/manage" 
+                   class="inline-flex items-center text-[#94A3B8] hover:text-[#E5E7EB] transition-colors">
+                    <i class="fas fa-arrow-left mr-2 text-sm"></i>
+                    <span class="text-sm">Back to Manage</span>
+                </a>
+            </div>
+            <p class="text-sm text-[#94A3B8]">Configure watchdog and backend request settings</p>
+        </div>
+
+        <!-- Settings Form -->
+        <form @submit.prevent="saveSettings()" class="space-y-6">
+            <!-- Watchdog Settings Section -->
+            <div class="bg-[#1E293B] border border-[#38BDF8]/20 rounded-lg p-6">
+                <h2 class="text-xl font-semibold text-[#E5E7EB] mb-4 flex items-center">
+                    <i class="fas fa-shield-alt mr-2 text-[#38BDF8] text-sm"></i>
+                    Watchdog Settings
+                </h2>
+                <p class="text-xs text-[#94A3B8] mb-4">
+                    Configure automatic monitoring and management of backend processes
+                </p>
+
+                <div class="space-y-4">
+                    <!-- Enable Watchdog -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Enable Watchdog</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Enable automatic monitoring of backend processes</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.watchdog_enabled" 
+                                   @change="updateWatchdogEnabled()"
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#38BDF8]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#38BDF8]"></div>
+                        </label>
+                    </div>
+
+                    <!-- Enable Idle Check -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Enable Idle Check</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Automatically stop backends that are idle for too long</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.watchdog_idle_enabled" 
+                                   :disabled="!settings.watchdog_enabled"
+                                   class="sr-only peer" :class="!settings.watchdog_enabled ? 'opacity-50' : ''">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#38BDF8]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#38BDF8]"></div>
+                        </label>
+                    </div>
+
+                    <!-- Idle Timeout -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">Idle Timeout</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">Time before an idle backend is stopped (e.g., 15m, 1h)</p>
+                        <input type="text" x-model="settings.watchdog_idle_timeout" 
+                               :disabled="!settings.watchdog_idle_enabled"
+                               placeholder="15m"
+                               class="w-full px-3 py-2 bg-[#101827] border border-[#38BDF8]/20 rounded text-sm text-[#E5E7EB] focus:outline-none focus:ring-2 focus:ring-[#38BDF8]/50"
+                               :class="!settings.watchdog_idle_enabled ? 'opacity-50 cursor-not-allowed' : ''">
+                    </div>
+
+                    <!-- Enable Busy Check -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Enable Busy Check</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Automatically stop backends that are busy for too long (stuck processes)</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.watchdog_busy_enabled" 
+                                   :disabled="!settings.watchdog_enabled"
+                                   class="sr-only peer" :class="!settings.watchdog_enabled ? 'opacity-50' : ''">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#38BDF8]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#38BDF8]"></div>
+                        </label>
+                    </div>
+
+                    <!-- Busy Timeout -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">Busy Timeout</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">Time before a busy backend is stopped (e.g., 5m, 30m)</p>
+                        <input type="text" x-model="settings.watchdog_busy_timeout" 
+                               :disabled="!settings.watchdog_busy_enabled"
+                               placeholder="5m"
+                               class="w-full px-3 py-2 bg-[#101827] border border-[#38BDF8]/20 rounded text-sm text-[#E5E7EB] focus:outline-none focus:ring-2 focus:ring-[#38BDF8]/50"
+                               :class="!settings.watchdog_busy_enabled ? 'opacity-50 cursor-not-allowed' : ''">
+                    </div>
+                </div>
+            </div>
+
+            <!-- Backend Request Settings Section -->
+            <div class="bg-[#1E293B] border border-[#8B5CF6]/20 rounded-lg p-6">
+                <h2 class="text-xl font-semibold text-[#E5E7EB] mb-4 flex items-center">
+                    <i class="fas fa-cogs mr-2 text-[#8B5CF6] text-sm"></i>
+                    Backend Request Settings
+                </h2>
+                <p class="text-xs text-[#94A3B8] mb-4">
+                    Configure how backends handle multiple requests
+                </p>
+
+                <div class="space-y-4">
+                    <!-- Single Backend Mode -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Single Backend Mode</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Allow only one backend to be active at a time</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.single_backend" 
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#8B5CF6]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#8B5CF6]"></div>
+                        </label>
+                    </div>
+
+                    <!-- Parallel Backend Requests -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Parallel Backend Requests</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Enable backends to handle multiple requests in parallel (if supported)</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.parallel_backend_requests" 
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#8B5CF6]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#8B5CF6]"></div>
+                        </label>
+                    </div>
+                </div>
+            </div>
+
+            <!-- Performance Settings Section -->
+            <div class="bg-[#1E293B] border border-[#10B981]/20 rounded-lg p-6">
+                <h2 class="text-xl font-semibold text-[#E5E7EB] mb-4 flex items-center">
+                    <i class="fas fa-tachometer-alt mr-2 text-[#10B981] text-sm"></i>
+                    Performance Settings
+                </h2>
+                <p class="text-xs text-[#94A3B8] mb-4">
+                    Configure default performance parameters for models
+                </p>
+
+                <div class="space-y-4">
+                    <!-- Threads -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">Default Threads</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">Number of threads to use for model inference (0 = auto)</p>
+                        <input type="number" x-model="settings.threads" 
+                               min="0"
+                               placeholder="0"
+                               class="w-full px-3 py-2 bg-[#101827] border border-[#10B981]/20 rounded text-sm text-[#E5E7EB] focus:outline-none focus:ring-2 focus:ring-[#10B981]/50">
+                    </div>
+
+                    <!-- Context Size -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">Default Context Size</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">Default context window size for models</p>
+                        <input type="number" x-model="settings.context_size" 
+                               min="0"
+                               placeholder="512"
+                               class="w-full px-3 py-2 bg-[#101827] border border-[#10B981]/20 rounded text-sm text-[#E5E7EB] focus:outline-none focus:ring-2 focus:ring-[#10B981]/50">
+                    </div>
+
+                    <!-- F16 -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">F16 Precision</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Use 16-bit floating point precision</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.f16" 
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#10B981]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#10B981]"></div>
+                        </label>
+                    </div>
+
+                    <!-- Debug -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Debug Mode</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Enable debug logging</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.debug" 
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#10B981]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#10B981]"></div>
+                        </label>
+                    </div>
+                </div>
+            </div>
+
+            <!-- API Settings Section -->
+            <div class="bg-[#1E293B] border border-[#F59E0B]/20 rounded-lg p-6">
+                <h2 class="text-xl font-semibold text-[#E5E7EB] mb-4 flex items-center">
+                    <i class="fas fa-globe mr-2 text-[#F59E0B] text-sm"></i>
+                    API Settings
+                </h2>
+                <p class="text-xs text-[#94A3B8] mb-4">
+                    Configure CORS and CSRF protection
+                </p>
+
+                <div class="space-y-4">
+                    <!-- CORS -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Enable CORS</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Enable Cross-Origin Resource Sharing</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.cors" 
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#F59E0B]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#F59E0B]"></div>
+                        </label>
+                    </div>
+
+                    <!-- CORS Allow Origins -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">CORS Allow Origins</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">Comma-separated list of allowed origins</p>
+                        <input type="text" x-model="settings.cors_allow_origins" 
+                               placeholder="*"
+                               class="w-full px-3 py-2 bg-[#101827] border border-[#F59E0B]/20 rounded text-sm text-[#E5E7EB] focus:outline-none focus:ring-2 focus:ring-[#F59E0B]/50">
+                    </div>
+
+                    <!-- CSRF -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Enable CSRF Protection</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Enable Cross-Site Request Forgery protection</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.csrf" 
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#F59E0B]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#F59E0B]"></div>
+                        </label>
+                    </div>
+                </div>
+            </div>
+
+            <!-- P2P Settings Section -->
+            <div class="bg-[#1E293B] border border-[#EC4899]/20 rounded-lg p-6">
+                <h2 class="text-xl font-semibold text-[#E5E7EB] mb-4 flex items-center">
+                    <i class="fas fa-network-wired mr-2 text-[#EC4899] text-sm"></i>
+                    P2P Settings
+                </h2>
+                <p class="text-xs text-[#94A3B8] mb-4">
+                    Configure peer-to-peer networking
+                </p>
+
+                <div class="space-y-4">
+                    <!-- P2P Token -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">P2P Token</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">Authentication token for P2P network (set to 0 to generate a new token)</p>
+                        <input type="text" x-model="settings.p2p_token" 
+                               placeholder=""
+                               class="w-full px-3 py-2 bg-[#101827] border border-[#EC4899]/20 rounded text-sm text-[#E5E7EB] focus:outline-none focus:ring-2 focus:ring-[#EC4899]/50">
+                    </div>
+
+                    <!-- P2P Network ID -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">P2P Network ID</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">Network identifier for P2P connections</p>
+                        <input type="text" x-model="settings.p2p_network_id" 
+                               placeholder=""
+                               class="w-full px-3 py-2 bg-[#101827] border border-[#EC4899]/20 rounded text-sm text-[#E5E7EB] focus:outline-none focus:ring-2 focus:ring-[#EC4899]/50">
+                    </div>
+
+                    <!-- Federated -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Federated Mode</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Enable federated instance mode</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.federated" 
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#EC4899]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#EC4899]"></div>
+                        </label>
+                    </div>
+                </div>
+            </div>
+
+            <!-- API Keys Settings Section -->
+            <div class="bg-[#1E293B] border border-[#EF4444]/20 rounded-lg p-6">
+                <h2 class="text-xl font-semibold text-[#E5E7EB] mb-4 flex items-center">
+                    <i class="fas fa-key mr-2 text-[#EF4444] text-sm"></i>
+                    API Keys
+                </h2>
+                <p class="text-xs text-[#94A3B8] mb-4">
+                    Manage API keys for authentication. Keys from environment variables are always included.
+                </p>
+
+                <div class="space-y-4">
+                    <!-- API Keys List -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">API Keys</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">List of API keys (one per line or comma-separated)</p>
+                        <textarea x-model="settings.api_keys_text" 
+                                  rows="4"
+                                  placeholder="sk-1234567890abcdef&#10;sk-0987654321fedcba"
+                                  class="w-full px-3 py-2 bg-[#101827] border border-[#EF4444]/20 rounded text-sm text-[#E5E7EB] font-mono focus:outline-none focus:ring-2 focus:ring-[#EF4444]/50"></textarea>
+                        <p class="text-xs text-[#94A3B8] mt-1">Note: API keys are sensitive. Handle with care.</p>
+                    </div>
+                </div>
+            </div>
+
+            <!-- Gallery Settings Section -->
+            <div class="bg-[#1E293B] border border-[#6366F1]/20 rounded-lg p-6">
+                <h2 class="text-xl font-semibold text-[#E5E7EB] mb-4 flex items-center">
+                    <i class="fas fa-images mr-2 text-[#6366F1] text-sm"></i>
+                    Gallery Settings
+                </h2>
+                <p class="text-xs text-[#94A3B8] mb-4">
+                    Configure model and backend galleries
+                </p>
+
+                <div class="space-y-4">
+                    <!-- Autoload Galleries -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Autoload Galleries</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Automatically load model galleries on startup</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.autoload_galleries" 
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#6366F1]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#6366F1]"></div>
+                        </label>
+                    </div>
+
+                    <!-- Autoload Backend Galleries -->
+                    <div class="flex items-center justify-between">
+                        <div>
+                            <label class="text-sm font-medium text-[#E5E7EB]">Autoload Backend Galleries</label>
+                            <p class="text-xs text-[#94A3B8] mt-1">Automatically load backend galleries on startup</p>
+                        </div>
+                        <label class="relative inline-flex items-center cursor-pointer">
+                            <input type="checkbox" x-model="settings.autoload_backend_galleries" 
+                                   class="sr-only peer">
+                            <div class="w-11 h-6 bg-[#101827] peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-[#6366F1]/20 rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-[#6366F1]"></div>
+                        </label>
+                    </div>
+
+                    <!-- Galleries (JSON) -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">Model Galleries (JSON)</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">Array of gallery objects with 'url' and 'name' fields</p>
+                        <textarea x-model="settings.galleries_json" 
+                                  rows="4"
+                                  placeholder='[{"url": "https://example.com", "name": "Example Gallery"}]'
+                                  class="w-full px-3 py-2 bg-[#101827] border border-[#6366F1]/20 rounded text-sm text-[#E5E7EB] font-mono focus:outline-none focus:ring-2 focus:ring-[#6366F1]/50"></textarea>
+                    </div>
+
+                    <!-- Backend Galleries (JSON) -->
+                    <div>
+                        <label class="block text-sm font-medium text-[#E5E7EB] mb-2">Backend Galleries (JSON)</label>
+                        <p class="text-xs text-[#94A3B8] mb-2">Array of backend gallery objects with 'url' and 'name' fields</p>
+                        <textarea x-model="settings.backend_galleries_json" 
+                                  rows="4"
+                                  placeholder='[{"url": "https://example.com", "name": "Example Backend Gallery"}]'
+                                  class="w-full px-3 py-2 bg-[#101827] border border-[#6366F1]/20 rounded text-sm text-[#E5E7EB] font-mono focus:outline-none focus:ring-2 focus:ring-[#6366F1]/50"></textarea>
+                    </div>
+                </div>
+            </div>
+
+            <!-- Source Info -->
+            <div class="bg-yellow-500/10 border border-yellow-500/20 rounded-lg p-4" x-show="sourceInfo">
+                <div class="flex items-start">
+                    <i class="fas fa-info-circle text-yellow-400 mr-2 mt-0.5"></i>
+                    <div class="flex-1">
+                        <p class="text-sm text-yellow-300 font-medium mb-1">Configuration Source</p>
+                        <p class="text-xs text-yellow-200" x-text="'Settings are currently loaded from: ' + sourceInfo"></p>
+                        <p class="text-xs text-yellow-200 mt-1" x-show="sourceInfo === 'env'">
+                            Environment variables take precedence. To modify settings via the UI, unset the relevant environment variables first.
+                        </p>
+                    </div>
+                </div>
+            </div>
+
+            <!-- Save Button -->
+            <div class="flex justify-end">
+                <button type="submit" 
+                        :disabled="saving"
+                        class="inline-flex items-center bg-[#38BDF8] hover:bg-[#38BDF8]/90 disabled:opacity-50 disabled:cursor-not-allowed text-white py-2 px-6 rounded-lg font-medium transition-colors">
+                    <i class="fas fa-save mr-2" :class="saving ? 'fa-spin fa-spinner' : ''"></i>
+                    <span x-text="saving ? 'Saving...' : 'Save Settings'"></span>
+                </button>
+            </div>
+        </form>
+    </div>
+
+    {{template "views/partials/footer" .}}
+</div>
+
+<script>
+function settingsDashboard() {
+    return {
+        notifications: [],
+        settings: {
+            watchdog_enabled: false,
+            watchdog_idle_enabled: false,
+            watchdog_busy_enabled: false,
+            watchdog_idle_timeout: '15m',
+            watchdog_busy_timeout: '5m',
+            single_backend: false,
+            parallel_backend_requests: false,
+            threads: 0,
+            context_size: 0,
+            f16: false,
+            debug: false,
+            cors: false,
+            csrf: false,
+            cors_allow_origins: '',
+            p2p_token: '',
+            p2p_network_id: '',
+            federated: false,
+            autoload_galleries: false,
+            autoload_backend_galleries: false,
+            galleries_json: '[]',
+            backend_galleries_json: '[]',
+            api_keys_text: ''
+        },
+        sourceInfo: '',
+        saving: false,
+        
+        init() {
+            this.loadSettings();
+        },
+        
+        async loadSettings() {
+            try {
+                const response = await fetch('/api/settings');
+                const data = await response.json();
+                
+                if (response.ok) {
+                    this.settings = {
+                        watchdog_enabled: data.watchdog_enabled,
+                        watchdog_idle_enabled: data.watchdog_idle_enabled,
+                        watchdog_busy_enabled: data.watchdog_busy_enabled,
+                        watchdog_idle_timeout: data.watchdog_idle_timeout || '15m',
+                        watchdog_busy_timeout: data.watchdog_busy_timeout || '5m',
+                        single_backend: data.single_backend,
+                        parallel_backend_requests: data.parallel_backend_requests,
+                        threads: data.threads || 0,
+                        context_size: data.context_size || 0,
+                        f16: data.f16 || false,
+                        debug: data.debug || false,
+                        cors: data.cors || false,
+                        csrf: data.csrf || false,
+                        cors_allow_origins: data.cors_allow_origins || '',
+                        p2p_token: data.p2p_token || '',
+                        p2p_network_id: data.p2p_network_id || '',
+                        federated: data.federated || false,
+                        autoload_galleries: data.autoload_galleries || false,
+                        autoload_backend_galleries: data.autoload_backend_galleries || false,
+                        galleries_json: JSON.stringify(data.galleries || [], null, 2),
+                        backend_galleries_json: JSON.stringify(data.backend_galleries || [], null, 2),
+                        api_keys_text: (data.api_keys || []).join('\n')
+                    };
+                    this.sourceInfo = data.source || 'default';
+                } else {
+                    this.addNotification('Failed to load settings: ' + (data.error || 'Unknown error'), 'error');
+                }
+            } catch (error) {
+                console.error('Error loading settings:', error);
+                this.addNotification('Failed to load settings: ' + error.message, 'error');
+            }
+        },
+        
+        updateWatchdogEnabled() {
+            if (!this.settings.watchdog_enabled) {
+                this.settings.watchdog_idle_enabled = false;
+                this.settings.watchdog_busy_enabled = false;
+            }
+        },
+        
+        async saveSettings() {
+            if (this.saving) return;
+            
+            this.saving = true;
+            
+            try {
+                const payload = {};
+                
+                // Only include changed values
+                if (this.settings.watchdog_enabled !== undefined) {
+                    payload.watchdog_enabled = this.settings.watchdog_enabled;
+                }
+                if (this.settings.watchdog_idle_enabled !== undefined) {
+                    payload.watchdog_idle_enabled = this.settings.watchdog_idle_enabled;
+                }
+                if (this.settings.watchdog_busy_enabled !== undefined) {
+                    payload.watchdog_busy_enabled = this.settings.watchdog_busy_enabled;
+                }
+                if (this.settings.watchdog_idle_timeout) {
+                    payload.watchdog_idle_timeout = this.settings.watchdog_idle_timeout;
+                }
+                if (this.settings.watchdog_busy_timeout) {
+                    payload.watchdog_busy_timeout = this.settings.watchdog_busy_timeout;
+                }
+                if (this.settings.single_backend !== undefined) {
+                    payload.single_backend = this.settings.single_backend;
+                }
+                if (this.settings.parallel_backend_requests !== undefined) {
+                    payload.parallel_backend_requests = this.settings.parallel_backend_requests;
+                }
+                if (this.settings.threads !== undefined) {
+                    payload.threads = parseInt(this.settings.threads) || 0;
+                }
+                if (this.settings.context_size !== undefined) {
+                    payload.context_size = parseInt(this.settings.context_size) || 0;
+                }
+                if (this.settings.f16 !== undefined) {
+                    payload.f16 = this.settings.f16;
+                }
+                if (this.settings.debug !== undefined) {
+                    payload.debug = this.settings.debug;
+                }
+                if (this.settings.cors !== undefined) {
+                    payload.cors = this.settings.cors;
+                }
+                if (this.settings.csrf !== undefined) {
+                    payload.csrf = this.settings.csrf;
+                }
+                if (this.settings.cors_allow_origins !== undefined) {
+                    payload.cors_allow_origins = this.settings.cors_allow_origins;
+                }
+                if (this.settings.p2p_token !== undefined) {
+                    payload.p2p_token = this.settings.p2p_token;
+                }
+                if (this.settings.p2p_network_id !== undefined) {
+                    payload.p2p_network_id = this.settings.p2p_network_id;
+                }
+                if (this.settings.federated !== undefined) {
+                    payload.federated = this.settings.federated;
+                }
+                if (this.settings.autoload_galleries !== undefined) {
+                    payload.autoload_galleries = this.settings.autoload_galleries;
+                }
+                if (this.settings.autoload_backend_galleries !== undefined) {
+                    payload.autoload_backend_galleries = this.settings.autoload_backend_galleries;
+                }
+                // Parse API keys from text (split by newline or comma, trim whitespace, filter empty)
+                if (this.settings.api_keys_text !== undefined) {
+                    const keys = this.settings.api_keys_text
+                        .split(/[\n,]/)
+                        .map(k => k.trim())
+                        .filter(k => k.length > 0);
+                    if (keys.length > 0) {
+                        payload.api_keys = keys;
+                    } else {
+                        // If empty, send empty array to clear keys
+                        payload.api_keys = [];
+                    }
+                }
+                // Parse galleries JSON
+                if (this.settings.galleries_json) {
+                    try {
+                        payload.galleries = JSON.parse(this.settings.galleries_json);
+                    } catch (e) {
+                        this.addNotification('Invalid galleries JSON: ' + e.message, 'error');
+                        this.saving = false;
+                        return;
+                    }
+                }
+                if (this.settings.backend_galleries_json) {
+                    try {
+                        payload.backend_galleries = JSON.parse(this.settings.backend_galleries_json);
+                    } catch (e) {
+                        this.addNotification('Invalid backend galleries JSON: ' + e.message, 'error');
+                        this.saving = false;
+                        return;
+                    }
+                }
+                
+                const response = await fetch('/api/settings', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify(payload)
+                });
+                
+                const data = await response.json();
+                
+                if (response.ok && data.success) {
+                    this.addNotification('Settings saved successfully!', 'success');
+                    // Reload settings to get updated source info
+                    setTimeout(() => this.loadSettings(), 1000);
+                } else {
+                    this.addNotification('Failed to save settings: ' + (data.error || 'Unknown error'), 'error');
+                }
+            } catch (error) {
+                console.error('Error saving settings:', error);
+                this.addNotification('Failed to save settings: ' + error.message, 'error');
+            } finally {
+                this.saving = false;
+            }
+        },
+        
+        addNotification(message, type = 'success') {
+            const id = Date.now();
+            this.notifications.push({ id, message, type });
+            setTimeout(() => this.dismissNotification(id), 5000);
+        },
+        
+        dismissNotification(id) {
+            this.notifications = this.notifications.filter(n => n.id !== id);
+        }
+    }
+}
+</script>
+
+</body>
+</html>
+

core/p2p/sync.go

@@ -1,102 +0,0 @@
-package p2p
-
-import (
-	"context"
-	"slices"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/mudler/LocalAI/core/application"
-	"github.com/mudler/LocalAI/core/gallery"
-	"github.com/mudler/LocalAI/core/services"
-
-	"github.com/mudler/edgevpn/pkg/node"
-	zlog "github.com/rs/zerolog/log"
-)
-
-func syncState(ctx context.Context, n *node.Node, app *application.Application) error {
-	zlog.Debug().Msg("[p2p-sync] Syncing state")
-
-	whatWeHave := []string{}
-	for _, model := range app.ModelConfigLoader().GetAllModelsConfigs() {
-		whatWeHave = append(whatWeHave, model.Name)
-	}
-
-	ledger, _ := n.Ledger()
-	currentData := ledger.CurrentData()
-	zlog.Debug().Msgf("[p2p-sync] Current data: %v", currentData)
-	data, exists := ledger.GetKey("shared_state", "models")
-	if !exists {
-		ledger.AnnounceUpdate(ctx, time.Minute, "shared_state", "models", whatWeHave)
-		zlog.Debug().Msgf("No models found in the ledger, announced our models: %v", whatWeHave)
-	}
-
-	models := []string{}
-	if err := data.Unmarshal(&models); err != nil {
-		zlog.Warn().Err(err).Msg("error unmarshalling models")
-		return nil
-	}
-
-	zlog.Debug().Msgf("[p2p-sync] Models that are present in this instance: %v\nModels that are in the ledger: %v", whatWeHave, models)
-
-	// Sync with our state
-	whatIsNotThere := []string{}
-	for _, model := range whatWeHave {
-		if !slices.Contains(models, model) {
-			whatIsNotThere = append(whatIsNotThere, model)
-		}
-	}
-	if len(whatIsNotThere) > 0 {
-		zlog.Debug().Msgf("[p2p-sync] Announcing our models: %v", append(models, whatIsNotThere...))
-		ledger.AnnounceUpdate(
-			ctx,
-			1*time.Minute,
-			"shared_state",
-			"models",
-			append(models, whatIsNotThere...),
-		)
-	}
-
-	// Check if we have a model that is not in our state, otherwise install it
-	for _, model := range models {
-		if slices.Contains(whatWeHave, model) {
-			zlog.Debug().Msgf("[p2p-sync] Model %s is already present in this instance", model)
-			continue
-		}
-
-		// we install model
-		zlog.Info().Msgf("[p2p-sync] Installing model which is not present in this instance: %s", model)
-
-		uuid, err := uuid.NewUUID()
-		if err != nil {
-			zlog.Error().Err(err).Msg("error generating UUID")
-			continue
-		}
-
-		app.GalleryService().ModelGalleryChannel <- services.GalleryOp[gallery.GalleryModel, gallery.ModelConfig]{
-			ID:                 uuid.String(),
-			GalleryElementName: model,
-			Galleries:          app.ApplicationConfig().Galleries,
-			BackendGalleries:   app.ApplicationConfig().BackendGalleries,
-		}
-	}
-
-	return nil
-}
-
-func Sync(ctx context.Context, n *node.Node, app *application.Application) error {
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-time.After(1 * time.Minute):
-				if err := syncState(ctx, n, app); err != nil {
-					zlog.Error().Err(err).Msg("error syncing state")
-				}
-			}
-
-		}
-	}()
-	return nil
-}

docs/content/advanced/vram-management.md

@@ -48,12 +48,15 @@ curl http://localhost:8080/v1/chat/completions -d '{"model": "model-b", ...}'
 
 For more flexible memory management, LocalAI provides watchdog mechanisms that automatically unload models based on their activity state. This allows multiple models to be loaded simultaneously, but automatically frees memory when models become inactive or stuck.
 
+> **Note:** Watchdog settings can be configured via the [Runtime Settings]({{%relref "features/runtime-settings#watchdog-settings" %}}) web interface, which allows you to adjust settings without restarting the application.
+
 ### Idle Watchdog
 
 The idle watchdog monitors models that haven't been used for a specified period and automatically unloads them to free VRAM.
 
 #### Configuration
 
+Via environment variables or CLI:
 ```bash
 LOCALAI_WATCHDOG_IDLE=true ./local-ai
 
@@ -62,12 +65,15 @@ LOCALAI_WATCHDOG_IDLE=true LOCALAI_WATCHDOG_IDLE_TIMEOUT=10m ./local-ai
 ./local-ai --enable-watchdog-idle --watchdog-idle-timeout=10m
 ```
 
+Via web UI: Navigate to Settings → Watchdog Settings and enable "Watchdog Idle Enabled" with your desired timeout.
+
 ### Busy Watchdog
 
 The busy watchdog monitors models that have been processing requests for an unusually long time and terminates them if they exceed a threshold. This is useful for detecting and recovering from stuck or hung backends.
 
 #### Configuration
 
+Via environment variables or CLI:
 ```bash
 LOCALAI_WATCHDOG_BUSY=true ./local-ai
 
@@ -76,6 +82,8 @@ LOCALAI_WATCHDOG_BUSY=true LOCALAI_WATCHDOG_BUSY_TIMEOUT=10m ./local-ai
 ./local-ai --enable-watchdog-busy --watchdog-busy-timeout=10m
 ```
 
+Via web UI: Navigate to Settings → Watchdog Settings and enable "Watchdog Busy Enabled" with your desired timeout.
+
 ### Combined Configuration
 
 You can enable both watchdogs simultaneously for comprehensive memory management:

docs/content/features/_index.en.md

@@ -32,6 +32,7 @@ LocalAI provides a comprehensive set of features for running AI models locally.
 - **[Stores](stores/)** - Vector similarity search for embeddings
 - **[Model Gallery](model-gallery/)** - Browse and install pre-configured models
 - **[Backends](backends/)** - Learn about available backends and how to manage them
+- **[Runtime Settings](runtime-settings/)** - Configure application settings via web UI without restarting
 
 ## Getting Started
 

docs/content/features/model-gallery.md

@@ -33,12 +33,18 @@ Navigate the WebUI interface in the "Models" section from the navbar at the top.
 
 ## Add other galleries
 
-You can add other galleries by setting the `GALLERIES` environment variable. The `GALLERIES` environment variable is a list of JSON objects, where each object has a `name` and a `url` field. The `name` field is the name of the gallery, and the `url` field is the URL of the gallery's index file, for example:
+You can add other galleries by:
+
+1. **Using the Web UI**: Navigate to the [Runtime Settings]({{%relref "features/runtime-settings#gallery-settings" %}}) page and configure galleries through the interface.
+
+2. **Using Environment Variables**: Set the `GALLERIES` environment variable. The `GALLERIES` environment variable is a list of JSON objects, where each object has a `name` and a `url` field. The `name` field is the name of the gallery, and the `url` field is the URL of the gallery's index file, for example:
 
 ```json
 GALLERIES=[{"name":"<GALLERY_NAME>", "url":"<GALLERY_URL"}]
 ```
 
+3. **Using Configuration Files**: Add galleries to `runtime_settings.json` in the `LOCALAI_CONFIG_DIR` directory.
+
 The models in the gallery will be automatically indexed and available for installation.
 
 ## API Reference

docs/content/features/runtime-settings.md

@@ -0,0 +1,180 @@
++++
+disableToc = false
+title = "⚙️ Runtime Settings"
+weight = 25
+url = '/features/runtime-settings'
++++
+
+LocalAI provides a web-based interface for managing application settings at runtime. These settings can be configured through the web UI and are automatically persisted to a configuration file, allowing changes to take effect immediately without requiring a restart.
+
+## Accessing Runtime Settings
+
+Navigate to the **Settings** page from the management interface at `http://localhost:8080/manage`. The settings page provides a comprehensive interface for configuring various aspects of LocalAI.
+
+## Available Settings
+
+### Watchdog Settings
+
+The watchdog monitors backend activity and can automatically stop idle or overly busy models to free up resources.
+
+- **Watchdog Enabled**: Master switch to enable/disable the watchdog
+- **Watchdog Idle Enabled**: Enable stopping backends that are idle longer than the idle timeout
+- **Watchdog Busy Enabled**: Enable stopping backends that are busy longer than the busy timeout
+- **Watchdog Idle Timeout**: Duration threshold for idle backends (default: `15m`)
+- **Watchdog Busy Timeout**: Duration threshold for busy backends (default: `5m`)
+
+Changes to watchdog settings are applied immediately by restarting the watchdog service.
+
+### Backend Configuration
+
+- **Single Backend**: Allow only one backend to run at a time
+- **Parallel Backend Requests**: Enable backends to handle multiple requests in parallel if supported
+
+### Performance Settings
+
+- **Threads**: Number of threads used for parallel computation (recommended: number of physical cores)
+- **Context Size**: Default context size for models (default: `512`)
+- **F16**: Enable GPU acceleration using 16-bit floating point
+
+### Debug and Logging
+
+- **Debug Mode**: Enable debug logging (deprecated, use log-level instead)
+
+### API Security
+
+- **CORS**: Enable Cross-Origin Resource Sharing
+- **CORS Allow Origins**: Comma-separated list of allowed CORS origins
+- **CSRF**: Enable CSRF protection middleware
+- **API Keys**: Manage API keys for authentication (one per line or comma-separated)
+
+### P2P Settings
+
+Configure peer-to-peer networking for distributed inference:
+
+- **P2P Token**: Authentication token for P2P network
+- **P2P Network ID**: Network identifier for P2P connections
+- **Federated Mode**: Enable federated mode for P2P network
+
+Changes to P2P settings automatically restart the P2P stack with the new configuration.
+
+### Gallery Settings
+
+Manage model and backend galleries:
+
+- **Model Galleries**: JSON array of gallery objects with `url` and `name` fields
+- **Backend Galleries**: JSON array of backend gallery objects
+- **Autoload Galleries**: Automatically load model galleries on startup
+- **Autoload Backend Galleries**: Automatically load backend galleries on startup
+
+## Configuration Persistence
+
+All settings are automatically saved to `runtime_settings.json` in the `LOCALAI_CONFIG_DIR` directory (default: `BASEPATH/configuration`). This file is watched for changes, so modifications made directly to the file will also be applied at runtime.
+
+## Environment Variable Precedence
+
+Environment variables take precedence over settings configured via the web UI or configuration files. If a setting is controlled by an environment variable, it cannot be modified through the web interface. The settings page will indicate when a setting is controlled by an environment variable.
+
+The precedence order is:
+1. **Environment variables** (highest priority)
+2. **Configuration files** (`runtime_settings.json`, `api_keys.json`)
+3. **Default values** (lowest priority)
+
+## Example Configuration
+
+The `runtime_settings.json` file follows this structure:
+
+```json
+{
+  "watchdog_enabled": true,
+  "watchdog_idle_enabled": true,
+  "watchdog_busy_enabled": false,
+  "watchdog_idle_timeout": "15m",
+  "watchdog_busy_timeout": "5m",
+  "single_backend": false,
+  "parallel_backend_requests": true,
+  "threads": 8,
+  "context_size": 2048,
+  "f16": false,
+  "debug": false,
+  "cors": true,
+  "csrf": false,
+  "cors_allow_origins": "*",
+  "p2p_token": "",
+  "p2p_network_id": "",
+  "federated": false,
+  "galleries": [
+    {
+      "url": "github:mudler/LocalAI/gallery/index.yaml@master",
+      "name": "localai"
+    }
+  ],
+  "backend_galleries": [
+    {
+      "url": "github:mudler/LocalAI/backend/index.yaml@master",
+      "name": "localai"
+    }
+  ],
+  "autoload_galleries": true,
+  "autoload_backend_galleries": true,
+  "api_keys": []
+}
+```
+
+## API Keys Management
+
+API keys can be managed through the runtime settings interface. Keys can be entered one per line or comma-separated. 
+
+**Important Notes:**
+- API keys from environment variables are always included and cannot be removed via the UI
+- Runtime API keys are stored in `runtime_settings.json`
+- For backward compatibility, API keys can also be managed via `api_keys.json`
+- Empty arrays will clear all runtime API keys (but preserve environment variable keys)
+
+## Dynamic Configuration
+
+The runtime settings system supports dynamic configuration file watching. When `LOCALAI_CONFIG_DIR` is set, LocalAI monitors the following files for changes:
+
+- `runtime_settings.json` - Unified runtime settings
+- `api_keys.json` - API keys (for backward compatibility)
+- `external_backends.json` - External backend configurations
+
+Changes to these files are automatically detected and applied without requiring a restart.
+
+## Best Practices
+
+1. **Use Environment Variables for Production**: For production deployments, use environment variables for critical settings to ensure they cannot be accidentally changed via the web UI.
+
+2. **Backup Configuration Files**: Before making significant changes, consider backing up your `runtime_settings.json` file.
+
+3. **Monitor Resource Usage**: When enabling watchdog features, monitor your system to ensure the timeout values are appropriate for your workload.
+
+4. **Secure API Keys**: API keys are sensitive information. Ensure proper file permissions on configuration files (they should be readable only by the LocalAI process).
+
+5. **Test Changes**: Some settings (like watchdog timeouts) may require testing to find optimal values for your specific use case.
+
+## Troubleshooting
+
+### Settings Not Applying
+
+If settings are not being applied:
+1. Check if the setting is controlled by an environment variable
+2. Verify the `LOCALAI_CONFIG_DIR` is set correctly
+3. Check file permissions on `runtime_settings.json`
+4. Review application logs for configuration errors
+
+### Watchdog Not Working
+
+If the watchdog is not functioning:
+1. Ensure "Watchdog Enabled" is turned on
+2. Verify at least one of the idle or busy watchdogs is enabled
+3. Check that timeout values are reasonable for your workload
+4. Review logs for watchdog-related messages
+
+### P2P Not Starting
+
+If P2P is not starting:
+1. Verify the P2P token is set (non-empty)
+2. Check network connectivity
+3. Ensure the P2P network ID matches across nodes (if using federated mode)
+4. Review logs for P2P-related errors
+

docs/content/reference/cli-reference.md

@@ -24,7 +24,7 @@ Complete reference for all LocalAI command-line interface (CLI) parameters and e
 | `--models-path` | `BASEPATH/models` | Path containing models used for inferencing | `$LOCALAI_MODELS_PATH`, `$MODELS_PATH` |
 | `--generated-content-path` | `/tmp/generated/content` | Location for assets generated by backends (e.g. stablediffusion, images, audio, videos) | `$LOCALAI_GENERATED_CONTENT_PATH`, `$GENERATED_CONTENT_PATH` |
 | `--upload-path` | `/tmp/localai/upload` | Path to store uploads from files API | `$LOCALAI_UPLOAD_PATH`, `$UPLOAD_PATH` |
-| `--localai-config-dir` | `BASEPATH/configuration` | Directory for dynamic loading of certain configuration files (currently api_keys.json and external_backends.json) | `$LOCALAI_CONFIG_DIR` |
+| `--localai-config-dir` | `BASEPATH/configuration` | Directory for dynamic loading of certain configuration files (currently runtime_settings.json, api_keys.json, and external_backends.json). See [Runtime Settings]({{%relref "features/runtime-settings" %}}) for web-based configuration. | `$LOCALAI_CONFIG_DIR` |
 | `--localai-config-dir-poll-interval` | | Time duration to poll the LocalAI Config Dir if your system has broken fsnotify events (example: `1m`) | `$LOCALAI_CONFIG_DIR_POLL_INTERVAL` |
 | `--models-config-file` | | YAML file containing a list of model backend configs (alias: `--config-file`) | `$LOCALAI_MODELS_CONFIG_FILE`, `$CONFIG_FILE` |
 
@@ -80,6 +80,7 @@ For more information on VRAM management, see [VRAM and Memory Management]({{%rel
 | `--upload-limit` | `15` | Default upload-limit in MB | `$LOCALAI_UPLOAD_LIMIT`, `$UPLOAD_LIMIT` |
 | `--api-keys` | | List of API Keys to enable API authentication. When this is set, all requests must be authenticated with one of these API keys | `$LOCALAI_API_KEY`, `$API_KEY` |
 | `--disable-webui` | `false` | Disables the web user interface. When set to true, the server will only expose API endpoints without serving the web interface | `$LOCALAI_DISABLE_WEBUI`, `$DISABLE_WEBUI` |
+| `--disable-runtime-settings` | `false` | Disables the runtime settings feature. When set to true, the server will not load runtime settings from the `runtime_settings.json` file and the settings web interface will be disabled | `$LOCALAI_DISABLE_RUNTIME_SETTINGS`, `$DISABLE_RUNTIME_SETTINGS` |
 | `--disable-gallery-endpoint` | `false` | Disable the gallery endpoints | `$LOCALAI_DISABLE_GALLERY_ENDPOINT`, `$DISABLE_GALLERY_ENDPOINT` |
 | `--disable-metrics-endpoint` | `false` | Disable the `/metrics` endpoint | `$LOCALAI_DISABLE_METRICS_ENDPOINT`, `$DISABLE_METRICS_ENDPOINT` |
 | `--machine-tag` | | If not empty, add that string to Machine-Tag header in each response. Useful to track response from different machines using multiple P2P federated nodes | `$LOCALAI_MACHINE_TAG`, `$MACHINE_TAG` |

pkg/model/loader.go

@@ -44,6 +44,10 @@ func (ml *ModelLoader) SetWatchDog(wd *WatchDog) {
 	ml.wd = wd
 }
 
+func (ml *ModelLoader) GetWatchDog() *WatchDog {
+	return ml.wd
+}
+
 func (ml *ModelLoader) ExistsInModelPath(s string) bool {
 	return utils.ExistsInPath(ml.ModelPath, s)
 }

pkg/model/watchdog.go

@@ -51,6 +51,7 @@ func NewWatchDog(pm ProcessManager, timeoutBusy, timeoutIdle time.Duration, busy
 func (wd *WatchDog) Shutdown() {
 	wd.Lock()
 	defer wd.Unlock()
+	log.Info().Msg("[WatchDog] Shutting down watchdog")
 	wd.stop <- true
 }