Scraper and plugin manager (#4242)

* Add package manager
* Add SettingModal validate
* Reverse modal button order
* Add plugin package management
* Refactor ClearableInput

internal/api/resolver_mutation_configure.go

@@ -347,6 +347,18 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen
 		c.Set(config.DrawFunscriptHeatmapRange, input.DrawFunscriptHeatmapRange)
 	}
 
+	refreshScraperSource := false
+	if input.ScraperPackageSources != nil {
+		c.Set(config.ScraperPackageSources, input.ScraperPackageSources)
+		refreshScraperSource = true
+	}
+
+	refreshPluginSource := false
+	if input.PluginPackageSources != nil {
+		c.Set(config.PluginPackageSources, input.PluginPackageSources)
+		refreshPluginSource = true
+	}
+
 	if err := c.Write(); err != nil {
 		return makeConfigGeneralResult(), err
 	}
@@ -361,6 +373,12 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen
 	if refreshBlobStorage {
 		manager.GetInstance().SetBlobStoreOptions()
 	}
+	if refreshScraperSource {
+		manager.GetInstance().RefreshScraperSourceManager()
+	}
+	if refreshPluginSource {
+		manager.GetInstance().RefreshPluginSourceManager()
+	}
 
 	return makeConfigGeneralResult(), nil
 }

internal/api/resolver_mutation_package.go

@@ -0,0 +1,82 @@
+package api
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/stashapp/stash/internal/manager"
+	"github.com/stashapp/stash/internal/manager/task"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+)
+
+func refreshPackageType(typeArg PackageType) {
+	mgr := manager.GetInstance()
+
+	if typeArg == PackageTypePlugin {
+		if err := mgr.PluginCache.LoadPlugins(); err != nil {
+			logger.Errorf("Error reading plugin configs: %v", err)
+		}
+	} else if typeArg == PackageTypeScraper {
+		if err := mgr.ScraperCache.ReloadScrapers(); err != nil {
+			logger.Errorf("Error reading scraper configs: %v", err)
+		}
+	}
+}
+
+func (r *mutationResolver) InstallPackages(ctx context.Context, typeArg PackageType, packages []*models.PackageSpecInput) (string, error) {
+	pm, err := getPackageManager(typeArg)
+	if err != nil {
+		return "", err
+	}
+
+	mgr := manager.GetInstance()
+	t := &task.InstallPackagesJob{
+		PackagesJob: task.PackagesJob{
+			PackageManager: pm,
+			OnComplete:     func() { refreshPackageType(typeArg) },
+		},
+		Packages: packages,
+	}
+	jobID := mgr.JobManager.Add(ctx, "Installing packages...", t)
+
+	return strconv.Itoa(jobID), nil
+}
+
+func (r *mutationResolver) UpdatePackages(ctx context.Context, typeArg PackageType, packages []*models.PackageSpecInput) (string, error) {
+	pm, err := getPackageManager(typeArg)
+	if err != nil {
+		return "", err
+	}
+
+	mgr := manager.GetInstance()
+	t := &task.UpdatePackagesJob{
+		PackagesJob: task.PackagesJob{
+			PackageManager: pm,
+			OnComplete:     func() { refreshPackageType(typeArg) },
+		},
+		Packages: packages,
+	}
+	jobID := mgr.JobManager.Add(ctx, "Updating packages...", t)
+
+	return strconv.Itoa(jobID), nil
+}
+
+func (r *mutationResolver) UninstallPackages(ctx context.Context, typeArg PackageType, packages []*models.PackageSpecInput) (string, error) {
+	pm, err := getPackageManager(typeArg)
+	if err != nil {
+		return "", err
+	}
+
+	mgr := manager.GetInstance()
+	t := &task.UninstallPackagesJob{
+		PackagesJob: task.PackagesJob{
+			PackageManager: pm,
+			OnComplete:     func() { refreshPackageType(typeArg) },
+		},
+		Packages: packages,
+	}
+	jobID := mgr.JobManager.Add(ctx, "Updating packages...", t)
+
+	return strconv.Itoa(jobID), nil
+}

internal/api/resolver_query_configuration.go

@@ -127,6 +127,8 @@ func makeConfigGeneralResult() *ConfigGeneralResult {
 		LiveTranscodeInputArgs:        config.GetLiveTranscodeInputArgs(),
 		LiveTranscodeOutputArgs:       config.GetLiveTranscodeOutputArgs(),
 		DrawFunscriptHeatmapRange:     config.GetDrawFunscriptHeatmapRange(),
+		ScraperPackageSources:         config.GetScraperPackageSources(),
+		PluginPackageSources:          config.GetPluginPackageSources(),
 	}
 }
 

internal/api/resolver_query_package.go

@@ -0,0 +1,194 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"strings"
+
+	"github.com/99designs/gqlgen/graphql"
+	"github.com/stashapp/stash/internal/manager"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/pkg"
+	"github.com/stashapp/stash/pkg/sliceutil"
+)
+
+var ErrInvalidPackageType = errors.New("invalid package type")
+
+func getPackageManager(typeArg PackageType) (*pkg.Manager, error) {
+	var pm *pkg.Manager
+	switch typeArg {
+	case PackageTypeScraper:
+		pm = manager.GetInstance().ScraperPackageManager
+	case PackageTypePlugin:
+		pm = manager.GetInstance().PluginPackageManager
+	default:
+		return nil, ErrInvalidPackageType
+	}
+
+	return pm, nil
+}
+
+func manifestToPackage(p pkg.Manifest) *Package {
+	ret := &Package{
+		PackageID: p.ID,
+		Name:      p.Name,
+		SourceURL: p.RepositoryURL,
+	}
+
+	if len(p.Version) > 0 {
+		ret.Version = &p.Version
+	}
+	if !p.Date.IsZero() {
+		ret.Date = &p.Date.Time
+	}
+
+	ret.Metadata = p.Metadata
+	if ret.Metadata == nil {
+		ret.Metadata = make(map[string]interface{})
+	}
+
+	return ret
+}
+
+func remotePackageToPackage(p pkg.RemotePackage, index pkg.RemotePackageIndex) *Package {
+	ret := &Package{
+		PackageID: p.ID,
+		Name:      p.Name,
+	}
+
+	if len(p.Version) > 0 {
+		ret.Version = &p.Version
+	}
+	if !p.Date.IsZero() {
+		ret.Date = &p.Date.Time
+	}
+
+	ret.Metadata = p.Metadata
+	if ret.Metadata == nil {
+		ret.Metadata = make(map[string]interface{})
+	}
+
+	ret.SourceURL = p.Repository.Path()
+
+	for _, r := range p.Requires {
+		// required packages must come from the same source
+		spec := models.PackageSpecInput{
+			ID:        r,
+			SourceURL: p.Repository.Path(),
+		}
+
+		req, found := index[spec]
+		if !found {
+			// shouldn't happen, but we'll ignore it
+			continue
+		}
+
+		ret.Requires = append(ret.Requires, remotePackageToPackage(req, index))
+	}
+
+	return ret
+}
+
+func sortedPackageSpecKeys[V any](m map[models.PackageSpecInput]V) []models.PackageSpecInput {
+	// sort keys
+	var keys []models.PackageSpecInput
+	for k := range m {
+		keys = append(keys, k)
+	}
+
+	sort.Slice(keys, func(i, j int) bool {
+		if strings.EqualFold(keys[i].ID, keys[j].ID) {
+			return keys[i].ID < keys[j].ID
+		}
+
+		return strings.ToLower(keys[i].ID) < strings.ToLower(keys[j].ID)
+	})
+
+	return keys
+}
+
+func (r *queryResolver) getInstalledPackagesWithUpgrades(ctx context.Context, pm *pkg.Manager) ([]*Package, error) {
+	// get all installed packages
+	installed, err := pm.ListInstalled(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// get remotes for all installed packages
+	allRemoteList, err := pm.ListInstalledRemotes(ctx, installed)
+	if err != nil {
+		return nil, err
+	}
+
+	packageStatusIndex := pkg.MakePackageStatusIndex(installed, allRemoteList)
+
+	ret := make([]*Package, len(packageStatusIndex))
+	i := 0
+
+	for _, k := range sortedPackageSpecKeys(packageStatusIndex) {
+		v := packageStatusIndex[k]
+		p := manifestToPackage(*v.Local)
+		if v.Upgradable() {
+			pp := remotePackageToPackage(*v.Remote, allRemoteList)
+			p.Upgrade = pp
+		}
+		ret[i] = p
+		i++
+	}
+
+	return ret, nil
+}
+
+func (r *queryResolver) InstalledPackages(ctx context.Context, typeArg PackageType) ([]*Package, error) {
+	pm, err := getPackageManager(typeArg)
+	if err != nil {
+		return nil, err
+	}
+
+	installed, err := pm.ListInstalled(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var ret []*Package
+
+	if sliceutil.Contains(graphql.CollectAllFields(ctx), "upgrade") {
+		ret, err = r.getInstalledPackagesWithUpgrades(ctx, pm)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		ret = make([]*Package, len(installed))
+		i := 0
+		for _, k := range sortedPackageSpecKeys(installed) {
+			ret[i] = manifestToPackage(installed[k])
+			i++
+		}
+	}
+
+	return ret, nil
+}
+
+func (r *queryResolver) AvailablePackages(ctx context.Context, typeArg PackageType, source string) ([]*Package, error) {
+	pm, err := getPackageManager(typeArg)
+	if err != nil {
+		return nil, err
+	}
+
+	available, err := pm.ListRemote(ctx, source)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := make([]*Package, len(available))
+	i := 0
+	for _, k := range sortedPackageSpecKeys(available) {
+		p := available[k]
+		ret[i] = remotePackageToPackage(p, available)
+
+		i++
+	}
+
+	return ret, nil
+}

internal/manager/config/config.go

@@ -21,6 +21,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/paths"
+	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/utils"
 )
 
@@ -137,6 +138,9 @@ const (
 	PluginsSettingPrefix = PluginsSetting + "."
 	DisabledPlugins      = "plugins.disabled"
 
+	PluginPackageSources  = "plugins.package_sources"
+	ScraperPackageSources = "scrapers.package_sources"
+
 	// i18n
 	Language = "language"
 
@@ -1520,6 +1524,61 @@ func (i *Instance) ActivatePublicAccessTripwire(requestIP string) error {
 	return i.Write()
 }
 
+func (i *Instance) getPackageSources(key string) []*models.PackageSource {
+	var sources []*models.PackageSource
+	if err := i.unmarshalKey(key, &sources); err != nil {
+		logger.Warnf("error in unmarshalkey: %v", err)
+	}
+
+	return sources
+}
+
+func (i *Instance) GetPluginPackageSources() []*models.PackageSource {
+	return i.getPackageSources(PluginPackageSources)
+}
+
+func (i *Instance) GetScraperPackageSources() []*models.PackageSource {
+	return i.getPackageSources(ScraperPackageSources)
+}
+
+type packagePathGetter struct {
+	getterFn func() []*models.PackageSource
+}
+
+func (g packagePathGetter) GetAllSourcePaths() []string {
+	p := g.getterFn()
+	var ret []string
+	for _, v := range p {
+		ret = sliceutil.AppendUnique(ret, v.LocalPath)
+	}
+
+	return ret
+}
+
+func (g packagePathGetter) GetSourcePath(srcURL string) string {
+	p := g.getterFn()
+
+	for _, v := range p {
+		if v.URL == srcURL {
+			return v.LocalPath
+		}
+	}
+
+	return ""
+}
+
+func (i *Instance) GetPluginPackagePathGetter() packagePathGetter {
+	return packagePathGetter{
+		getterFn: i.GetPluginPackageSources,
+	}
+}
+
+func (i *Instance) GetScraperPackagePathGetter() packagePathGetter {
+	return packagePathGetter{
+		getterFn: i.GetScraperPackageSources,
+	}
+}
+
 func (i *Instance) Validate() error {
 	i.RLock()
 	defer i.RUnlock()

internal/manager/manager.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -29,6 +30,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/models/paths"
+	"github.com/stashapp/stash/pkg/pkg"
 	"github.com/stashapp/stash/pkg/plugin"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/scraper"
@@ -130,6 +132,9 @@ type Manager struct {
 	PluginCache  *plugin.Cache
 	ScraperCache *scraper.Cache
 
+	PluginPackageManager  *pkg.Manager
+	ScraperPackageManager *pkg.Manager
+
 	DownloadStore *DownloadStore
 
 	DLNAService *dlna.Service
@@ -229,6 +234,9 @@ func initialize() error {
 	dlnaRepository := dlna.NewRepository(repo)
 	instance.DLNAService = dlna.NewService(dlnaRepository, cfg, &sceneServer)
 
+	instance.RefreshPluginSourceManager()
+	instance.RefreshScraperSourceManager()
+
 	if !cfg.IsNewSystem() {
 		logger.Infof("using config file: %s", cfg.GetConfigFile())
 
@@ -280,6 +288,26 @@ func initialize() error {
 	return nil
 }
 
+func initialisePackageManager(localPath string, srcPathGetter pkg.SourcePathGetter, cachePathGetter pkg.CachePathGetter) *pkg.Manager {
+	const timeout = 10 * time.Second
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+		},
+		Timeout: timeout,
+	}
+
+	return &pkg.Manager{
+		Local: &pkg.Store{
+			BaseDir:      localPath,
+			ManifestFile: pkg.ManifestFile,
+		},
+		PackagePathGetter: srcPathGetter,
+		Client:            httpClient,
+		CachePathGetter:   cachePathGetter,
+	}
+}
+
 func videoFileFilter(ctx context.Context, f models.File) bool {
 	return useAsVideo(f.Base().Path)
 }
@@ -566,6 +594,14 @@ func (s *Manager) RefreshStreamManager() {
 	s.StreamManager = ffmpeg.NewStreamManager(cacheDir, s.FFMPEG, s.FFProbe, s.Config, s.ReadLockManager)
 }
 
+func (s *Manager) RefreshScraperSourceManager() {
+	s.ScraperPackageManager = initialisePackageManager(s.Config.GetScrapersPath(), s.Config.GetScraperPackagePathGetter(), s.Config)
+}
+
+func (s *Manager) RefreshPluginSourceManager() {
+	s.PluginPackageManager = initialisePackageManager(s.Config.GetPluginsPath(), s.Config.GetPluginPackagePathGetter(), s.Config)
+}
+
 func setSetupDefaults(input *SetupInput) {
 	if input.ConfigLocation == "" {
 		input.ConfigLocation = filepath.Join(fsutil.GetHomeDirectory(), ".stash", "config.yml")

internal/manager/task/packages.go

@@ -0,0 +1,134 @@
+package task
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/stashapp/stash/pkg/job"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/pkg"
+)
+
+type PackagesJob struct {
+	PackageManager *pkg.Manager
+	OnComplete     func()
+}
+
+func (j *PackagesJob) installPackage(ctx context.Context, p models.PackageSpecInput, progress *job.Progress) error {
+	defer progress.Increment()
+
+	if err := j.PackageManager.Install(ctx, p); err != nil {
+		return fmt.Errorf("installing package: %w", err)
+	}
+
+	return nil
+}
+
+type InstallPackagesJob struct {
+	PackagesJob
+	Packages []*models.PackageSpecInput
+}
+
+func (j *InstallPackagesJob) Execute(ctx context.Context, progress *job.Progress) {
+	progress.SetTotal(len(j.Packages))
+
+	for _, p := range j.Packages {
+		if job.IsCancelled(ctx) {
+			logger.Info("Cancelled installing packages")
+			return
+		}
+
+		logger.Infof("Installing package %s", p.ID)
+		taskDesc := fmt.Sprintf("Installing %s", p.ID)
+		progress.ExecuteTask(taskDesc, func() {
+			if err := j.installPackage(ctx, *p, progress); err != nil {
+				logger.Errorf("Error installing package %s from %s: %v", p.ID, p.SourceURL, err)
+			}
+		})
+	}
+
+	if j.OnComplete != nil {
+		j.OnComplete()
+	}
+
+	logger.Infof("Finished installing packages")
+}
+
+type UpdatePackagesJob struct {
+	PackagesJob
+	Packages []*models.PackageSpecInput
+}
+
+func (j *UpdatePackagesJob) Execute(ctx context.Context, progress *job.Progress) {
+	// if no packages are specified, update all
+	if len(j.Packages) == 0 {
+		installed, err := j.PackageManager.InstalledStatus(ctx)
+		if err != nil {
+			logger.Errorf("Error getting installed packages: %v", err)
+			return
+		}
+
+		for _, p := range installed {
+			if p.Upgradable() {
+				j.Packages = append(j.Packages, &models.PackageSpecInput{
+					ID:        p.Local.ID,
+					SourceURL: p.Remote.Repository.Path(),
+				})
+			}
+		}
+	}
+
+	progress.SetTotal(len(j.Packages))
+
+	for _, p := range j.Packages {
+		if job.IsCancelled(ctx) {
+			logger.Info("Cancelled updating packages")
+			return
+		}
+
+		logger.Infof("Updating package %s", p.ID)
+		taskDesc := fmt.Sprintf("Updating %s", p.ID)
+		progress.ExecuteTask(taskDesc, func() {
+			if err := j.installPackage(ctx, *p, progress); err != nil {
+				logger.Errorf("Error updating package %s from %s: %v", p.ID, p.SourceURL, err)
+			}
+		})
+	}
+
+	if j.OnComplete != nil {
+		j.OnComplete()
+	}
+
+	logger.Infof("Finished updating packages")
+}
+
+type UninstallPackagesJob struct {
+	PackagesJob
+	Packages []*models.PackageSpecInput
+}
+
+func (j *UninstallPackagesJob) Execute(ctx context.Context, progress *job.Progress) {
+	progress.SetTotal(len(j.Packages))
+
+	for _, p := range j.Packages {
+		if job.IsCancelled(ctx) {
+			logger.Info("Cancelled installing packages")
+			return
+		}
+
+		logger.Infof("Uninstalling package %s", p.ID)
+		taskDesc := fmt.Sprintf("Uninstalling %s", p.ID)
+		progress.ExecuteTask(taskDesc, func() {
+			if err := j.PackageManager.Uninstall(ctx, *p); err != nil {
+				logger.Errorf("Error uninstalling package %s: %v", p.ID, err)
+			}
+		})
+	}
+
+	if j.OnComplete != nil {
+		j.OnComplete()
+	}
+
+	logger.Infof("Finished uninstalling packages")
+}