From a1c04c5061a6fb890fcc25837acff3cfc3dcb9ca Mon Sep 17 00:00:00 2001 From: Abhishek Shroff Date: Sun, 8 Jun 2025 00:33:36 +0530 Subject: [PATCH] [server][storage] single shared temp directory --- server/internal/command/config.defaults.yml | 3 ++- server/internal/storage/config.go | 3 ++- server/internal/storage/drivers.go | 2 +- server/internal/storage/local_storage.go | 11 +++------ server/internal/storage/minio_storage.go | 22 ++++++----------- server/internal/storage/storage.go | 27 ++++++++++++++++----- 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/server/internal/command/config.defaults.yml b/server/internal/command/config.defaults.yml index 79a9096b..60ab52aa 100644 --- a/server/internal/command/config.defaults.yml +++ b/server/internal/command/config.defaults.yml @@ -8,7 +8,8 @@ db: trace: false # Trace queries (dev) storage: - location: storage + root: storage + tmp: tmp # relative to root. Must start with '/' for absolute path user: password: diff --git a/server/internal/storage/config.go b/server/internal/storage/config.go index 42b99297..9108195b 100644 --- a/server/internal/storage/config.go +++ b/server/internal/storage/config.go @@ -1,5 +1,6 @@ package storage type Config struct { - Location string `koanf:"location"` + Root string `koanf:"root"` + Temp string `koanf:"tmp"` } diff --git a/server/internal/storage/drivers.go b/server/internal/storage/drivers.go index 861766e0..ba4a3c2a 100644 --- a/server/internal/storage/drivers.go +++ b/server/internal/storage/drivers.go @@ -10,7 +10,7 @@ var drivers = map[string]Driver{ }, }, "minio": { - Params: []string{"endpoint", "key_id", "access_key", "tmp", "bucket_name", "prefix"}, + Params: []string{"endpoint", "key_id", "access_key", "bucket_name", "prefix"}, Create: createMinioBackend, }, } diff --git a/server/internal/storage/local_storage.go b/server/internal/storage/local_storage.go index e581ede5..8a874679 100644 --- a/server/internal/storage/local_storage.go +++ b/server/internal/storage/local_storage.go @@ -11,7 +11,6 @@ import ( type localStorage struct { name string root string - tmp string } func createLocalBackend(name string, params map[string]string) (LocalBackend, error) { @@ -22,14 +21,10 @@ func createLocalBackend(name string, params map[string]string) (LocalBackend, er l := localStorage{ name: name, root: params["root"], - tmp: filepath.Join(params["root"], "tmp"), } - if err := os.RemoveAll(l.tmp); err != nil { - return nil, errors.New("failed to remove tmp directory: " + err.Error()) - } - if err := os.MkdirAll(l.tmp, 0750); err != nil { - return nil, errors.New("failed to create tmp directory: " + err.Error()) + if err := os.MkdirAll(l.root, 0700); err != nil { + return nil, errors.New("failed to create root directory: " + err.Error()) } return l, nil @@ -51,7 +46,7 @@ func (l localStorage) OpenRead(name string, start, length int) (io.ReadCloser, e } func (l localStorage) OpenWrite(name string) (io.WriteCloser, error) { - f, err := os.CreateTemp(l.tmp, "upload-*") + f, err := os.CreateTemp(tempDir, "phylum-*") if err != nil { return nil, err } diff --git a/server/internal/storage/minio_storage.go b/server/internal/storage/minio_storage.go index ddc5a1e6..75ba3c73 100644 --- a/server/internal/storage/minio_storage.go +++ b/server/internal/storage/minio_storage.go @@ -16,30 +16,22 @@ var ErrRangeNotSupported = errors.New("byte range not suppoered") type minioStorage struct { name string client *minio.Client - tmp string bucketName string prefix string } func createMinioBackend(name string, params map[string]string) (Backend, error) { if params["endpoint"] == "" { - return nil, errors.New("minio endpoint not provided") + return nil, errors.New("endpoint not provided") } if params["key_id"] == "" { - return nil, errors.New("minio key id not provided") + return nil, errors.New("key id not provided") } if params["access_key"] == "" { - return nil, errors.New("minio access key not provided") - } - if params["tmp"] == "" { - return nil, errors.New("minio tmp not provided") + return nil, errors.New("access key not provided") } if params["bucket_name"] == "" { - return nil, errors.New("minio bucket name not provided") - } - err := os.MkdirAll(params["tmp"], 0750) - if err != nil { - return nil, err + return nil, errors.New("bucket name not provided") } client, err := minio.New(params["endpoint"], &minio.Options{ @@ -49,7 +41,7 @@ func createMinioBackend(name string, params map[string]string) (Backend, error) if err != nil { return nil, err } - return minioStorage{name: name, client: client, tmp: params["tmp"], bucketName: params["bucket_name"], prefix: params["prefix"]}, nil + return minioStorage{name: name, client: client, bucketName: params["bucket_name"], prefix: params["prefix"]}, nil } func (s minioStorage) Name() string { @@ -73,7 +65,7 @@ func (s minioStorage) Copy(name string, src Backend) error { if l, ok := src.(LocalBackend); ok { localPath = l.path(name) } else { - f, err := os.CreateTemp(s.tmp, "phylum-*") + f, err := os.CreateTemp(tempDir, "phylum-*") if err != nil { return err } @@ -130,7 +122,7 @@ func (s minioStorage) DeleteAll(names []string) []error { } func (s minioStorage) String() string { - return fmt.Sprintf("minio (endpoint: %s, bucket: %s, prefix: %s, tmp: %s)", s.client.EndpointURL(), s.bucketName, s.prefix, s.tmp) + return fmt.Sprintf("minio (endpoint: %s, bucket: %s, prefix: %s)", s.client.EndpointURL(), s.bucketName, s.prefix) } // type progressReader struct { diff --git a/server/internal/storage/storage.go b/server/internal/storage/storage.go index c3f95004..d3837405 100644 --- a/server/internal/storage/storage.go +++ b/server/internal/storage/storage.go @@ -6,6 +6,8 @@ import ( "fmt" "os" "path" + "path/filepath" + "strings" "github.com/jackc/pgx/v5" "github.com/shroff/phylum/server/internal/db" @@ -18,6 +20,7 @@ const DefaultBackendName = "_" var defaultBackend LocalBackend var backends map[string]Backend +var tempDir string type BackendConfig struct { Name string `json:"name"` @@ -26,19 +29,31 @@ type BackendConfig struct { } func Initialize(db db.Handler) error { - if err := os.MkdirAll(Cfg.Location, 0700); err != nil { - return errors.New("failed to create storage location(" + Cfg.Location + "): " + err.Error()) + if err := os.MkdirAll(Cfg.Root, 0700); err != nil { + return errors.New("failed to create storage root(" + Cfg.Root + "): " + err.Error()) } else { - if stat, err := os.Stat(Cfg.Location); err != nil { - return errors.New("failed to stat storage location(" + Cfg.Location + "): " + err.Error()) + if stat, err := os.Stat(Cfg.Root); err != nil { + return errors.New("failed to stat storage root(" + Cfg.Root + "): " + err.Error()) } else if stat.Mode()&0xfff != 0700 { - return errors.New("storage location does not have correct permissions (0700): " + fmt.Sprintf("%o", stat.Mode()&0xfff)) + return errors.New("storage root does not have correct permissions (0700): " + fmt.Sprintf("%o", stat.Mode()&0xfff)) } } + + tempDir = filepath.Join(Cfg.Root, Cfg.Temp) + if strings.HasPrefix(Cfg.Temp, string(filepath.Separator)) { + tempDir = Cfg.Temp + } + if err := os.RemoveAll(tempDir); err != nil { + return errors.New("failed to clear temp directory: " + err.Error()) + } + if err := os.MkdirAll(tempDir, 0700); err != nil { + return errors.New("failed to create temp directory: " + err.Error()) + } + if restoredBackends, err := restoreBackends(db); err != nil { return errors.New("failed to restore backends: " + err.Error()) - } else if b, err := createLocalBackend(DefaultBackendName, map[string]string{"root": path.Join(Cfg.Location, "default")}); err != nil { + } else if b, err := createLocalBackend(DefaultBackendName, map[string]string{"root": path.Join(Cfg.Root, "default")}); err != nil { return errors.New("failed to create default backend: " + err.Error()) } else { backends = restoredBackends