Storage Backends

internal/command/admin.go

@@ -10,8 +10,9 @@ func setupAdminCommand() *cobra.Command {
 		Short: "Server Administration",
 	}
 	cmd.AddCommand([]*cobra.Command{
-		setupLibraryCommand(),
+		setupStorageCommand(),
 		setupUserCommand(),
+		setupLibraryCommand(),
 	}...)
 
 	return cmd

internal/command/command.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/shroff/phylum/server/internal/library"
 	"github.com/shroff/phylum/server/internal/sql"
+	"github.com/shroff/phylum/server/internal/storage"
 	"github.com/shroff/phylum/server/internal/user"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -14,8 +15,9 @@ import (
 )
 
 var debug bool = false
-var libraryManager library.Manager
-var userManager user.Manager
+var libraryManager *library.Manager
+var userManager *user.Manager
+var storageManager *storage.Manager
 
 func SetupCommand() {
 	viper.SetEnvPrefix("phylum")
@@ -57,8 +59,19 @@ func SetupCommand() {
 		if err != nil {
 			logrus.Fatal(err)
 		}
-		libraryManager = library.NewManager(db)
-		userManager = user.NewManager(db)
+
+		userManager, err = user.NewManager(db)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		storageManager, err = storage.NewManager(db)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		libraryManager, err = library.NewManager(db, storageManager)
+		if err != nil {
+			logrus.Fatal(err)
+		}
 	}
 
 	defer func() {

internal/command/library.go

@@ -22,18 +22,25 @@ func setupLibraryCommand() *cobra.Command {
 
 func setupLibraryCreateCommand() *cobra.Command {
 	return &cobra.Command{
-		Use:   "create owner display-name",
+		Use:   "create storage-backend owner display-name",
 		Short: "Create Library",
-		Args:  cobra.ExactArgs(2),
+		Args:  cobra.ExactArgs(3),
 		Run: func(cmd *cobra.Command, args []string) {
 			id := uuid.New()
-			username := args[0]
+
+			storageName := args[0]
+			storage := storageManager.Find(storageName)
+			if storage == nil {
+				logrus.Fatal("Storage not found: " + storageName)
+			}
+
+			username := args[1]
 			user, err := userManager.FindUser(context.Background(), username)
 			if err != nil {
 				logrus.Fatal("User not found: " + username)
 			}
-			name := args[1]
-			if err := libraryManager.Create(context.Background(), id, user.ID, name); err != nil {
+			name := args[2]
+			if err := libraryManager.Create(context.Background(), id, storageName, user.ID, name); err != nil {
 				logrus.Fatal(err)
 			}
 			logrus.Info("Created " + id.String())

internal/command/storage.go

@@ -0,0 +1,75 @@
+package command
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/shroff/phylum/server/internal/storage"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func setupStorageCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "storage",
+		Short: "Storage Management",
+	}
+	cmd.AddCommand([]*cobra.Command{
+		setupStorageCreateCommand(),
+		setupStorageListCommand(),
+	}...)
+	return cmd
+}
+
+func setupStorageCreateCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "create name driver",
+		Short: "Create new storage backend",
+		Args:  cobra.ExactArgs(2),
+		Run: func(cmd *cobra.Command, args []string) {
+			name := args[0]
+			if storageManager.Find(name) != nil {
+				logrus.Fatal("Storage backand already exists: " + name)
+			}
+
+			driver := args[1]
+			paramNames, err := storage.ParamNames(driver)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			params := map[string]string{}
+			reader := bufio.NewReader(os.Stdin)
+			for _, paramName := range paramNames {
+				os.Stdout.WriteString(paramName + ": ")
+				val, err := reader.ReadString('\n')
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				val = strings.TrimSpace(val)
+				params[paramName] = val
+			}
+
+			if err := storageManager.Create(name, driver, params); err != nil {
+				logrus.Fatal(err)
+			}
+
+			logrus.Info("Storage backend created: " + name)
+		},
+	}
+}
+
+func setupStorageListCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "list",
+		Short: "List all storage backends",
+		Run: func(cmd *cobra.Command, args []string) {
+			logrus.Info("Available storage backends:")
+			for k, v := range storageManager.All() {
+				logrus.Info(fmt.Sprintf("%s: %s", k, v))
+			}
+		},
+	}
+}

internal/library/library_manager.go

@@ -2,6 +2,7 @@ package library
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/google/uuid"
@@ -10,16 +11,12 @@ import (
 )
 
 type Manager struct {
-	db *sql.DbHandler
-	cs storage.Storage
+	db             *sql.DbHandler
+	storageManager *storage.Manager
 }
 
-func NewManager(db *sql.DbHandler) Manager {
-	cs, err := storage.NewLocalStorage("srv")
-	if err != nil {
-		panic(err)
-	}
-	return Manager{db: db, cs: cs}
+func NewManager(db *sql.DbHandler, storageManager *storage.Manager) (*Manager, error) {
+	return &Manager{db: db, storageManager: storageManager}, nil
 }
 
 func (b Manager) Get(ctx context.Context, id uuid.UUID) (*Library, error) {
@@ -28,13 +25,22 @@ func (b Manager) Get(ctx context.Context, id uuid.UUID) (*Library, error) {
 	if err != nil {
 		return nil, err
 	}
+	cs := b.storageManager.Find(lib.StorageBackend)
+	if cs == nil {
+		return nil, errors.New("storage backend not found: " + lib.StorageBackend)
+	}
 
-	return &Library{db: b.db, root: lib.ID, cs: b.cs}, nil
+	return &Library{db: b.db, root: lib.ID, cs: cs}, nil
 }
 
-func (b Manager) Create(ctx context.Context, id uuid.UUID, owner int32, displayName string) error {
+func (b Manager) Create(ctx context.Context, id uuid.UUID, storageBackend string, owner int32, displayName string) error {
 	return b.db.RunInTx(ctx, func(q *sql.Queries) error {
-		if err := q.CreateLibrary(ctx, sql.CreateLibraryParams{ID: id, Owner: owner, DisplayName: displayName}); err != nil {
+		if err := q.CreateLibrary(ctx, sql.CreateLibraryParams{
+			ID:             id,
+			StorageBackend: storageBackend,
+			Owner:          owner,
+			DisplayName:    displayName,
+		}); err != nil {
 			return err
 		}
 		if err := q.CreateResource(ctx, sql.CreateResourceParams{ID: id, Dir: true}); err != nil {

internal/sql/libraries.sql.go

@@ -13,25 +13,31 @@ import (
 
 const createLibrary = `-- name: CreateLibrary :exec
 INSERT INTO libraries(
-    id, owner, display_name
+    id, owner, display_name, storage_backend
 ) VALUES(
-    $1, $2, $3
+    $1, $2, $3, $4
 )
 `
 
 type CreateLibraryParams struct {
-	ID          uuid.UUID
-	Owner       int32
-	DisplayName string
+	ID             uuid.UUID
+	Owner          int32
+	DisplayName    string
+	StorageBackend string
 }
 
 func (q *Queries) CreateLibrary(ctx context.Context, arg CreateLibraryParams) error {
-	_, err := q.db.Exec(ctx, createLibrary, arg.ID, arg.Owner, arg.DisplayName)
+	_, err := q.db.Exec(ctx, createLibrary,
+		arg.ID,
+		arg.Owner,
+		arg.DisplayName,
+		arg.StorageBackend,
+	)
 	return err
 }
 
 const libraryById = `-- name: LibraryById :one
-SELECT id, owner, display_name, deleted from libraries where id = $1
+SELECT id, owner, display_name, deleted, storage_backend from libraries where id = $1
 `
 
 func (q *Queries) LibraryById(ctx context.Context, id uuid.UUID) (Library, error) {
@@ -42,6 +48,7 @@ func (q *Queries) LibraryById(ctx context.Context, id uuid.UUID) (Library, error
 		&i.Owner,
 		&i.DisplayName,
 		&i.Deleted,
+		&i.StorageBackend,
 	)
 	return i, err
 }

internal/sql/models.go

@@ -10,10 +10,11 @@ import (
 )
 
 type Library struct {
-	ID          uuid.UUID
-	Owner       int32
-	DisplayName string
-	Deleted     pgtype.Timestamp
+	ID             uuid.UUID
+	Owner          int32
+	DisplayName    string
+	Deleted        pgtype.Timestamp
+	StorageBackend string
 }
 
 type Resource struct {
@@ -28,6 +29,12 @@ type Resource struct {
 	Etag     pgtype.Text
 }
 
+type StorageBackend struct {
+	Name   string
+	Driver string
+	Params map[string]string
+}
+
 type User struct {
 	ID           int32
 	DisplayName  string

internal/sql/storage_backends.sql.go

@@ -0,0 +1,53 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.25.0
+// source: storage_backends.sql
+
+package sql
+
+import (
+	"context"
+)
+
+const allStorageBackends = `-- name: AllStorageBackends :many
+SELECT name, driver, params from storage_backends
+`
+
+func (q *Queries) AllStorageBackends(ctx context.Context) ([]StorageBackend, error) {
+	rows, err := q.db.Query(ctx, allStorageBackends)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []StorageBackend
+	for rows.Next() {
+		var i StorageBackend
+		if err := rows.Scan(&i.Name, &i.Driver, &i.Params); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const createStorageBackend = `-- name: CreateStorageBackend :exec
+INSERT INTO storage_backends(
+    name, driver, params
+) VALUES(
+    $1, $2, $3
+)
+`
+
+type CreateStorageBackendParams struct {
+	Name   string
+	Driver string
+	Params map[string]string
+}
+
+func (q *Queries) CreateStorageBackend(ctx context.Context, arg CreateStorageBackendParams) error {
+	_, err := q.db.Exec(ctx, createStorageBackend, arg.Name, arg.Driver, arg.Params)
+	return err
+}

internal/storage/local_storage.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"crypto/md5"
+	"errors"
 	"hash"
 	"io"
 	"os"
@@ -13,6 +14,17 @@ import (
 
 type localStorage string
 
+func newLocalStorage(root string) (Storage, error) {
+	if root == "" {
+		return nil, errors.New("local storage root not provided")
+	}
+	err := os.MkdirAll(root, 0750)
+	if err != nil {
+		return nil, err
+	}
+	return localStorage(root), nil
+}
+
 func (l localStorage) Open(id uuid.UUID, flag int, closeCallback func(hash.Hash, int, error)) (file io.ReadWriteCloser, err error) {
 	file, err = os.OpenFile(l.path(id), flag, 0640)
 	if err != nil {
@@ -36,3 +48,7 @@ func (l localStorage) Delete(id uuid.UUID) error {
 func (l localStorage) path(id uuid.UUID) string {
 	return filepath.Join(string(l), id.String())
 }
+
+func (l localStorage) String() string {
+	return "local (" + string(l) + ")"
+}

internal/storage/storage.go

@@ -3,7 +3,6 @@ package storage
 import (
 	"hash"
 	"io"
-	"os"
 
 	"github.com/google/uuid"
 )
@@ -11,12 +10,5 @@ import (
 type Storage interface {
 	Open(id uuid.UUID, flag int, closeCallback func(hash.Hash, int, error)) (io.ReadWriteCloser, error)
 	Delete(id uuid.UUID) error
-}
-
-func NewLocalStorage(root string) (Storage, error) {
-	err := os.MkdirAll(root, 0750)
-	if err != nil {
-		return nil, err
-	}
-	return localStorage(root), nil
+	String() string
 }

internal/storage/storage_manager.go

@@ -0,0 +1,73 @@
+package storage
+
+import (
+	"context"
+	"errors"
+
+	"github.com/shroff/phylum/server/internal/sql"
+)
+
+type Manager struct {
+	db       *sql.DbHandler
+	backends map[string]Storage
+}
+
+func NewManager(db *sql.DbHandler) (*Manager, error) {
+	backends, err := db.Queries().AllStorageBackends(context.Background())
+	if err != nil {
+		return nil, err
+	}
+	manager := &Manager{
+		db:       db,
+		backends: map[string]Storage{},
+	}
+	for _, b := range backends {
+		store, err := openStorage(b.Driver, b.Params)
+		if err != nil {
+			return nil, err
+		}
+		manager.backends[b.Name] = store
+	}
+	return manager, err
+}
+
+func (m Manager) Create(name string, driver string, params map[string]string) error {
+	storage, err := openStorage(driver, params)
+	if err != nil {
+		return nil
+	}
+	err = m.db.Queries().CreateStorageBackend(context.Background(), sql.CreateStorageBackendParams{
+		Name:   name,
+		Driver: driver,
+		Params: params,
+	})
+	if err != nil {
+		return err
+	}
+	m.backends[name] = storage
+	return nil
+}
+
+func (m Manager) Find(name string) Storage {
+	return m.backends[name]
+}
+
+func (m Manager) All() map[string]Storage {
+	return m.backends
+}
+
+func openStorage(driver string, params map[string]string) (Storage, error) {
+	switch driver {
+	case "local":
+		return newLocalStorage(params["root"])
+	}
+	return nil, errors.New("unrecognized storage driver: " + driver)
+}
+
+func ParamNames(driver string) ([]string, error) {
+	switch driver {
+	case "local":
+		return []string{"root"}, nil
+	}
+	return nil, errors.New("unrecognized storage driver: " + driver)
+}

internal/user/user_manager.go

@@ -10,8 +10,8 @@ type Manager struct {
 	db *sql.DbHandler
 }
 
-func NewManager(db *sql.DbHandler) Manager {
-	return Manager{db: db}
+func NewManager(db *sql.DbHandler) (*Manager, error) {
+	return &Manager{db: db}, nil
 }
 
 func (m Manager) CreateUser(ctx context.Context, username, displayName, passwordHash string) error {

internal/webdav_xnet/handler.go

@@ -15,11 +15,11 @@ import (
 )
 
 type handler struct {
-	libraryManager library.Manager
+	libraryManager *library.Manager
 	prefix         string
 }
 
-func SetupHandler(r *gin.RouterGroup, libraryManager library.Manager, userManager user.Manager) {
+func SetupHandler(r *gin.RouterGroup, libraryManager *library.Manager, userManager *user.Manager) {
 	logrus.Info("Setting up WebDAV access at " + r.BasePath())
 	handler := &handler{
 		libraryManager: libraryManager,

sql/migrations/003_storage_backends.down.sql

@@ -0,0 +1 @@
+DROP TABLE storage_backends;

sql/migrations/003_storage_backends.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE storage_backends(
+    name         TEXT  PRIMARY KEY,
+    driver       TEXT  NOT NULL,
+    params       JSONB NOT NULL
+);

sql/migrations/004_libraries.up.sql

@@ -2,5 +2,6 @@ CREATE TABLE libraries(
     id           uuid   PRIMARY KEY,
     owner        SERIAL NOT NULL REFERENCES users(id) ON UPDATE CASCADE ON DELETE CASCADE,
     display_name TEXT   NOT NULL,
-    deleted      TIMESTAMP
+    deleted      TIMESTAMP,
+    storage_backend TEXT NOT NULL REFERENCES storage_backends(name) ON UPDATE CASCADE ON DELETE CASCADE
 );

sql/queries/libraries.sql

@@ -3,7 +3,7 @@ SELECT * from libraries where id = $1;
 
 -- name: CreateLibrary :exec
 INSERT INTO libraries(
-    id, owner, display_name
+    id, owner, display_name, storage_backend
 ) VALUES(
-    $1, $2, $3
+    $1, $2, $3, $4
 );

sql/queries/storage_backends.sql

@@ -0,0 +1,9 @@
+-- name: AllStorageBackends :many
+SELECT * from storage_backends;
+
+-- name: CreateStorageBackend :exec
+INSERT INTO storage_backends(
+    name, driver, params
+) VALUES(
+    $1, $2, $3
+);

sqlc.yml

@@ -18,4 +18,7 @@ sql:
             go_type:
               import: "github.com/google/uuid"
               type: "UUID"
-              pointer: true
+              pointer: true
+          - column: "storage_backends.params"
+            go_type:
+              type: "map[string]string"