diff --git a/server/internal/command/appcmd/appcmd.go b/server/internal/command/appcmd/appcmd.go index 10f110aa..395a2703 100644 --- a/server/internal/command/appcmd/appcmd.go +++ b/server/internal/command/appcmd/appcmd.go @@ -1,13 +1,7 @@ package appcmd import ( - "context" - "fmt" - "os" - - "github.com/shroff/phylum/server/internal/core/app" - "github.com/shroff/phylum/server/internal/core/db" - "github.com/shroff/phylum/server/internal/core/storage" + "github.com/shroff/phylum/server/internal/command/common" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -21,21 +15,14 @@ func SetupCommand() *cobra.Command { for ; c.Parent() != nil; c = c.Parent() { } c.PersistentPreRun(cmd, args) - if err := db.Default.CheckVersion(context.Background(), !viper.GetBool("no_auto_migrate")); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - if err := storage.Create(context.Background(), db.Default, viper.GetString("default_storage_dir")); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } - if err := app.Create(context.Background(), db.Default, storage.Default); err != nil { - fmt.Println(err.Error()) - os.Exit(1) + + if viper.GetBool("no_auto_migrate") { + common.MustInitializeDB(false) } + common.MustInitializeApp() }, } - flags := cmd.Flags() + flags := cmd.PersistentFlags() flags.Bool("no-auto-migrate", false, "Do not automatically migrate database schema") viper.BindPFlag("no_auto_migrate", flags.Lookup("no-auto-migrate")) diff --git a/server/internal/command/command.go b/server/internal/command/command.go index 82a90e90..67365121 100644 --- a/server/internal/command/command.go +++ b/server/internal/command/command.go @@ -1,7 +1,6 @@ package command import ( - "context" "os" "path" @@ -45,11 +44,6 @@ func SetupCommand() { os.Mkdir(workDir, 0750) os.Chdir(workDir) } - - var err error - if err = db.Create(context.Background(), viper.GetString("database_url"), debug && viper.GetBool("trace_sql")); err != nil { - logrus.Fatal(err) - } } defer func() { diff --git a/server/internal/command/common/common.go b/server/internal/command/common/common.go new file mode 100644 index 00000000..51778752 --- /dev/null +++ b/server/internal/command/common/common.go @@ -0,0 +1,37 @@ +package common + +import ( + "context" + "fmt" + "os" + + "github.com/shroff/phylum/server/internal/core/app" + "github.com/shroff/phylum/server/internal/core/db" + "github.com/shroff/phylum/server/internal/core/storage" + "github.com/spf13/viper" +) + +func MustInitializeDB(autoMigrate bool) { + databaseURL := viper.GetString("database_url") + traceSQL := viper.GetBool("debug") && viper.GetBool("trace_sql") + if err := db.Initialize(context.Background(), databaseURL, traceSQL, autoMigrate); err != nil { + fmt.Println("could create database: " + err.Error()) + os.Exit(1) + } +} + +func MustInitializeStorage() { + MustInitializeDB(true) + if err := storage.Initialize(context.Background(), db.Default, viper.GetString("default_storage_dir")); err != nil { + fmt.Println("could not open storage: " + err.Error()) + os.Exit(2) + } +} + +func MustInitializeApp() { + MustInitializeStorage() + if err := app.Initialize(context.Background(), db.Default, storage.Default); err != nil { + fmt.Println("could not initialize app: " + err.Error()) + os.Exit(3) + } +} diff --git a/server/internal/command/fs/chperm.go b/server/internal/command/fs/chperm.go index 434abca5..1d7401a3 100644 --- a/server/internal/command/fs/chperm.go +++ b/server/internal/command/fs/chperm.go @@ -5,7 +5,6 @@ import ( "fmt" "os" - "github.com/shroff/phylum/server/internal/command/util" "github.com/shroff/phylum/server/internal/core/app" "github.com/shroff/phylum/server/internal/core/fs" "github.com/spf13/cobra" @@ -17,11 +16,6 @@ func setupChpermCommand() *cobra.Command { Short: "Change Permissions", Args: cobra.ExactArgs(3), Run: func(cmd *cobra.Command, args []string) { - f, err := util.OpenFileSystemFromFlags(cmd) - if err != nil { - fmt.Println("could not open filesystem: " + err.Error()) - os.Exit(1) - } pathOrUuid := args[0] r, err := f.ResourceByPathOrUuid(pathOrUuid) if err != nil { diff --git a/server/internal/command/fs/cmd.go b/server/internal/command/fs/cmd.go index d450c6dd..af6287ee 100644 --- a/server/internal/command/fs/cmd.go +++ b/server/internal/command/fs/cmd.go @@ -2,16 +2,19 @@ package fs import ( "context" + "errors" "fmt" "os" + "github.com/shroff/phylum/server/internal/command/common" "github.com/shroff/phylum/server/internal/core/app" - "github.com/shroff/phylum/server/internal/core/db" - "github.com/shroff/phylum/server/internal/core/storage" + "github.com/shroff/phylum/server/internal/core/fs" "github.com/spf13/cobra" "github.com/spf13/viper" ) +var f fs.FileSystem + func SetupCommand() *cobra.Command { cmd := &cobra.Command{ Use: "fs", @@ -23,18 +26,11 @@ func SetupCommand() *cobra.Command { } c.PersistentPreRun(cmd, args) - if err := db.Default.CheckVersion(context.Background(), !viper.GetBool("no_auto_migrate")); err != nil { - fmt.Println("could not migrate database schema: " + err.Error()) - os.Exit(1) - } - if err := storage.Create(context.Background(), db.Default, viper.GetString("default_storage_dir")); err != nil { - fmt.Println("could not open storage: " + err.Error()) - os.Exit(1) - } - if err := app.Create(context.Background(), db.Default, storage.Default); err != nil { - fmt.Println("could not initialized app: " + err.Error()) - os.Exit(1) + if viper.GetBool("no_auto_migrate") { + common.MustInitializeDB(false) } + common.MustInitializeApp() + mustOpenFileSystemFromFlags(cmd) }, } flags := cmd.PersistentFlags() @@ -56,3 +52,24 @@ func SetupCommand() *cobra.Command { return cmd } + +func mustOpenFileSystemFromFlags(cmd *cobra.Command) { + var err error + f, err = openFileSystemFromFlags(cmd) + if err != nil { + fmt.Println("could not open filesystem: " + err.Error()) + os.Exit(4) + } +} + +func openFileSystemFromFlags(cmd *cobra.Command) (fs.FileSystem, error) { + if value, err := cmd.Flags().GetString("user"); err != nil { + return nil, errors.New("could not read user: " + err.Error()) + } else { + if user, err := app.Default.UserByEmail(context.Background(), value); err != nil { + return nil, errors.New("could not find user '" + value + "': " + err.Error()) + } else { + return app.Default.OpenFileSystem(context.Background(), user), nil + } + } +} diff --git a/server/internal/command/fs/cp.go b/server/internal/command/fs/cp.go index c8f048af..fca35018 100644 --- a/server/internal/command/fs/cp.go +++ b/server/internal/command/fs/cp.go @@ -5,7 +5,6 @@ import ( "os" "github.com/google/uuid" - "github.com/shroff/phylum/server/internal/command/util" "github.com/spf13/cobra" ) @@ -15,12 +14,7 @@ func setupCpCommand() *cobra.Command { Short: "Copy Resource", Args: cobra.ExactArgs(2), Run: func(cmd *cobra.Command, args []string) { - f, err := util.OpenFileSystemFromFlags(cmd) - if err != nil { - fmt.Println("could not open filesystem: " + err.Error()) - os.Exit(1) - } - + mustOpenFileSystemFromFlags(cmd) srcPathOrUuid := args[0] src, err := f.ResourceByPathOrUuid(srcPathOrUuid) if err != nil { diff --git a/server/internal/command/fs/ls.go b/server/internal/command/fs/ls.go index 2c41af7f..6988d087 100644 --- a/server/internal/command/fs/ls.go +++ b/server/internal/command/fs/ls.go @@ -6,7 +6,6 @@ import ( "sort" "strings" - "github.com/shroff/phylum/server/internal/command/util" "github.com/shroff/phylum/server/internal/core/fs" "github.com/spf13/cobra" ) @@ -40,12 +39,6 @@ func setupLsCommand() *cobra.Command { Short: "List Resource Details", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - f, err := util.OpenFileSystemFromFlags(cmd) - if err != nil { - fmt.Println("could not open filesystem: " + err.Error()) - os.Exit(1) - } - pathOrUuid := args[0] r, err := f.ResourceByPathOrUuid(pathOrUuid) if err != nil { diff --git a/server/internal/command/fs/mkdir.go b/server/internal/command/fs/mkdir.go index 9411aa04..0d770962 100644 --- a/server/internal/command/fs/mkdir.go +++ b/server/internal/command/fs/mkdir.go @@ -1,12 +1,9 @@ package fs import ( - "fmt" - "os" "strings" "github.com/google/uuid" - "github.com/shroff/phylum/server/internal/command/util" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -17,12 +14,6 @@ func setupMkdirCommand() *cobra.Command { Short: "Create Directory", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - f, err := util.OpenFileSystemFromFlags(cmd) - if err != nil { - fmt.Println("could not open filesystem: " + err.Error()) - os.Exit(1) - } - path := args[0] if _, err := f.ResourceByPath(path); err == nil { logrus.Fatal("Resource already exists: " + path) diff --git a/server/internal/command/fs/mv.go b/server/internal/command/fs/mv.go index c7aff336..0bfd755c 100644 --- a/server/internal/command/fs/mv.go +++ b/server/internal/command/fs/mv.go @@ -4,7 +4,6 @@ import ( "fmt" "os" - "github.com/shroff/phylum/server/internal/command/util" "github.com/spf13/cobra" ) @@ -14,12 +13,6 @@ func setupMvCommand() *cobra.Command { Short: "Move Resource", Args: cobra.ExactArgs(2), Run: func(cmd *cobra.Command, args []string) { - f, err := util.OpenFileSystemFromFlags(cmd) - if err != nil { - fmt.Println("could not open filesystem: " + err.Error()) - os.Exit(1) - } - srcPathOrUuid := args[0] src, err := f.ResourceByPathOrUuid(srcPathOrUuid) if err != nil { diff --git a/server/internal/command/fs/rm.go b/server/internal/command/fs/rm.go index 35ff51d1..735b4199 100644 --- a/server/internal/command/fs/rm.go +++ b/server/internal/command/fs/rm.go @@ -4,7 +4,6 @@ import ( "fmt" "os" - "github.com/shroff/phylum/server/internal/command/util" "github.com/spf13/cobra" ) @@ -14,12 +13,6 @@ func setupRmCommand() *cobra.Command { Short: "Delete Resource", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - f, err := util.OpenFileSystemFromFlags(cmd) - if err != nil { - fmt.Println("could not open filesystem: " + err.Error()) - os.Exit(1) - } - pathOrUuid := args[0] r, err := f.ResourceByPathOrUuid(pathOrUuid) if err != nil { diff --git a/server/internal/command/schema/schema.go b/server/internal/command/schema/schema.go index 2502d5a2..76fa4492 100644 --- a/server/internal/command/schema/schema.go +++ b/server/internal/command/schema/schema.go @@ -6,6 +6,7 @@ import ( "os" "strconv" + "github.com/shroff/phylum/server/internal/command/common" "github.com/shroff/phylum/server/internal/core/db" "github.com/spf13/cobra" ) @@ -28,7 +29,7 @@ func setupSchemaResetCommand() *cobra.Command { Short: "Reset Database Schema", Run: func(cmd *cobra.Command, args []string) { if err := db.Default.DeleteSchema(context.Background()); err != nil { - fmt.Println(err.Error()) + fmt.Println("unable to delete databse schema: " + err.Error()) os.Exit(1) } }, @@ -43,12 +44,9 @@ func setupSchemaMigrateCommand() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { ctx := context.Background() if args[0] == "auto" || args[0] == "latest" { - if err := db.Default.CheckVersion(ctx, true); err != nil { - fmt.Println(err.Error()) - os.Exit(2) - } + common.MustInitializeDB(true) } else { - db.Default.CheckVersion(ctx, false) + common.MustInitializeDB(false) if v, err := strconv.Atoi(args[0]); err != nil { fmt.Println(err.Error()) os.Exit(3) diff --git a/server/internal/command/util/fs.go b/server/internal/command/util/fs.go deleted file mode 100644 index 73478d17..00000000 --- a/server/internal/command/util/fs.go +++ /dev/null @@ -1,22 +0,0 @@ -package util - -import ( - "context" - "errors" - - "github.com/shroff/phylum/server/internal/core/app" - "github.com/shroff/phylum/server/internal/core/fs" - "github.com/spf13/cobra" -) - -func OpenFileSystemFromFlags(cmd *cobra.Command) (fs.FileSystem, error) { - if value, err := cmd.Flags().GetString("user"); err != nil { - return nil, errors.New("could not read user: " + err.Error()) - } else { - if user, err := app.Default.UserByEmail(context.Background(), value); err != nil { - return nil, errors.New("could not find user '" + value + "': " + err.Error()) - } else { - return app.Default.OpenFileSystem(context.Background(), user), nil - } - } -} diff --git a/server/internal/core/app/app.go b/server/internal/core/app/app.go index bcdf8314..5934150a 100644 --- a/server/internal/core/app/app.go +++ b/server/internal/core/app/app.go @@ -22,7 +22,10 @@ type App struct { var Default *App -func Create(ctx context.Context, db *db.DbHandler, cs storage.Storage) error { +func Initialize(ctx context.Context, db *db.DbHandler, cs storage.Storage) error { + if Default != nil { + return nil + } Default = &App{ db: db, cs: cs, diff --git a/server/internal/core/db/db_handler.go b/server/internal/core/db/db_handler.go index 79841c4e..287bb65c 100644 --- a/server/internal/core/db/db_handler.go +++ b/server/internal/core/db/db_handler.go @@ -7,7 +7,6 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" - "github.com/shroff/phylum/server/internal/core/db/migrations" "github.com/sirupsen/logrus" ) @@ -17,9 +16,6 @@ var ( ErrMigrationNoAutoDowngrade = errors.New("will not auto-downgrade schema to prevent data loss") ) -var currentSchemaVersion int -var latestSchemaVersion int - var Default *DbHandler type DbHandler struct { @@ -30,7 +26,10 @@ type DbHandler struct { pool *pgxpool.Pool } -func Create(ctx context.Context, dsn string, trace bool) error { +func Initialize(ctx context.Context, dsn string, autoMigrate, trace bool) error { + if Default != nil { + return nil + } config, err := pgxpool.ParseConfig(dsn) if err != nil { return errors.New("Unable to parse DSN: " + err.Error()) @@ -47,27 +46,14 @@ func Create(ctx context.Context, dsn string, trace bool) error { logrus.Debug("Connected to " + config.ConnConfig.Database + " at " + config.ConnConfig.Host + ":" + fmt.Sprint(config.ConnConfig.Port)) - conn, err := pool.Acquire(ctx) - if err != nil { - return err - } - migrator, err := migrations.NewMigrator(ctx, conn.Conn()) - if err != nil { - return err - } - currentSchemaVersion, err = migrator.GetCurrentVersion(ctx) - if err != nil { - return err - } - latestSchemaVersion = migrator.GetLatestVersion() - conn.Release() - Default = &DbHandler{ Queries: *New(pool), tx: pool, pool: pool, } + Default.checkVersion(ctx, autoMigrate) + return nil } diff --git a/server/internal/core/db/schema.go b/server/internal/core/db/schema.go index 04ed025e..b84a8264 100644 --- a/server/internal/core/db/schema.go +++ b/server/internal/core/db/schema.go @@ -8,23 +8,39 @@ import ( "github.com/sirupsen/logrus" ) -func (d DbHandler) CheckVersion(ctx context.Context, autoMigrate bool) error { - logrus.Debug(fmt.Sprintf("Schema version %d", currentSchemaVersion)) - if currentSchemaVersion != latestSchemaVersion { - if autoMigrate { - if currentSchemaVersion > latestSchemaVersion { - return ErrMigrationNoAutoDowngrade - } - if currentSchemaVersion == latestSchemaVersion { - return nil - } - return d.Migrate(ctx, latestSchemaVersion) - } else { - logrus.Warn(fmt.Sprintf("Schema version is not at latest (%d / %d)", currentSchemaVersion, latestSchemaVersion)) - return nil - } +func (d DbHandler) checkVersion(ctx context.Context, autoMigrate bool) error { + conn, err := d.pool.Acquire(ctx) + if err != nil { + return err } - return nil + defer conn.Release() + + migrator, err := migrations.NewMigrator(ctx, conn.Conn()) + currentSchemaVersion, err := migrator.GetCurrentVersion(ctx) + if err != nil { + return err + } + latestSchemaVersion := migrator.GetLatestVersion() + if err != nil { + return err + } + + // Nothing to do + if currentSchemaVersion == latestSchemaVersion { + return nil + } + + if !autoMigrate { + logrus.Warn(fmt.Sprintf("Database schema is not at current version: %d/%d", currentSchemaVersion, latestSchemaVersion)) + return nil + } + if currentSchemaVersion > latestSchemaVersion { + logrus.Warn(fmt.Sprintf("Not automatically downgrading schema from %d to %d", currentSchemaVersion, latestSchemaVersion)) + return nil + } + + logrus.Info(fmt.Sprintf("Migrating database from version %d to %d", currentSchemaVersion, latestSchemaVersion)) + return migrator.MigrateTo(ctx, int32(latestSchemaVersion)) } func (d DbHandler) Migrate(ctx context.Context, version int) error { @@ -33,21 +49,26 @@ func (d DbHandler) Migrate(ctx context.Context, version int) error { return err } defer conn.Release() - if version < 0 { - return ErrMigrationTargetTooLow - } - if version > latestSchemaVersion { - return ErrMigrationTargetTooHigh - } + migrator, err := migrations.NewMigrator(ctx, conn.Conn()) + currentSchemaVersion, err := migrator.GetCurrentVersion(ctx) if err != nil { return err } - logrus.Debug(fmt.Sprintf("Migrating database from version %d to %d", currentSchemaVersion, version)) - if err = migrator.MigrateTo(ctx, int32(version)); err != nil { + latestSchemaVersion := migrator.GetLatestVersion() + if err != nil { return err } - return nil + if version < 0 { + version = latestSchemaVersion + } else if version == 0 { + return d.DeleteSchema(ctx) + } else if version > latestSchemaVersion { + return ErrMigrationTargetTooHigh + } + + logrus.Info(fmt.Sprintf("Migrating database from version %d to %d", currentSchemaVersion, version)) + return migrator.MigrateTo(ctx, int32(version)) } func (d DbHandler) DeleteSchema(ctx context.Context) error { diff --git a/server/internal/core/storage/storage.go b/server/internal/core/storage/storage.go index a6c218b0..b5ad7ef1 100644 --- a/server/internal/core/storage/storage.go +++ b/server/internal/core/storage/storage.go @@ -28,7 +28,10 @@ type storage struct { var Default Storage -func Create(ctx context.Context, db *db.DbHandler, defaultStorageDir string) error { +func Initialize(ctx context.Context, db *db.DbHandler, defaultStorageDir string) error { + if Default != nil { + return nil + } if backends, err := restoreBackends(db, ctx); err != nil { return err } else if defaultBackend, err := newLocalStorage("", defaultStorageDir); err != nil {