diff --git a/readme.md b/readme.md index 9511cc0..4248229 100644 --- a/readme.md +++ b/readme.md @@ -7,7 +7,6 @@

null CupofDalek null -Clayton Stone null null null diff --git a/src/image.go b/src/image.go index bfa7380..608a4f7 100644 --- a/src/image.go +++ b/src/image.go @@ -242,110 +242,110 @@ func CommitMigrationPre014() { utils.SetBaseMainConfig(config) } -func MigratePre014() { - config := utils.ReadConfigFromFile() - if config.LastMigration != "" { - c, err := utils.CompareSemver(config.LastMigration, "0.14.0") - if err != nil { - utils.Fatal("Can't read field 'lastMigration' from config", err) - } - if(c > 0) { - return - } - } +// func MigratePre014() { +// config := utils.ReadConfigFromFile() +// if config.LastMigration != "" { +// c, err := utils.CompareSemver(config.LastMigration, "0.14.0") +// if err != nil { +// utils.Fatal("Can't read field 'lastMigration' from config", err) +// } +// if(c > 0) { +// return +// } +// } - if _, err := os.Stat(utils.CONFIGFOLDER + "database"); err != nil { - utils.Log("MigratePre014: No database found, trying to migrate pre-0.14 data") +// if _, err := os.Stat(utils.CONFIGFOLDER + "database"); err != nil { +// utils.Log("MigratePre014: No database found, trying to migrate pre-0.14 data") - cu2, errCo := utils.GetCollection(utils.GetRootAppId(), "users") - if errCo != nil { - // Assuming we dont need to migrate - utils.Log("MigratePre014: No database, assuming we dont need to migrate") - utils.Debug(errCo.Error()) - CommitMigrationPre014() - return - } +// cu2, errCo := utils.GetCollection(utils.GetRootAppId(), "users") +// if errCo != nil { +// // Assuming we dont need to migrate +// utils.Log("MigratePre014: No database, assuming we dont need to migrate") +// utils.Debug(errCo.Error()) +// CommitMigrationPre014() +// return +// } - cd2, errCo := utils.GetCollection(utils.GetRootAppId(), "devices") - if errCo != nil { - // Assuming we dont need to migrate - utils.Log("MigratePre014: No database, assuming we dont need to migrate") - utils.Debug(errCo.Error()) - CommitMigrationPre014() - return - } +// cd2, errCo := utils.GetCollection(utils.GetRootAppId(), "devices") +// if errCo != nil { +// // Assuming we dont need to migrate +// utils.Log("MigratePre014: No database, assuming we dont need to migrate") +// utils.Debug(errCo.Error()) +// CommitMigrationPre014() +// return +// } - utils.Log("MigratePre014: Migrating pre-0.14 data") +// utils.Log("MigratePre014: Migrating pre-0.14 data") - cu, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users") +// cu, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "users") - if errCo != nil { - utils.Fatal("Error trying to migrate pre-0.14 data [1]", errCo) - return - } +// if errCo != nil { +// utils.Fatal("Error trying to migrate pre-0.14 data [1]", errCo) +// return +// } - // copy users from cu2 to cu - cursor, err := cu2.Find(nil, map[string]interface{}{}) - if err != nil && err != mongo.ErrNoDocuments { - utils.Fatal("Error trying to migrate pre-0.14 data [3]", err) - return - } - defer cursor.Close(nil) +// // copy users from cu2 to cu +// cursor, err := cu2.Find(nil, map[string]interface{}{}) +// if err != nil && err != mongo.ErrNoDocuments { +// utils.Fatal("Error trying to migrate pre-0.14 data [3]", err) +// return +// } +// defer cursor.Close(nil) - users := []utils.User{} +// users := []utils.User{} - if err = cursor.All(nil, &users); err != nil { - utils.Fatal("Error trying to migrate pre-0.14 data [4]", err) - return - } +// if err = cursor.All(nil, &users); err != nil { +// utils.Fatal("Error trying to migrate pre-0.14 data [4]", err) +// return +// } - for _, user := range users { - _, err := cu.InsertOne(nil, user) - if err != nil { - utils.Fatal("Error trying to migrate pre-0.14 data [5]", err) - return - } - } +// for _, user := range users { +// _, err := cu.InsertOne(nil, user) +// if err != nil { +// utils.Fatal("Error trying to migrate pre-0.14 data [5]", err) +// return +// } +// } - closeDb() +// closeDb() - utils.Log("MigratePre014: Migrated " + fmt.Sprint(len(users)) + " users") +// utils.Log("MigratePre014: Migrated " + fmt.Sprint(len(users)) + " users") - cd, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices") - defer closeDb() +// cd, closeDb, errCo := utils.GetEmbeddedCollection(utils.GetRootAppId(), "devices") +// defer closeDb() - if errCo != nil { - utils.Fatal("Error trying to migrate pre-0.14 data [2]", errCo) - return - } +// if errCo != nil { +// utils.Fatal("Error trying to migrate pre-0.14 data [2]", errCo) +// return +// } - // copy devices from cd2 to cd - cursor, err = cd2.Find(nil, map[string]interface{}{}) - if err != nil && err != mongo.ErrNoDocuments { - utils.Fatal("Error trying to migrate pre-0.14 data [6]", err) - return - } +// // copy devices from cd2 to cd +// cursor, err = cd2.Find(nil, map[string]interface{}{}) +// if err != nil && err != mongo.ErrNoDocuments { +// utils.Fatal("Error trying to migrate pre-0.14 data [6]", err) +// return +// } - defer cursor.Close(nil) +// defer cursor.Close(nil) - devices := []utils.ConstellationDevice{} +// devices := []utils.ConstellationDevice{} - if err = cursor.All(context.Background(), &devices); err != nil { - utils.Fatal("Error trying to migrate pre-0.14 data [7]", err) - return - } +// if err = cursor.All(context.Background(), &devices); err != nil { +// utils.Fatal("Error trying to migrate pre-0.14 data [7]", err) +// return +// } - for _, device := range devices { - _, err := cd.InsertOne(context.Background(), device) - if err != nil { - utils.Fatal("Error trying to migrate pre-0.14 data [8]", err) - return - } - } +// for _, device := range devices { +// _, err := cd.InsertOne(context.Background(), device) +// if err != nil { +// utils.Fatal("Error trying to migrate pre-0.14 data [8]", err) +// return +// } +// } - utils.Log("MigratePre014: Migrated " + fmt.Sprint(len(devices)) + " devices") +// utils.Log("MigratePre014: Migrated " + fmt.Sprint(len(devices)) + " devices") - CommitMigrationPre014() - } -} \ No newline at end of file +// CommitMigrationPre014() +// } +// } \ No newline at end of file diff --git a/src/migrate.go b/src/migrate.go new file mode 100644 index 0000000..fdb2742 --- /dev/null +++ b/src/migrate.go @@ -0,0 +1,149 @@ +package main + +import ( + "os" + "strings" + "context" + // "fmt" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" + "go.mongodb.org/mongo-driver/mongo/writeconcern" + "go.mongodb.org/mongo-driver/bson" + + "github.com/azukaar/cosmos-server/src/utils" +) +func MigratePre014Coll(collection string, from *mongo.Client) { + name := os.Getenv("MONGODB_NAME") + if name == "" { + name = "COSMOS" + } + + utils.Log("Migrating collection " + collection + " from database " + name) + + applicationId := utils.GetRootAppId() + + cf := from.Database(name).Collection(applicationId + "_" + collection) + ct, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), collection) + if err != nil { + utils.Error("Error getting collection " + applicationId + "_" + collection + " from database " + name, err) + return + } + defer closeDb() + + // get all documents + opts := options.Find() + cur, err := cf.Find(context.Background(), bson.D{}, opts) + if err != nil { + utils.Error("Error getting documents from " + collection + " collection", err) + return + } + defer cur.Close(context.Background()) + + var batch []interface{} + batchSize := 100 // Define a suitable batch size + + for cur.Next(context.Background()) { + var elem bson.D + if err := cur.Decode(&elem); err != nil { + utils.Error("Error decoding document from " + collection + " collection", err) + continue + } + + batch = append(batch, elem) + + if len(batch) >= batchSize { + if _, err := ct.InsertMany(context.Background(), batch); err != nil { + utils.Error("Error inserting batch into " + collection + " collection", err) + } + batch = batch[:0] // Clear the batch + } + } + + // Insert any remaining documents + if len(batch) > 0 { + if _, err := ct.InsertMany(context.Background(), batch); err != nil { + utils.Error("Error inserting remaining documents into " + collection + " collection", err) + } + } + + if err := cur.Err(); err != nil { + utils.Error("Error iterating over documents from " + collection + " collection", err) + } +} + + +func MigratePre014() { + config := utils.GetMainConfig() + + // check if COSMOS.db does NOT exist + if _, err := os.Stat(utils.CONFIGFOLDER + "database"); err != nil && config.MongoDB != "" { + // connect to MongoDB + utils.Log("Connecting to MongoDB...") + + + mongoURL := utils.GetMainConfig().MongoDB + + var err error + + opts := options.Client().ApplyURI(mongoURL).SetRetryWrites(true).SetWriteConcern(writeconcern.New(writeconcern.WMajority())) + + if os.Getenv("HOSTNAME") == "" || utils.IsHostNetwork { + hostname := opts.Hosts[0] + // split port + hostnameParts := strings.Split(hostname, ":") + hostname = hostnameParts[0] + port := "27017" + + if len(hostnameParts) > 1 { + port = hostnameParts[1] + } + + utils.Log("Getting Mongo DB IP from name : " + hostname + " (port " + port + ")") + + ip, _ := utils.GetContainerIPByName(hostname) + if ip != "" { + // IsDBaContainer = true + opts.SetHosts([]string{ip + ":" + port}) + utils.Log("Mongo DB IP : " + ip) + } + } + + client, err := mongo.Connect(context.TODO(), opts) + + if err != nil { + panic(err) + } + + // Ping the primary + if err := client.Ping(context.TODO(), readpref.Primary()); err != nil { + panic(err) + } + + utils.Log("Successfully connected to the database.") + + ct, closeDb, err := utils.GetEmbeddedCollection(utils.GetRootAppId(), "events") + if err != nil { + return + } + defer closeDb() + + // Create a date index + model := mongo.IndexModel{ + Keys: bson.M{"Date": -1}, + } + + // Creating the index + _, err = ct.Indexes().CreateOne(context.Background(), model) + if err != nil { + utils.Error("Metrics - Create Index", err) + return // Handle error appropriately + } + + MigratePre014Coll("users", client) + MigratePre014Coll("devices", client) + // MigratePre014Coll("events", client) + // MigratePre014Coll("notifications", client) + // MigratePre014Coll("metrics", client) + } +} \ No newline at end of file