From 1bc4a04374c943df39930dd0a75b204b5d3b0c64 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Tue, 6 Jun 2023 21:15:47 -0700 Subject: [PATCH 01/45] support event execution for sql server --- go/cmd/dolt/commands/engine/sqlengine.go | 42 ++++++++++++++++--- go/cmd/dolt/commands/sqlserver/server.go | 21 ++++++++++ .../dolt/commands/sqlserver/serverconfig.go | 19 +++++++++ go/cmd/dolt/commands/sqlserver/sqlserver.go | 17 ++++++++ go/cmd/dolt/commands/sqlserver/yaml_config.go | 17 ++++++++ go/go.mod | 2 +- go/go.sum | 4 +- go/libraries/doltcore/sqle/database.go | 36 ++++++++++++---- integration-tests/bats/events.bats | 33 +++++++++++++++ 9 files changed, 174 insertions(+), 17 deletions(-) create mode 100644 integration-tests/bats/events.bats diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index b0b0a8d5df..5962be80eb 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -21,6 +21,7 @@ import ( "strings" gms "github.com/dolthub/go-mysql-server" + "github.com/dolthub/go-mysql-server/eventscheduler" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/analyzer" "github.com/dolthub/go-mysql-server/sql/binlogreplication" @@ -66,6 +67,7 @@ type SqlEngineConfig struct { JwksConfig []JwksConfig ClusterController *cluster.Controller BinlogReplicaController binlogreplication.BinlogReplicaController + EventSchedulerStatus eventscheduler.SchedulerStatus } // NewSqlEngine returns a SqlEngine @@ -172,10 +174,17 @@ func NewSqlEngine( return nil, err } - sessionFactory := doltSessionFactory(pro, mrEnv.Config(), bcController, config.Autocommit) + sessFactory := doltSessionFactory(pro, mrEnv.Config(), bcController, config.Autocommit) + + if engine.EventScheduler == nil { + err = configureEventScheduler(config, engine, sessFactory, pro) + if err != nil { + return nil, err + } + } if config.BinlogReplicaController != nil { - binLogSession, err := sessionFactory(sql.NewBaseSession(), pro) + binLogSession, err := sessFactory(sql.NewBaseSession(), pro) if err != nil { return nil, err } @@ -189,7 +198,7 @@ func NewSqlEngine( return &SqlEngine{ provider: pro, contextFactory: sqlContextFactory(), - dsessFactory: sessionFactory, + dsessFactory: sessFactory, engine: engine, }, nil } @@ -266,9 +275,8 @@ func (se *SqlEngine) Close() error { // configureBinlogReplicaController configures the binlog replication controller with the |engine|. func configureBinlogReplicaController(config *SqlEngineConfig, engine *gms.Engine, session *dsess.DoltSession) error { - contextFactory := sqlContextFactory() - - executionCtx, err := contextFactory(context.Background(), session) + ctxFactory := sqlContextFactory() + executionCtx, err := ctxFactory(context.Background(), session) if err != nil { return err } @@ -278,6 +286,28 @@ func configureBinlogReplicaController(config *SqlEngineConfig, engine *gms.Engin return nil } +// configureEventScheduler configures the event scheduler with the |engine|. +func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFactory sessionFactory, pro dsqle.DoltDatabaseProvider) error { + // need to give correct user, use the definer as user to run the event definition queries + ctxFactory := sqlContextFactory() + // get new session each time query executes? + getCtxFunc := func() (*sql.Context, error) { + sess, err := sessFactory(sql.NewBaseSession(), pro) + if err != nil { + return nil, err + } + + newCtx, err := ctxFactory(context.Background(), sess) + if err != nil { + return nil, err + } + + return newCtx, nil + } + + return engine.InitializeEventScheduler(getCtxFunc, config.EventSchedulerStatus) +} + // sqlContextFactory returns a contextFactory that creates a new sql.Context with the initial database provided func sqlContextFactory() contextFactory { return func(ctx context.Context, session sql.Session) (*sql.Context, error) { diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index cca02db080..17be2d3a35 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -23,8 +23,10 @@ import ( "net/http" "runtime" "strconv" + "strings" "time" + "github.com/dolthub/go-mysql-server/eventscheduler" "github.com/dolthub/go-mysql-server/server" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/types" @@ -182,6 +184,12 @@ func Serve( ClusterController: clusterController, BinlogReplicaController: binlogreplication.DoltBinlogReplicaController, } + esStatus, err := getEventSchedulerStatus(serverConfig.EventSchedulerStatus()) + if err != nil { + return err, nil + } + config.EventSchedulerStatus = esStatus + sqlEngine, err := engine.NewSqlEngine( ctx, mrEnv, @@ -532,3 +540,16 @@ func checkForUnixSocket(config ServerConfig) (string, bool, error) { return "", false, nil } + +func getEventSchedulerStatus(status string) (eventscheduler.SchedulerStatus, error) { + switch strings.ToLower(status) { + case "on", "1": + return eventscheduler.SchedulerOn, nil + case "off", "0": + return eventscheduler.SchedulerOff, nil + case "disabled": + return eventscheduler.SchedulerDisabled, nil + default: + return eventscheduler.SchedulerDisabled, fmt.Errorf("Error while setting value '%s' to 'event_scheduler'.", status) + } +} \ No newline at end of file diff --git a/go/cmd/dolt/commands/sqlserver/serverconfig.go b/go/cmd/dolt/commands/sqlserver/serverconfig.go index 7c387dde59..cdd80ad5b0 100644 --- a/go/cmd/dolt/commands/sqlserver/serverconfig.go +++ b/go/cmd/dolt/commands/sqlserver/serverconfig.go @@ -168,6 +168,8 @@ type ServerConfig interface { RemotesapiPort() *int // ClusterConfig is the configuration for clustering in this sql-server. ClusterConfig() cluster.Config + // EventSchedulerStatus is the configuration for enabling or disabling the event scheduler in this server. + EventSchedulerStatus() string } type validatingServerConfig interface { @@ -204,6 +206,7 @@ type commandLineServerConfig struct { socket string remotesapiPort *int goldenMysqlConn string + eventSchedulerStatus string } var _ ServerConfig = (*commandLineServerConfig)(nil) @@ -494,6 +497,22 @@ func (cfg *commandLineServerConfig) withGoldenMysqlConnectionString(cs string) * return cfg } +func (cfg *commandLineServerConfig) EventSchedulerStatus() string { + switch cfg.eventSchedulerStatus { + case "", "1": + return "ON" + case "0": + return "OFF" + default: + return strings.ToUpper(cfg.eventSchedulerStatus) + } +} + +func (cfg *commandLineServerConfig) withEventScheduler(es string) *commandLineServerConfig { + cfg.eventSchedulerStatus = es + return cfg +} + // DefaultServerConfig creates a `*ServerConfig` that has all of the options set to their default values. func DefaultServerConfig() *commandLineServerConfig { return &commandLineServerConfig{ diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index f55c3ef68a..cf8afbbcdd 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -48,6 +48,7 @@ const ( socketFlag = "socket" remotesapiPortFlag = "remotesapi-port" goldenMysqlConn = "golden" + eventSchedulerStatus = "event-scheduler" ) func indentLines(s string) string { @@ -158,6 +159,7 @@ func (cmd SqlServerCmd) ArgParserWithName(name string) *argparser.ArgParser { ap.SupportsOptionalString(socketFlag, "", "socket file", "Path for the unix socket file. Defaults to '/tmp/mysql.sock'.") ap.SupportsUint(remotesapiPortFlag, "", "remotesapi port", "Sets the port for a server which can expose the databases in this sql-server over remotesapi.") ap.SupportsString(goldenMysqlConn, "", "mysql connection string", "Provides a connection string to a MySQL instance to be used to validate query results") + ap.SupportsString(eventSchedulerStatus, "", "status", "Determines whether the Event Scheduler is enabled and running on the server. It has one of the following values: 'ON', 'OFF' or 'DISABLED'.") return ap } @@ -442,6 +444,15 @@ func getCommandLineServerConfig(apr *argparser.ArgParseResults) (ServerConfig, e serverConfig.withGoldenMysqlConnectionString(connStr) } + if esStatus, ok := apr.GetValue(eventSchedulerStatus); ok { + // make sure to assign eventSchedulerStatus first here + serverConfig.withEventScheduler(strings.ToUpper(esStatus)) + err := sql.SystemVariables.SetGlobal("event_scheduler", serverConfig.EventSchedulerStatus()) + if err != nil { + return nil, fmt.Errorf("failed to set event_scheduler. Error: %s", err.Error()) + } + } + return serverConfig, nil } @@ -475,6 +486,12 @@ func getYAMLServerConfig(fs filesys.Filesys, path string) (ServerConfig, error) return nil, fmt.Errorf("Failed to set net_write_timeout from yaml file '%s'. Error: %s", path, err.Error()) } } + if cfg.BehaviorConfig.EventSchedulerStatus != nil { + err = sql.SystemVariables.SetGlobal("event_scheduler", cfg.EventSchedulerStatus()) + if err != nil { + return nil, fmt.Errorf("Failed to set event_scheduler from yaml file '%s'. Error: %s", path, err.Error()) + } + } return cfg, nil } diff --git a/go/cmd/dolt/commands/sqlserver/yaml_config.go b/go/cmd/dolt/commands/sqlserver/yaml_config.go index 8126443fbd..c0830fea66 100644 --- a/go/cmd/dolt/commands/sqlserver/yaml_config.go +++ b/go/cmd/dolt/commands/sqlserver/yaml_config.go @@ -73,6 +73,8 @@ type BehaviorYAMLConfig struct { // DoltTransactionCommit enables the @@dolt_transaction_commit system variable, which // automatically creates a Dolt commit when any SQL transaction is committed. DoltTransactionCommit *bool `yaml:"dolt_transaction_commit"` + + EventSchedulerStatus *string `yaml:"event_scheduler"` } // UserYAMLConfig contains server configuration regarding the user account clients must use to connect @@ -174,6 +176,7 @@ func serverConfigAsYAMLConfig(cfg ServerConfig) YAMLConfig { strPtr(cfg.PersistenceBehavior()), boolPtr(cfg.DisableClientMultiStatements()), boolPtr(cfg.DoltTransactionCommit()), + strPtr(cfg.EventSchedulerStatus()), }, UserConfig: UserYAMLConfig{strPtr(cfg.User()), strPtr(cfg.Password())}, ListenerConfig: ListenerYAMLConfig{ @@ -515,6 +518,20 @@ func (cfg YAMLConfig) ClusterConfig() cluster.Config { return cfg.ClusterCfg } +func (cfg YAMLConfig) EventSchedulerStatus() string { + if cfg.BehaviorConfig.EventSchedulerStatus == nil { + return "ON" + } + switch *cfg.BehaviorConfig.EventSchedulerStatus { + case "1": + return "ON" + case "0": + return "OFF" + default: + return strings.ToUpper(*cfg.BehaviorConfig.EventSchedulerStatus) + } +} + type ClusterYAMLConfig struct { StandbyRemotes_ []StandbyRemoteYAMLConfig `yaml:"standby_remotes"` BootstrapRole_ string `yaml:"bootstrap_role"` diff --git a/go/go.mod b/go/go.mod index 23b41577ba..5c7e475d3d 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.15.1-0.20230606174340-20dde39da840 + github.com/dolthub/go-mysql-server v0.15.1-0.20230606214426-d45e4317965b github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index d45196d791..10126776b3 100644 --- a/go/go.sum +++ b/go/go.sum @@ -168,8 +168,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.15.1-0.20230606174340-20dde39da840 h1:LNrH1zxCioDcpTMkTV/cr3LI4AlyxrmILI17OUbijUU= -github.com/dolthub/go-mysql-server v0.15.1-0.20230606174340-20dde39da840/go.mod h1:TP8QrAsULBEK7nP0BHRSfZ9l8oiAeaRzIREHVk2wnz0= +github.com/dolthub/go-mysql-server v0.15.1-0.20230606214426-d45e4317965b h1:WFcEp/m19FTok5GgVLNbf3t/aEDlZaQIaXaWRjiZ1+U= +github.com/dolthub/go-mysql-server v0.15.1-0.20230606214426-d45e4317965b/go.mod h1:TP8QrAsULBEK7nP0BHRSfZ9l8oiAeaRzIREHVk2wnz0= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 53cfccfc68..3333acbd66 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -645,7 +645,12 @@ func (db Database) GetRoot(ctx *sql.Context) (*doltdb.RootValue, error) { return nil, fmt.Errorf("no root value found in session") } - return dbState.GetRoots().Working, nil + wr := dbState.GetRoots().Working + if wr == nil { + return nil, doltdb.ErrOperationNotSupportedInDetachedHead + } + + return wr, nil } // GetWorkingSet gets the current working set for the database. @@ -1218,6 +1223,9 @@ func (db Database) GetEvent(ctx *sql.Context, name string) (sql.EventDefinition, Name: frag.name, CreateStatement: frag.fragment, CreatedAt: frag.created, + // TODO: fill LastAltered, it cannot be nil/zero value + LastAltered: frag.created, + // TODO: fill TimezoneOffset and LastExecuted }, true, nil } } @@ -1245,19 +1253,25 @@ func (db Database) GetEvents(ctx *sql.Context) ([]sql.EventDefinition, error) { Name: frag.name, CreateStatement: frag.fragment, CreatedAt: frag.created, + // TODO: fill LastAltered, it cannot be nil/zero value + LastAltered: frag.created, + // TODO: fill TimezoneOffset and LastExecuted + }) } return events, nil } // SaveEvent implements sql.EventDatabase. -func (db Database) SaveEvent(ctx *sql.Context, ed sql.EventDefinition) error { +func (db Database) SaveEvent(ctx *sql.Context, ed sql.EventDetails) error { + evDef := ed.GetEventStorageDefinition() + // TODO: store LastAltered, LastExecuted and TimezoneOffset in appropriate place return db.addFragToSchemasTable(ctx, eventFragment, - ed.Name, - ed.CreateStatement, - ed.CreatedAt, - sql.ErrEventAlreadyExists.New(ed.Name), + evDef.Name, + evDef.CreateStatement, + evDef.CreatedAt, + sql.ErrEventAlreadyExists.New(evDef.Name), ) } @@ -1267,8 +1281,8 @@ func (db Database) DropEvent(ctx *sql.Context, name string) error { } // UpdateEvent implements sql.EventDatabase. -func (db Database) UpdateEvent(ctx *sql.Context, originalName string, ed sql.EventDefinition) error { - // TODO: any EVENT STATUS change should also update the branch-specific event scheduling +func (db Database) UpdateEvent(ctx *sql.Context, originalName string, ed sql.EventDetails) error { + // TODO: only in Dolt, any EVENT STATUS change should also update the branch-specific event scheduling err := db.DropEvent(ctx, originalName) if err != nil { return err @@ -1276,6 +1290,12 @@ func (db Database) UpdateEvent(ctx *sql.Context, originalName string, ed sql.Eve return db.SaveEvent(ctx, ed) } +// UpdateLastExecuted implements sql.EventDatabase +func (db Database) UpdateLastExecuted(ctx *sql.Context, eventName string, lastExecuted time.Time) error { + // TODO: update LastExecuted in appropriate place + return nil +} + // GetStoredProcedure implements sql.StoredProcedureDatabase. func (db Database) GetStoredProcedure(ctx *sql.Context, name string) (sql.StoredProcedureDetails, bool, error) { procedures, err := DoltProceduresGetAll(ctx, db, strings.ToLower(name)) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats new file mode 100644 index 0000000000..109e630647 --- /dev/null +++ b/integration-tests/bats/events.bats @@ -0,0 +1,33 @@ +#!/usr/bin/env bats +load $BATS_TEST_DIRNAME/helper/common.bash +load $BATS_TEST_DIRNAME/helper/query-server-common.bash + +make_repo() { + mkdir "$1" + cd "$1" + dolt init + cd .. +} + +setup() { + skiponwindows "tests are flaky on Windows" + setup_no_dolt_init + make_repo repo1 + make_repo repo2 +} + +teardown() { + stop_sql_server 1 && sleep 0.5 + rm -rf $BATS_TMPDIR/sql-server-test$$ + teardown_common +} + +@test "events: simple insert into table event" { + cd repo1 + dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, c3 int)" + + start_sql_server + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 5 SECOND DO INSERT INTO totals (c3) VALUES (1); SELECT SLEEP(11); SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 3 |" ]] || false +} From 96b9287afe57eb94d88cf9da8b302e959b119262 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Wed, 7 Jun 2023 04:28:16 +0000 Subject: [PATCH 02/45] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/cmd/dolt/commands/sqlserver/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index 17be2d3a35..f5b68d8de6 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -552,4 +552,4 @@ func getEventSchedulerStatus(status string) (eventscheduler.SchedulerStatus, err default: return eventscheduler.SchedulerDisabled, fmt.Errorf("Error while setting value '%s' to 'event_scheduler'.", status) } -} \ No newline at end of file +} From 9604746dbb486d5152936680fd96e55df8a5c04f Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 8 Jun 2023 11:24:00 -0700 Subject: [PATCH 03/45] fix test --- go/cmd/dolt/commands/sqlserver/yaml_config_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/cmd/dolt/commands/sqlserver/yaml_config_test.go b/go/cmd/dolt/commands/sqlserver/yaml_config_test.go index ce573a3b35..ed70adba96 100644 --- a/go/cmd/dolt/commands/sqlserver/yaml_config_test.go +++ b/go/cmd/dolt/commands/sqlserver/yaml_config_test.go @@ -36,6 +36,7 @@ behavior: dolt_transaction_commit: true persistence_behavior: load disable_client_multi_statements: false + event_scheduler: ON user: name: "" From 812edb2ec7aa2c382da13254346b15f81007d407 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Fri, 9 Jun 2023 21:16:37 -0700 Subject: [PATCH 04/45] fix --- go/libraries/doltcore/sqle/database.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index b721c2dfd5..ed40fd6a5c 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -22,6 +22,12 @@ import ( "strings" "time" + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/parse" + "github.com/dolthub/go-mysql-server/sql/plan" + "github.com/dolthub/go-mysql-server/sql/types" + "gopkg.in/src-d/go-errors.v1" + "github.com/dolthub/dolt/go/libraries/doltcore/branch_control" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" @@ -34,10 +40,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/parse" - "github.com/dolthub/go-mysql-server/sql/plan" - "github.com/dolthub/go-mysql-server/sql/types" ) var ErrInvalidTableName = errors.NewKind("Invalid table name %s. Table names must match the regular expression " + doltdb.TableNameRegexStr) From 97184df845b39ba438b14e740287b5bb4a813827 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Wed, 14 Jun 2023 17:19:21 -0700 Subject: [PATCH 05/45] pass get ctx function and commit transaction function to event scheduler --- go/cmd/dolt/commands/engine/sqlengine.go | 29 ++++++-- go/libraries/doltcore/sqle/database.go | 4 +- integration-tests/bats/events.bats | 86 +++++++++++++++++++++++- 3 files changed, 112 insertions(+), 7 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 5962be80eb..17cffc8378 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -290,19 +290,38 @@ func configureBinlogReplicaController(config *SqlEngineConfig, engine *gms.Engin func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFactory sessionFactory, pro dsqle.DoltDatabaseProvider) error { // need to give correct user, use the definer as user to run the event definition queries ctxFactory := sqlContextFactory() - // get new session each time query executes? - getCtxFunc := func() (*sql.Context, error) { + + // getCtxFunc is used to create new session context for event scheduler. + // It starts a transaction that needs to be committed using the function returned. + getCtxFunc := func() (*sql.Context, func() error, error) { sess, err := sessFactory(sql.NewBaseSession(), pro) if err != nil { - return nil, err + return nil, func() error {return nil}, err } newCtx, err := ctxFactory(context.Background(), sess) if err != nil { - return nil, err + return nil, func() error {return nil}, err } - return newCtx, nil + err = sess.SetSessionVariable(newCtx, sql.AutoCommitSessionVar, true) + if err != nil { + return nil, func() error {return nil}, err + } + + tr, err := sess.StartTransaction(newCtx, sql.ReadWrite) + if err != nil { + return nil, func() error {return nil}, err + } + + ts, ok := newCtx.Session.(sql.TransactionSession) + if !ok { + return nil, func() error {return nil}, nil + } + + return newCtx, func() error { + return ts.CommitTransaction(newCtx, tr) + }, nil } return engine.InitializeEventScheduler(getCtxFunc, config.EventSchedulerStatus) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 59db2e9f96..6fe00c9f61 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1288,6 +1288,7 @@ func (db Database) GetEvents(ctx *sql.Context) ([]sql.EventDefinition, error) { // SaveEvent implements sql.EventDatabase. func (db Database) SaveEvent(ctx *sql.Context, ed sql.EventDetails) error { + // TODO: check if the db revision name is default branch name, if not always DISABLE the event. evDef := ed.GetEventStorageDefinition() // TODO: store LastAltered, LastExecuted and TimezoneOffset in appropriate place return db.addFragToSchemasTable(ctx, @@ -1306,7 +1307,8 @@ func (db Database) DropEvent(ctx *sql.Context, name string) error { // UpdateEvent implements sql.EventDatabase. func (db Database) UpdateEvent(ctx *sql.Context, originalName string, ed sql.EventDetails) error { - // TODO: only in Dolt, any EVENT STATUS change should also update the branch-specific event scheduling + // TODO: check if the db revision name is default branch name, if not always DISABLE the event. + // TODO: any EVENT STATUS change should also update the branch-specific event scheduling err := db.DropEvent(ctx, originalName) if err != nil { return err diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index f4c1ac5369..18754168d4 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -27,7 +27,7 @@ teardown() { dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" start_sql_server - run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 5 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(11); SELECT COUNT(*) FROM totals;" + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 3 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 3 |" ]] || false } @@ -42,3 +42,87 @@ teardown() { [ $status -eq 0 ] [[ $output =~ "DISABLED" ]] || false } + +@test "events: disabling current_timestamp one time event after execution" { + cd repo1 + dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" + + start_sql_server + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert9 ON SCHEDULE AT CURRENT_TIMESTAMP DO INSERT INTO totals (int_col) VALUES (9);" + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 1 |" ]] || false + + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM information_schema.events;" + [ $status -eq 0 ] + [[ $output =~ "| 0 |" ]] || false + + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert8 ON SCHEDULE AT CURRENT_TIMESTAMP ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (8);" + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 2 |" ]] || false + + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SHOW CREATE EVENT insert8;" + [ $status -eq 0 ] + [[ $output =~ "ON COMPLETION PRESERVE DISABLE" ]] || false + + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "ALTER EVENT insert8 ON COMPLETION NOT PRESERVE; SELECT COUNT(*) FROM information_schema.events;" + [ $status -eq 0 ] + [[ $output =~ "| 1 |" ]] || false + + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "ALTER EVENT insert8 ENABLE; SELECT COUNT(*) FROM information_schema.events;" + [ $status -eq 0 ] + [[ $output =~ "| 0 |" ]] || false +} + +@test "events: disabling future one time event after execution from the scheduler" { + cd repo1 + dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" + + start_sql_server + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert9 ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 3 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (9); SHOW CREATE EVENT insert9;" + [ $status -eq 0 ] + [[ $output =~ "ON COMPLETION PRESERVE ENABLE" ]] || false + + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(4); SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 1 |" ]] || false + + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM information_schema.events;" + [ $status -eq 0 ] + [[ $output =~ "| 1 |" ]] || false + + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SHOW CREATE EVENT insert9;" + [ $status -eq 0 ] + [[ $output =~ "ON COMPLETION PRESERVE DISABLE" ]] || false +} + +@test "events: recurring event with STARTS and ENDS defined" { + cd repo1 + dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" + + start_sql_server + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 2 |" ]] || false + + # should be dropped + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM information_schema.events;" + [ $status -eq 0 ] + [[ $output =~ "| 0 |" ]] || false +} + +@test "events: recurring event with ENDS defined" { + cd repo1 + dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" + + start_sql_server + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 3 |" ]] || false + + # should be disabled + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT * FROM information_schema.events;" + [ $status -eq 0 ] + [[ $output =~ "DISABLED" ]] || false +} From 45c432810d922831b3a3ee50a5c1df5385adbbb8 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 15 Jun 2023 00:38:00 +0000 Subject: [PATCH 06/45] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/cmd/dolt/commands/engine/sqlengine.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 17cffc8378..bbe9b09c0e 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -296,27 +296,27 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa getCtxFunc := func() (*sql.Context, func() error, error) { sess, err := sessFactory(sql.NewBaseSession(), pro) if err != nil { - return nil, func() error {return nil}, err + return nil, func() error { return nil }, err } newCtx, err := ctxFactory(context.Background(), sess) if err != nil { - return nil, func() error {return nil}, err + return nil, func() error { return nil }, err } err = sess.SetSessionVariable(newCtx, sql.AutoCommitSessionVar, true) if err != nil { - return nil, func() error {return nil}, err + return nil, func() error { return nil }, err } tr, err := sess.StartTransaction(newCtx, sql.ReadWrite) if err != nil { - return nil, func() error {return nil}, err + return nil, func() error { return nil }, err } ts, ok := newCtx.Session.(sql.TransactionSession) if !ok { - return nil, func() error {return nil}, nil + return nil, func() error { return nil }, nil } return newCtx, func() error { From 1c0af9750c7829e4aa4d0b5fae0505c4a4272a8a Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 15 Jun 2023 13:01:49 -0700 Subject: [PATCH 07/45] skip replication error for event scheduler ctxGetter function --- go/cmd/dolt/commands/engine/sqlengine.go | 14 ++++++++++++-- integration-tests/bats/events.bats | 9 ++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index bbe9b09c0e..4c0428928a 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -304,12 +304,15 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa return nil, func() error { return nil }, err } - err = sess.SetSessionVariable(newCtx, sql.AutoCommitSessionVar, true) + // Starting transaction at the start of the server to load all events to + // the event scheduler is causing read replica test to fail as it tries + // pulling from remotes for branches that its remoteRef does not exist yet. + err = sql.SystemVariables.SetGlobal(dsess.SkipReplicationErrors, true) if err != nil { return nil, func() error { return nil }, err } - tr, err := sess.StartTransaction(newCtx, sql.ReadWrite) + err = sess.SetSessionVariable(newCtx, sql.AutoCommitSessionVar, true) if err != nil { return nil, func() error { return nil }, err } @@ -319,6 +322,13 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa return nil, func() error { return nil }, nil } + tr, err := sess.StartTransaction(newCtx, sql.ReadWrite) + if err != nil { + return nil, func() error { return nil }, err + } + + ts.SetTransaction(tr) + return newCtx, func() error { return ts.CommitTransaction(newCtx, tr) }, nil diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index 18754168d4..dd76871cb4 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -27,7 +27,8 @@ teardown() { dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" start_sql_server - run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 3 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 3 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7); DROP EVENT insert1;" + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 3 |" ]] || false } @@ -102,7 +103,8 @@ teardown() { dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" start_sql_server - run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND DO INSERT INTO totals (int_col) VALUES (1);" + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 2 |" ]] || false @@ -117,7 +119,8 @@ teardown() { dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" start_sql_server - run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (1);" + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 3 |" ]] || false From 5d8729f413efe8942014fe9fcd0f84946bbaa0f8 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 15 Jun 2023 14:51:30 -0700 Subject: [PATCH 08/45] use sleep --- integration-tests/bats/events.bats | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index dd76871cb4..d86edb27df 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -28,6 +28,7 @@ teardown() { start_sql_server dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 3 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7); DROP EVENT insert1;" + sleep 2 run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 3 |" ]] || false @@ -85,7 +86,8 @@ teardown() { [ $status -eq 0 ] [[ $output =~ "ON COMPLETION PRESERVE ENABLE" ]] || false - run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(4); SELECT COUNT(*) FROM totals;" + sleep 4 + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 1 |" ]] || false @@ -103,8 +105,9 @@ teardown() { dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" start_sql_server - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND DO INSERT INTO totals (int_col) VALUES (1);" - run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7);" + sleep 2 + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 2 |" ]] || false @@ -119,10 +122,11 @@ teardown() { dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" start_sql_server - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (1);" - run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(7); SELECT COUNT(*) FROM totals;" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 3 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(5);" + sleep 2 + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] - [[ $output =~ "| 3 |" ]] || false + [[ $output =~ "| 2 |" ]] || false # should be disabled run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT * FROM information_schema.events;" From 554db04f75e734ee4f009570752d3f1cf90c7be3 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Tue, 20 Jun 2023 10:03:32 -0700 Subject: [PATCH 09/45] check bats test --- integration-tests/bats/events.bats | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index d86edb27df..6742fd5658 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -27,11 +27,12 @@ teardown() { dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" start_sql_server - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 3 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7); DROP EVENT insert1;" - sleep 2 + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 5 SECOND DO INSERT INTO totals (int_col) VALUES (1);" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(7);" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "DROP EVENT insert1;" run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] - [[ $output =~ "| 3 |" ]] || false + [[ $output =~ "| 2 |" ]] || false } @test "events: disabling recurring event with ends not defined should not be dropped" { @@ -86,7 +87,7 @@ teardown() { [ $status -eq 0 ] [[ $output =~ "ON COMPLETION PRESERVE ENABLE" ]] || false - sleep 4 + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(4);" run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 1 |" ]] || false @@ -105,8 +106,8 @@ teardown() { dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" start_sql_server - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(7);" - sleep 2 + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND DO INSERT INTO totals (int_col) VALUES (1);" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(7);" run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 2 |" ]] || false From 24e92c4a49b87fa0fb34f3cf4e442d73d7c6a479 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 22 Jun 2023 17:41:55 -0700 Subject: [PATCH 10/45] load events from only default branch for event scheduler and not notify the event scheduler if event is added on non-default branch --- go/cmd/dolt/commands/engine/sqlengine.go | 10 +++++ go/libraries/doltcore/sqle/database.go | 37 +++++++++++++---- integration-tests/bats/events.bats | 51 ++++++++++++++++++++---- 3 files changed, 83 insertions(+), 15 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 4c0428928a..9eb923f5d5 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -317,6 +317,16 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa return nil, func() error { return nil }, err } + // set dolt_show_branch_databases to 'true' to get all revision databases + // when accessing all databases. When retrieving events from each database, + // only the default branch database will return events that have 'ENABLE' status. + // All other databases will return events with 'DISABLE' status regardless of the + // stored status. + err = sess.SetSessionVariable(newCtx, dsess.ShowBranchDatabases, true) + if err != nil { + return nil, func() error { return nil }, err + } + ts, ok := newCtx.Session.(sql.TransactionSession) if !ok { return nil, func() error { return nil }, nil diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 8101adcb69..ae82f1206e 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1245,7 +1245,7 @@ func (db Database) GetEvent(ctx *sql.Context, name string) (sql.EventDefinition, if strings.ToLower(frag.name) == strings.ToLower(name) { return sql.EventDefinition{ Name: frag.name, - CreateStatement: frag.fragment, + CreateStatement: updateEventStatusTemporarilyForNonDefaultBranch(db.revision, frag.fragment), CreatedAt: frag.created, // TODO: fill LastAltered, it cannot be nil/zero value LastAltered: frag.created, @@ -1275,7 +1275,7 @@ func (db Database) GetEvents(ctx *sql.Context) ([]sql.EventDefinition, error) { for _, frag := range frags { events = append(events, sql.EventDefinition{ Name: frag.name, - CreateStatement: frag.fragment, + CreateStatement: updateEventStatusTemporarilyForNonDefaultBranch(db.revision, frag.fragment), CreatedAt: frag.created, // TODO: fill LastAltered, it cannot be nil/zero value LastAltered: frag.created, @@ -1287,11 +1287,22 @@ func (db Database) GetEvents(ctx *sql.Context) ([]sql.EventDefinition, error) { } // SaveEvent implements sql.EventDatabase. -func (db Database) SaveEvent(ctx *sql.Context, ed sql.EventDetails) error { - // TODO: check if the db revision name is default branch name, if not always DISABLE the event. +func (db Database) SaveEvent(ctx *sql.Context, ed sql.EventDetails) (bool, error) { + // If the database is not the default branch database, then the event is disabled. + // TODO: need better way to determine the default branch; currently it checks only 'main' + if db.revision != env.DefaultInitBranch && ed.Status == sql.EventStatus_Enable.String() { + // using revision database name + ed.Status = sql.EventStatus_Disable.String() + ctx.Session.Warn(&sql.Warning{ + Level: "Warning", + Code: 1105, + Message: fmt.Sprintf("Event status cannot be enabled for revision database."), + }) + } + evDef := ed.GetEventStorageDefinition() // TODO: store LastAltered, LastExecuted and TimezoneOffset in appropriate place - return db.addFragToSchemasTable(ctx, + return ed.Status == sql.EventStatus_Enable.String(), db.addFragToSchemasTable(ctx, eventFragment, evDef.Name, evDef.CreateStatement, @@ -1306,12 +1317,11 @@ func (db Database) DropEvent(ctx *sql.Context, name string) error { } // UpdateEvent implements sql.EventDatabase. -func (db Database) UpdateEvent(ctx *sql.Context, originalName string, ed sql.EventDetails) error { - // TODO: check if the db revision name is default branch name, if not always DISABLE the event. +func (db Database) UpdateEvent(ctx *sql.Context, originalName string, ed sql.EventDetails) (bool, error) { // TODO: any EVENT STATUS change should also update the branch-specific event scheduling err := db.DropEvent(ctx, originalName) if err != nil { - return err + return false, err } return db.SaveEvent(ctx, ed) } @@ -1322,6 +1332,17 @@ func (db Database) UpdateLastExecuted(ctx *sql.Context, eventName string, lastEx return nil } +// updateEventStatusTemporarilyForNonDefaultBranch updates the event status from ENABLE to DISABLE if it's not default branch. +// The event status metadata is not updated in storage, but only for display purposes we return event status as 'DISABLE'. +// This function is used temporarily to implement logic of only allowing enabled events to be executed on default branch. +func updateEventStatusTemporarilyForNonDefaultBranch(defaultBranch, createStmt string) string { + // TODO: need better way to determine the default branch; currently it checks only 'main' + if defaultBranch == env.DefaultInitBranch { + return createStmt + } + return strings.Replace(createStmt, "ENABLE", "DISABLE", 1) +} + // GetStoredProcedure implements sql.StoredProcedureDatabase. func (db Database) GetStoredProcedure(ctx *sql.Context, name string) (sql.StoredProcedureDetails, bool, error) { procedures, err := DoltProceduresGetAll(ctx, db, strings.ToLower(name)) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index 6742fd5658..f8d5e55f9f 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -13,7 +13,6 @@ setup() { skiponwindows "tests are flaky on Windows" setup_no_dolt_init make_repo repo1 - make_repo repo2 } teardown() { @@ -27,15 +26,13 @@ teardown() { dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" start_sql_server - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 5 SECOND DO INSERT INTO totals (int_col) VALUES (1);" - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(7);" - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "DROP EVENT insert1;" - run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 1 DAY DO INSERT INTO totals (int_col) VALUES (1);" + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "ALTER EVENT insert1 DISABLE; SELECT * FROM information_schema.events;" [ $status -eq 0 ] - [[ $output =~ "| 2 |" ]] || false + [[ $output =~ "DISABLED" ]] || false } -@test "events: disabling recurring event with ends not defined should not be dropped" { +@test "events: disabling recurring event should not be dropped" { cd repo1 dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" @@ -134,3 +131,43 @@ teardown() { [ $status -eq 0 ] [[ $output =~ "DISABLED" ]] || false } + +@test "events: checking out a branch should disable all events leaving the working set dirty" { + cd repo1 + dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" + dolt add . + dolt commit -m "create table" + + start_sql_server + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 3 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(5);" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_COMMIT('-am','commit with an event')" + + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_CHECKOUT('-b','newbranch')" + # should be disabled + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT * FROM information_schema.events;" + [ $status -eq 0 ] + [[ $output =~ "DISABLED" ]] || false +} + +@test "events: events on default branch still runs after switching to non default branch" { + cd repo1 + dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" + dolt add . + dolt commit -m "create table" + + start_sql_server + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 5 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 1 MINUTE DO INSERT INTO totals (int_col) VALUES (1);" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_COMMIT('-am','commit with an event')" + + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_CHECKOUT('-b','newbranch')" + # should be disabled + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 1 |" ]] || false + + sleep 5 + # should not be 1 + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_CHECKOUT('main'); SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ ! $output =~ "| 1 |" ]] || false +} From 58d630a752f6c866af08d50a8c004134c67cf650 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Fri, 23 Jun 2023 14:45:35 -0700 Subject: [PATCH 11/45] skip clusterdb for revision access --- go/libraries/doltcore/sqle/clusterdb/database.go | 4 +++- go/libraries/doltcore/sqle/database_provider.go | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/clusterdb/database.go b/go/libraries/doltcore/sqle/clusterdb/database.go index 09188acb93..62b2824a7f 100644 --- a/go/libraries/doltcore/sqle/clusterdb/database.go +++ b/go/libraries/doltcore/sqle/clusterdb/database.go @@ -28,6 +28,8 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" ) +const DoltClusterDbName = "dolt_cluster" + type database struct { statusProvider ClusterStatusProvider } @@ -38,7 +40,7 @@ var _ dsess.SqlDatabase = database{} const StatusTableName = "dolt_cluster_status" func (database) Name() string { - return "dolt_cluster" + return DoltClusterDbName } func (db database) GetTableInsensitive(ctx *sql.Context, tblName string) (sql.Table, bool, error) { diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 100f35bf57..0187572b94 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -17,6 +17,7 @@ package sqle import ( "context" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/clusterdb" "sort" "strings" "sync" @@ -267,7 +268,7 @@ func (p DoltDatabaseProvider) AllDatabases(ctx *sql.Context) (all []sql.Database for _, db := range p.databases { all = append(all, db) - if showBranches { + if showBranches && db.Name() != clusterdb.DoltClusterDbName { revisionDbs, err := p.allRevisionDbs(ctx, db) if err != nil { // TODO: this interface is wrong, needs to return errors From c2f422f4e60ff3a178f35a32a2f538abe9348934 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Fri, 23 Jun 2023 14:54:22 -0700 Subject: [PATCH 12/45] try --- integration-tests/bats/events.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index f8d5e55f9f..72d7ccfdf7 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -104,7 +104,7 @@ teardown() { start_sql_server dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND DO INSERT INTO totals (int_col) VALUES (1);" - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(7);" + sleep 10 run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 2 |" ]] || false From b7f8593cc999d4b2d484b2e8a4bf77dfd4b33838 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Fri, 23 Jun 2023 22:03:24 +0000 Subject: [PATCH 13/45] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/database_provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 5e7cfa32d5..4e9389a571 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -17,7 +17,6 @@ package sqle import ( "context" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/clusterdb" "sort" "strings" "sync" @@ -31,6 +30,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/clusterdb" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dprocedures" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" From 6bb297b7f3a69967663a456549a004602bf9e8b6 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 9 Aug 2023 09:29:22 -0700 Subject: [PATCH 14/45] Latest GMS build from tip of jennifer/event-execution dev branch --- go/go.mod | 2 +- go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index 70827fc757..faf3d4a15d 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.16.1-0.20230808195654-505d461943f7 + github.com/dolthub/go-mysql-server v0.16.1-0.20230808175755-1ac374e90cbe github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index 0360311e1d..1c1d1d63e4 100644 --- a/go/go.sum +++ b/go/go.sum @@ -180,6 +180,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= +github.com/dolthub/go-mysql-server v0.16.1-0.20230808175755-1ac374e90cbe h1:s9TWCIanvB2ntlFbagSeOoWOSQscjmsNYUTdxsMKmuM= +github.com/dolthub/go-mysql-server v0.16.1-0.20230808175755-1ac374e90cbe/go.mod h1:nY1J1sV2kuGJbAZ6bcZARw4dF8TD3KpEfYVVs/HK/JY= github.com/dolthub/go-mysql-server v0.16.1-0.20230808195654-505d461943f7 h1:3Pz7dm++VwHb6YvooMCzKEX58JAEqWxOU3EjD3g3Ehs= github.com/dolthub/go-mysql-server v0.16.1-0.20230808195654-505d461943f7/go.mod h1:nY1J1sV2kuGJbAZ6bcZARw4dF8TD3KpEfYVVs/HK/JY= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= From ed80f43f035f7c28f66ee83d8e94d728b4dc9b70 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Fri, 25 Aug 2023 11:21:33 -0700 Subject: [PATCH 15/45] Fixing errors after last merge from main --- go/cmd/dolt/commands/sqlserver/server.go | 12 +++++++----- go/cmd/dolt/commands/sqlserver/sqlserver.go | 4 ++-- go/go.mod | 2 +- go/go.sum | 8 ++------ 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index a78815713f..d23bc8c472 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -114,12 +114,14 @@ func Serve( logrus.TraceLevel.String(), ), Default: logrus.GetLevel().String(), - NotifyChanged: func(scope sql.SystemVariableScope, v sql.SystemVarValue) { - if level, err := logrus.ParseLevel(v.Val.(string)); err == nil { - logrus.SetLevel(level) - } else { - logrus.Warnf("could not parse requested log level %s as a log level. dolt_log_level variable value and logging behavior will diverge.", v.Val.(string)) + NotifyChanged: func(scope sql.SystemVariableScope, v sql.SystemVarValue) error { + level, err := logrus.ParseLevel(v.Val.(string)) + if err != nil { + return fmt.Errorf("could not parse requested log level %s as a log level. dolt_log_level variable value and logging behavior will diverge.", v.Val.(string)) } + + logrus.SetLevel(level) + return nil }, }, }) diff --git a/go/cmd/dolt/commands/sqlserver/sqlserver.go b/go/cmd/dolt/commands/sqlserver/sqlserver.go index ecec67446c..204b6c01c2 100644 --- a/go/cmd/dolt/commands/sqlserver/sqlserver.go +++ b/go/cmd/dolt/commands/sqlserver/sqlserver.go @@ -492,8 +492,8 @@ func getCommandLineConfig(creds *cli.UserPassword, apr *argparser.ArgParseResult if esStatus, ok := apr.GetValue(eventSchedulerStatus); ok { // make sure to assign eventSchedulerStatus first here - serverConfig.withEventScheduler(strings.ToUpper(esStatus)) - err := sql.SystemVariables.SetGlobal("event_scheduler", serverConfig.EventSchedulerStatus()) + config.withEventScheduler(strings.ToUpper(esStatus)) + err := sql.SystemVariables.SetGlobal("event_scheduler", config.EventSchedulerStatus()) if err != nil { return nil, fmt.Errorf("failed to set event_scheduler. Error: %s", err.Error()) } diff --git a/go/go.mod b/go/go.mod index faf3d4a15d..664d1195f3 100644 --- a/go/go.mod +++ b/go/go.mod @@ -15,7 +15,7 @@ require ( github.com/dolthub/fslock v0.0.3 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 - github.com/dolthub/vitess v0.0.0-20230803210631-6a8ca1536779 + github.com/dolthub/vitess v0.0.0-20230823204737-4a21a94e90c3 github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.13.0 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 diff --git a/go/go.sum b/go/go.sum index 1c1d1d63e4..eacd0b11b0 100644 --- a/go/go.sum +++ b/go/go.sum @@ -180,10 +180,6 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.16.1-0.20230808175755-1ac374e90cbe h1:s9TWCIanvB2ntlFbagSeOoWOSQscjmsNYUTdxsMKmuM= -github.com/dolthub/go-mysql-server v0.16.1-0.20230808175755-1ac374e90cbe/go.mod h1:nY1J1sV2kuGJbAZ6bcZARw4dF8TD3KpEfYVVs/HK/JY= -github.com/dolthub/go-mysql-server v0.16.1-0.20230808195654-505d461943f7 h1:3Pz7dm++VwHb6YvooMCzKEX58JAEqWxOU3EjD3g3Ehs= -github.com/dolthub/go-mysql-server v0.16.1-0.20230808195654-505d461943f7/go.mod h1:nY1J1sV2kuGJbAZ6bcZARw4dF8TD3KpEfYVVs/HK/JY= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= @@ -194,8 +190,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= -github.com/dolthub/vitess v0.0.0-20230803210631-6a8ca1536779 h1:/ru4ji1X6Fj350rnYhKVZE1qClbKNS6hqYQNgiNanQk= -github.com/dolthub/vitess v0.0.0-20230803210631-6a8ca1536779/go.mod h1:IyoysiiOzrIs7QsEHC+yVF0yRQ6W70GXyCXqtI2vVTs= +github.com/dolthub/vitess v0.0.0-20230823204737-4a21a94e90c3 h1:lY3oQbYNMSVjT02n6f2M2H0u4icF6lGbS/IpWr27ti8= +github.com/dolthub/vitess v0.0.0-20230823204737-4a21a94e90c3/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= From 019dfbf01391671d08e570de36408e5b90d93abc Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Fri, 25 Aug 2023 11:26:39 -0700 Subject: [PATCH 16/45] Update go/libraries/doltcore/sqle/database.go --- go/libraries/doltcore/sqle/database.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 9c1ebcd00b..744f111c13 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1275,7 +1275,6 @@ func (db Database) GetEvent(ctx *sql.Context, name string) (sql.EventDefinition, Name: frag.name, CreateStatement: updateEventStatusTemporarilyForNonDefaultBranch(db.revision, frag.fragment), CreatedAt: frag.created, - // TODO: fill LastAltered, it cannot be nil/zero value LastAltered: frag.created, // TODO: fill TimezoneOffset and LastExecuted }, true, nil From 97d03dd2fdf5e812671166ceeb7c97bc95750d03 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Fri, 25 Aug 2023 11:26:49 -0700 Subject: [PATCH 17/45] Update go/libraries/doltcore/sqle/database.go --- go/libraries/doltcore/sqle/database.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 744f111c13..ecaab16ebf 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1304,7 +1304,6 @@ func (db Database) GetEvents(ctx *sql.Context) ([]sql.EventDefinition, error) { Name: frag.name, CreateStatement: updateEventStatusTemporarilyForNonDefaultBranch(db.revision, frag.fragment), CreatedAt: frag.created, - // TODO: fill LastAltered, it cannot be nil/zero value LastAltered: frag.created, // TODO: fill TimezoneOffset and LastExecuted From 9c3e2f3b4622eacaba2d6310b66c6043f245610e Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Fri, 25 Aug 2023 11:46:52 -0700 Subject: [PATCH 18/45] Tidying up BATS - tests are still flaky though --- integration-tests/bats/events.bats | 53 ++++-------------------------- 1 file changed, 6 insertions(+), 47 deletions(-) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index 72d7ccfdf7..960f4f492d 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -2,17 +2,20 @@ load $BATS_TEST_DIRNAME/helper/common.bash load $BATS_TEST_DIRNAME/helper/query-server-common.bash -make_repo() { +make_test_repo_and_start_server() { mkdir "$1" cd "$1" dolt init + dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" + dolt commit -Am "create table" + start_sql_server cd .. } setup() { skiponwindows "tests are flaky on Windows" setup_no_dolt_init - make_repo repo1 + make_test_repo_and_start_server repo1 } teardown() { @@ -21,22 +24,7 @@ teardown() { teardown_common } -@test "events: simple insert into table event" { - cd repo1 - dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" - - start_sql_server - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 1 DAY DO INSERT INTO totals (int_col) VALUES (1);" - run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "ALTER EVENT insert1 DISABLE; SELECT * FROM information_schema.events;" - [ $status -eq 0 ] - [[ $output =~ "DISABLED" ]] || false -} - @test "events: disabling recurring event should not be dropped" { - cd repo1 - dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" - - start_sql_server dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 1 DAY DO INSERT INTO totals (int_col) VALUES (1);" run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "ALTER EVENT insert1 DISABLE; SELECT * FROM information_schema.events;" [ $status -eq 0 ] @@ -44,10 +32,6 @@ teardown() { } @test "events: disabling current_timestamp one time event after execution" { - cd repo1 - dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" - - start_sql_server dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert9 ON SCHEDULE AT CURRENT_TIMESTAMP DO INSERT INTO totals (int_col) VALUES (9);" run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] @@ -76,15 +60,11 @@ teardown() { } @test "events: disabling future one time event after execution from the scheduler" { - cd repo1 - dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" - - start_sql_server run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert9 ON SCHEDULE AT CURRENT_TIMESTAMP + INTERVAL 3 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (9); SHOW CREATE EVENT insert9;" [ $status -eq 0 ] [[ $output =~ "ON COMPLETION PRESERVE ENABLE" ]] || false - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT SLEEP(4);" + sleep 4 run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 1 |" ]] || false @@ -99,10 +79,6 @@ teardown() { } @test "events: recurring event with STARTS and ENDS defined" { - cd repo1 - dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" - - start_sql_server dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 5 SECOND DO INSERT INTO totals (int_col) VALUES (1);" sleep 10 run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" @@ -116,10 +92,6 @@ teardown() { } @test "events: recurring event with ENDS defined" { - cd repo1 - dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" - - start_sql_server dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 3 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(5);" sleep 2 run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" @@ -133,12 +105,6 @@ teardown() { } @test "events: checking out a branch should disable all events leaving the working set dirty" { - cd repo1 - dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" - dolt add . - dolt commit -m "create table" - - start_sql_server dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 3 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(5);" dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_COMMIT('-am','commit with an event')" @@ -150,12 +116,6 @@ teardown() { } @test "events: events on default branch still runs after switching to non default branch" { - cd repo1 - dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" - dolt add . - dolt commit -m "create table" - - start_sql_server dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 5 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 1 MINUTE DO INSERT INTO totals (int_col) VALUES (1);" dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_COMMIT('-am','commit with an event')" @@ -166,7 +126,6 @@ teardown() { [[ $output =~ "| 1 |" ]] || false sleep 5 - # should not be 1 run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_CHECKOUT('main'); SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ ! $output =~ "| 1 |" ]] || false From 98c88e12d636b1cf50ea5ad3a83518c42f8e3e82 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Fri, 25 Aug 2023 14:12:45 -0700 Subject: [PATCH 19/45] Commenting out some customizations and sys var settings to see what breaks --- go/cmd/dolt/commands/engine/sqlengine.go | 43 +++++++++++++----------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 9eb923f5d5..06a5dde5d9 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -304,28 +304,31 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa return nil, func() error { return nil }, err } - // Starting transaction at the start of the server to load all events to - // the event scheduler is causing read replica test to fail as it tries - // pulling from remotes for branches that its remoteRef does not exist yet. - err = sql.SystemVariables.SetGlobal(dsess.SkipReplicationErrors, true) - if err != nil { - return nil, func() error { return nil }, err - } + // TODO: Seems like we either need to fix this test or change this behavior to + // not start a transaction until right before we need one. + //// Starting transaction at the start of the server to load all events to + //// the event scheduler is causing read replica test to fail as it tries + //// pulling from remotes for branches that its remoteRef does not exist yet. + //err = sql.SystemVariables.SetGlobal(dsess.SkipReplicationErrors, true) + //if err != nil { + // return nil, func() error { return nil }, err + //} - err = sess.SetSessionVariable(newCtx, sql.AutoCommitSessionVar, true) - if err != nil { - return nil, func() error { return nil }, err - } + // TODO: This should be enabled by default, plus it looks like we're calling commit anyway? + //err = sess.SetSessionVariable(newCtx, sql.AutoCommitSessionVar, true) + //if err != nil { + // return nil, func() error { return nil }, err + //} - // set dolt_show_branch_databases to 'true' to get all revision databases - // when accessing all databases. When retrieving events from each database, - // only the default branch database will return events that have 'ENABLE' status. - // All other databases will return events with 'DISABLE' status regardless of the - // stored status. - err = sess.SetSessionVariable(newCtx, dsess.ShowBranchDatabases, true) - if err != nil { - return nil, func() error { return nil }, err - } + //// set dolt_show_branch_databases to 'true' to get all revision databases + //// when accessing all databases. When retrieving events from each database, + //// only the default branch database will return events that have 'ENABLE' status. + //// All other databases will return events with 'DISABLE' status regardless of the + //// stored status. + //err = sess.SetSessionVariable(newCtx, dsess.ShowBranchDatabases, true) + //if err != nil { + // return nil, func() error { return nil }, err + //} ts, ok := newCtx.Session.(sql.TransactionSession) if !ok { From 9f9c1058d0248ebacc0547d05a25880eea7d1a5e Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Fri, 1 Sep 2023 14:28:38 -0700 Subject: [PATCH 20/45] Tidying up --- go/cmd/dolt/commands/sqlserver/server.go | 1 - go/libraries/doltcore/sqle/database.go | 24 +++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/server.go b/go/cmd/dolt/commands/sqlserver/server.go index fa00412a6e..4e5b54a574 100644 --- a/go/cmd/dolt/commands/sqlserver/server.go +++ b/go/cmd/dolt/commands/sqlserver/server.go @@ -270,7 +270,6 @@ func Serve( var remoteSrv *remotesrv.Server if serverConfig.RemotesapiPort() != nil { - port := *serverConfig.RemotesapiPort() if remoteSrvSqlCtx, err := sqlEngine.NewDefaultContext(ctx); err == nil { listenaddr := fmt.Sprintf(":%d", port) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index e222639dbf..89ff5cce97 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -22,8 +22,11 @@ import ( "strings" "time" + sqle "github.com/dolthub/go-mysql-server" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/fulltext" + "github.com/dolthub/go-mysql-server/sql/plan" + "github.com/dolthub/go-mysql-server/sql/planbuilder" "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/vitess/go/vt/sqlparser" "gopkg.in/src-d/go-errors.v1" @@ -1321,12 +1324,12 @@ func (db Database) GetEvents(ctx *sql.Context) ([]sql.EventDefinition, error) { } // SaveEvent implements sql.EventDatabase. -func (db Database) SaveEvent(ctx *sql.Context, ed sql.EventDetails) (bool, error) { +func (db Database) SaveEvent(ctx *sql.Context, event sql.EventDefinition) (bool, error) { // If the database is not the default branch database, then the event is disabled. // TODO: need better way to determine the default branch; currently it checks only 'main' - if db.revision != env.DefaultInitBranch && ed.Status == sql.EventStatus_Enable.String() { + if db.revision != env.DefaultInitBranch && event.Status == sql.EventStatus_Enable.String() { // using revision database name - ed.Status = sql.EventStatus_Disable.String() + event.Status = sql.EventStatus_Disable.String() ctx.Session.Warn(&sql.Warning{ Level: "Warning", Code: 1105, @@ -1334,14 +1337,13 @@ func (db Database) SaveEvent(ctx *sql.Context, ed sql.EventDetails) (bool, error }) } - evDef := ed.GetEventStorageDefinition() // TODO: store LastAltered, LastExecuted and TimezoneOffset in appropriate place - return ed.Status == sql.EventStatus_Enable.String(), db.addFragToSchemasTable(ctx, + return event.Status == sql.EventStatus_Enable.String(), db.addFragToSchemasTable(ctx, eventFragment, - evDef.Name, - evDef.CreateStatement, - evDef.CreatedAt, - sql.ErrEventAlreadyExists.New(evDef.Name), + event.Name, + event.CreateEventStatement(), + event.CreatedAt, + sql.ErrEventAlreadyExists.New(event.Name), ) } @@ -1351,13 +1353,13 @@ func (db Database) DropEvent(ctx *sql.Context, name string) error { } // UpdateEvent implements sql.EventDatabase. -func (db Database) UpdateEvent(ctx *sql.Context, originalName string, ed sql.EventDetails) (bool, error) { +func (db Database) UpdateEvent(ctx *sql.Context, originalName string, event sql.EventDefinition) (bool, error) { // TODO: any EVENT STATUS change should also update the branch-specific event scheduling err := db.DropEvent(ctx, originalName) if err != nil { return false, err } - return db.SaveEvent(ctx, ed) + return db.SaveEvent(ctx, event) } // UpdateLastExecuted implements sql.EventDatabase From 34018131143210fd266a8de235a79d666c682ff3 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Fri, 1 Sep 2023 14:54:16 -0700 Subject: [PATCH 21/45] Cleaning up how we construct EventDefinition instances from schema fragments --- go/libraries/doltcore/sqle/database.go | 73 ++++++++++++++++++++------ 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 89ff5cce97..c228c135d9 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -24,6 +24,7 @@ import ( sqle "github.com/dolthub/go-mysql-server" "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/analyzer" "github.com/dolthub/go-mysql-server/sql/fulltext" "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/planbuilder" @@ -1280,14 +1281,11 @@ func (db Database) GetEvent(ctx *sql.Context, name string) (sql.EventDefinition, for _, frag := range frags { if strings.ToLower(frag.name) == strings.ToLower(name) { - return sql.EventDefinition{ - Name: frag.name, - CreateStatement: updateEventStatusTemporarilyForNonDefaultBranch(db.revision, frag.fragment), - CreatedAt: frag.created, - SqlMode: frag.sqlMode, - LastAltered: frag.created, - // TODO: fill TimezoneOffset and LastExecuted - }, true, nil + event, err := db.createEventDefinitionFromFragment(ctx, frag) + if err != nil { + return sql.EventDefinition{}, false, err + } + return *event, true, nil } } return sql.EventDefinition{}, false, nil @@ -1310,19 +1308,60 @@ func (db Database) GetEvents(ctx *sql.Context) ([]sql.EventDefinition, error) { var events []sql.EventDefinition for _, frag := range frags { - events = append(events, sql.EventDefinition{ - Name: frag.name, - CreateStatement: updateEventStatusTemporarilyForNonDefaultBranch(db.revision, frag.fragment), - CreatedAt: frag.created, - SqlMode: frag.sqlMode, - LastAltered: frag.created, - // TODO: fill TimezoneOffset and LastExecuted - - }) + event, err := db.createEventDefinitionFromFragment(ctx, frag) + if err != nil { + return nil, err + } + events = append(events, *event) } return events, nil } +// createEventDefinitionFromFragment creates an EventDefinition instance from the schema fragment |frag|. +func (db Database) createEventDefinitionFromFragment(ctx *sql.Context, frag schemaFragment) (*sql.EventDefinition, error) { + catalog, err := db.getCatalog() + if err != nil { + return nil, err + } + + sqlMode := sql.NewSqlModeFromString(frag.sqlMode) + parsed, err := planbuilder.ParseWithOptions(ctx, catalog, updateEventStatusTemporarilyForNonDefaultBranch(db.revision, frag.fragment), sqlMode.ParserOptions()) + if err != nil { + return nil, err + } + + eventPlan, ok := parsed.(*plan.CreateEvent) + if !ok { + return nil, fmt.Errorf("unexpected type %T for create event statement", eventPlan) + } + + // NOTE: Time fields for events are assumed to be specified in the session's timezone, which defaults to the + // system's timezone. When we store them, we store them at UTC, and when we send them back to a caller + // we convert them back to the caller's session timezone. + // Here we are loading the events from disk, so they are already in UTC and don't need any other + // timezone applied, so we specify "+00:00". + event, err := eventPlan.GetParsedEventDefinition(ctx, frag.created, frag.created, frag.created, "+00:00") + if err != nil { + return nil, err + } + event.SqlMode = frag.sqlMode + + return &event, nil +} + +// getCatalog creates and returns a new analyzer.Catalog instance with access to only this database. +// NOTE: Because we don't have access to the real Catalog here, and we need it to call planbuilder.ParseWithOptions +// +// to parse create event fragments, we create a new DoltDB provider and a new engine, and then use that Catalog. +// It would be cleaner and more efficient if we could grab the real Catalog, but we don't have access here. +func (db Database) getCatalog() (*analyzer.Catalog, error) { + pro, err := NewDoltDatabaseProviderWithDatabase(env.DefaultInitBranch, nil, db, nil) + if err != nil { + return nil, err + } + return sqle.NewDefault(pro).Analyzer.Catalog, nil +} + // SaveEvent implements sql.EventDatabase. func (db Database) SaveEvent(ctx *sql.Context, event sql.EventDefinition) (bool, error) { // If the database is not the default branch database, then the event is disabled. From 48e33ffb5004bf2ab8fa4deb73569ac0c5310b88 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Fri, 1 Sep 2023 15:10:14 -0700 Subject: [PATCH 22/45] Updating GMS dependency to latest build from jennifer/event-execution dev branch --- go/go.mod | 2 +- go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index 089434ca36..b47b0525e2 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20230831170657-87fce68aac66 + github.com/dolthub/go-mysql-server v0.17.1-0.20230901220317-567487a88a38 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index 26bd87504a..9e19d9fe08 100644 --- a/go/go.sum +++ b/go/go.sum @@ -182,6 +182,8 @@ github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= github.com/dolthub/go-mysql-server v0.17.1-0.20230831170657-87fce68aac66 h1:3ojq1xvlX7TFX5Tlf1uOymmaloi6oOs4VC6biY7Lths= github.com/dolthub/go-mysql-server v0.17.1-0.20230831170657-87fce68aac66/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= +github.com/dolthub/go-mysql-server v0.17.1-0.20230901220317-567487a88a38 h1:yQjTLpBcysw6BNRr6APgBakJxvlZk8daI42STK1iGDg= +github.com/dolthub/go-mysql-server v0.17.1-0.20230901220317-567487a88a38/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= From 566e1584ca0e0b9fb72f0479bc957b1a9a992f21 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 6 Sep 2023 14:03:38 -0700 Subject: [PATCH 23/45] Fixing Catalog access --- go/libraries/doltcore/sqle/database.go | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index c228c135d9..f9c80f1809 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1319,11 +1319,7 @@ func (db Database) GetEvents(ctx *sql.Context) ([]sql.EventDefinition, error) { // createEventDefinitionFromFragment creates an EventDefinition instance from the schema fragment |frag|. func (db Database) createEventDefinitionFromFragment(ctx *sql.Context, frag schemaFragment) (*sql.EventDefinition, error) { - catalog, err := db.getCatalog() - if err != nil { - return nil, err - } - + catalog := db.getCatalog(ctx) sqlMode := sql.NewSqlModeFromString(frag.sqlMode) parsed, err := planbuilder.ParseWithOptions(ctx, catalog, updateEventStatusTemporarilyForNonDefaultBranch(db.revision, frag.fragment), sqlMode.ParserOptions()) if err != nil { @@ -1349,17 +1345,10 @@ func (db Database) createEventDefinitionFromFragment(ctx *sql.Context, frag sche return &event, nil } -// getCatalog creates and returns a new analyzer.Catalog instance with access to only this database. -// NOTE: Because we don't have access to the real Catalog here, and we need it to call planbuilder.ParseWithOptions -// -// to parse create event fragments, we create a new DoltDB provider and a new engine, and then use that Catalog. -// It would be cleaner and more efficient if we could grab the real Catalog, but we don't have access here. -func (db Database) getCatalog() (*analyzer.Catalog, error) { - pro, err := NewDoltDatabaseProviderWithDatabase(env.DefaultInitBranch, nil, db, nil) - if err != nil { - return nil, err - } - return sqle.NewDefault(pro).Analyzer.Catalog, nil +// getCatalog creates and returns the analyzer.Catalog instance for this database. +func (db Database) getCatalog(ctx *sql.Context) *analyzer.Catalog { + doltSession := dsess.DSessFromSess(ctx.Session) + return sqle.NewDefault(doltSession.Provider()).Analyzer.Catalog } // SaveEvent implements sql.EventDatabase. From f0da9e955c947e8cf8ba7dfe9f1e7b762120465a Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 7 Sep 2023 13:30:54 -0700 Subject: [PATCH 24/45] Cleaning up tests and removing some unnecessary code --- go/cmd/dolt/commands/engine/sqlengine.go | 16 ---------------- integration-tests/bats/events.bats | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index dda14b6600..e405894cb0 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -343,22 +343,6 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa // return nil, func() error { return nil }, err //} - // TODO: This should be enabled by default, plus it looks like we're calling commit anyway? - //err = sess.SetSessionVariable(newCtx, sql.AutoCommitSessionVar, true) - //if err != nil { - // return nil, func() error { return nil }, err - //} - - //// set dolt_show_branch_databases to 'true' to get all revision databases - //// when accessing all databases. When retrieving events from each database, - //// only the default branch database will return events that have 'ENABLE' status. - //// All other databases will return events with 'DISABLE' status regardless of the - //// stored status. - //err = sess.SetSessionVariable(newCtx, dsess.ShowBranchDatabases, true) - //if err != nil { - // return nil, func() error { return nil }, err - //} - ts, ok := newCtx.Session.(sql.TransactionSession) if !ok { return nil, func() error { return nil }, nil diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index 960f4f492d..64a5d7e423 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -3,13 +3,13 @@ load $BATS_TEST_DIRNAME/helper/common.bash load $BATS_TEST_DIRNAME/helper/query-server-common.bash make_test_repo_and_start_server() { + rm -rf ./"$1" mkdir "$1" cd "$1" - dolt init - dolt sql -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int)" - dolt commit -Am "create table" start_sql_server - cd .. + dolt sql-client -P $PORT -u dolt --use-db information_schema -q "CREATE DATABASE repo1;" + dolt sql-client -P $PORT -u dolt --use-db repo1 -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int);" + dolt sql-client -P $PORT -u dolt --use-db repo1 -q "call dolt_commit('-Am', 'creating table');" } setup() { @@ -20,7 +20,6 @@ setup() { teardown() { stop_sql_server 1 && sleep 0.5 - rm -rf $BATS_TMPDIR/sql-server-test$$ teardown_common } @@ -106,7 +105,7 @@ teardown() { @test "events: checking out a branch should disable all events leaving the working set dirty" { dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 2 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 3 SECOND ON COMPLETION PRESERVE DO INSERT INTO totals (int_col) VALUES (1); SELECT SLEEP(5);" - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_COMMIT('-am','commit with an event')" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_COMMIT('-am','commit event changes to totals table')" dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_CHECKOUT('-b','newbranch')" # should be disabled @@ -115,16 +114,17 @@ teardown() { [[ $output =~ "DISABLED" ]] || false } -@test "events: events on default branch still runs after switching to non default branch" { - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 5 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 1 MINUTE DO INSERT INTO totals (int_col) VALUES (1);" - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_COMMIT('-am','commit with an event')" +@test "events: events on default branch still run after switching to non-default branch" { + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT insert1 ON SCHEDULE EVERY 3 SECOND ENDS CURRENT_TIMESTAMP + INTERVAL 3 MINUTE DO INSERT INTO totals (int_col) VALUES (1);" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_COMMIT('-Am','commit with an event');" - dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_CHECKOUT('-b','newbranch')" + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_CHECKOUT('-b','newbranch');" # should be disabled run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] [[ $output =~ "| 1 |" ]] || false + # While we are sleeping, the event executor should still be running our event on schedule, on the main branch sleep 5 run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CALL DOLT_CHECKOUT('main'); SELECT COUNT(*) FROM totals;" [ $status -eq 0 ] From e7cae6259264866ab2c8762a0f579b92d8838ffe Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 7 Sep 2023 13:33:33 -0700 Subject: [PATCH 25/45] Bumping GMS dep to latest build from jennifer/event-execution dev branch --- go/go.mod | 2 +- go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index b47b0525e2..a632cb34c8 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20230901220317-567487a88a38 + github.com/dolthub/go-mysql-server v0.17.1-0.20230907200408-0eb4c4f9f397 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index 9e19d9fe08..ea04937490 100644 --- a/go/go.sum +++ b/go/go.sum @@ -184,6 +184,8 @@ github.com/dolthub/go-mysql-server v0.17.1-0.20230831170657-87fce68aac66 h1:3ojq github.com/dolthub/go-mysql-server v0.17.1-0.20230831170657-87fce68aac66/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= github.com/dolthub/go-mysql-server v0.17.1-0.20230901220317-567487a88a38 h1:yQjTLpBcysw6BNRr6APgBakJxvlZk8daI42STK1iGDg= github.com/dolthub/go-mysql-server v0.17.1-0.20230901220317-567487a88a38/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= +github.com/dolthub/go-mysql-server v0.17.1-0.20230907200408-0eb4c4f9f397 h1:UyWI4XDfk/N8Vrqa9fBOE2VVN4dFRBXz67j7Yh9JKuA= +github.com/dolthub/go-mysql-server v0.17.1-0.20230907200408-0eb4c4f9f397/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= From 53159ce2e362dbe0d94d42e0d4fcc2e238d63f5b Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 7 Sep 2023 14:45:31 -0700 Subject: [PATCH 26/45] Fixing replication test to configure replicate branch *after* it's been created --- go/cmd/dolt/commands/sqlserver/server_test.go | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/go/cmd/dolt/commands/sqlserver/server_test.go b/go/cmd/dolt/commands/sqlserver/server_test.go index 2e4fafefea..00f0142606 100644 --- a/go/cmd/dolt/commands/sqlserver/server_test.go +++ b/go/cmd/dolt/commands/sqlserver/server_test.go @@ -15,7 +15,6 @@ package sqlserver import ( - "fmt" "net/http" "os" "strings" @@ -445,7 +444,6 @@ func runDefaultBranchTests(t *testing.T, tests []defaultBranchTest, conn *dbr.Co } func TestReadReplica(t *testing.T) { - var err error cwd, err := os.Getwd() if err != nil { t.Fatalf("no working directory: %s", err.Error()) @@ -467,10 +465,8 @@ func TestReadReplica(t *testing.T) { sourceDbName := multiSetup.DbNames[1] localCfg, ok := multiSetup.GetEnv(readReplicaDbName).Config.GetConfig(env.LocalConfig) - if !ok { - t.Fatal("local config does not exist") - } - config.NewPrefixConfig(localCfg, env.SqlServerGlobalsPrefix).SetStrings(map[string]string{dsess.ReadReplicaRemote: "remote1", dsess.ReplicateHeads: "main,feature"}) + require.True(t, ok, "local config does not exist") + config.NewPrefixConfig(localCfg, env.SqlServerGlobalsPrefix).SetStrings(map[string]string{dsess.ReadReplicaRemote: "remote1", dsess.ReplicateHeads: "main"}) dsess.InitPersistedSystemVars(multiSetup.GetEnv(readReplicaDbName)) // start server as read replica @@ -480,14 +476,12 @@ func TestReadReplica(t *testing.T) { // set socket to nil to force tcp serverConfig = serverConfig.WithHost("127.0.0.1").WithSocket("") - func() { - os.Chdir(multiSetup.DbPaths[readReplicaDbName]) - go func() { - _, _ = Serve(context.Background(), "0.0.0", serverConfig, sc, multiSetup.GetEnv(readReplicaDbName)) - }() - err = sc.WaitForStart() + os.Chdir(multiSetup.DbPaths[readReplicaDbName]) + go func() { + err, _ = Serve(context.Background(), "0.0.0", serverConfig, sc, multiSetup.GetEnv(readReplicaDbName)) require.NoError(t, err) }() + require.NoError(t, sc.WaitForStart()) defer sc.StopServer() replicatedTable := "new_table" @@ -502,14 +496,16 @@ func TestReadReplica(t *testing.T) { require.NoError(t, err) sess := conn.NewSession(nil) - newBranch := "feature" - multiSetup.NewBranch(sourceDbName, newBranch) - multiSetup.CheckoutBranch(sourceDbName, newBranch) - multiSetup.PushToRemote(sourceDbName, "remote1", newBranch) + multiSetup.NewBranch(sourceDbName, "feature") + multiSetup.CheckoutBranch(sourceDbName, "feature") + multiSetup.PushToRemote(sourceDbName, "remote1", "feature") + + // Configure the read replica to pull the new feature branch we just created + config.NewPrefixConfig(localCfg, env.SqlServerGlobalsPrefix).SetStrings(map[string]string{dsess.ReadReplicaRemote: "remote1", dsess.ReplicateHeads: "main,feature"}) + dsess.InitPersistedSystemVars(multiSetup.GetEnv(readReplicaDbName)) var res []int - - q := sess.SelectBySql(fmt.Sprintf("call dolt_checkout('%s')", newBranch)) + q := sess.SelectBySql("call dolt_checkout('feature');") _, err = q.LoadContext(context.Background(), &res) require.NoError(t, err) assert.ElementsMatch(t, res, []int{0}) From 6d5d5d57465a6143693b6892594d5e7a494222d7 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 7 Sep 2023 15:58:12 -0700 Subject: [PATCH 27/45] Tidying up --- go/cmd/dolt/commands/engine/sqlengine.go | 10 ---------- go/libraries/doltcore/sqle/database.go | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index e405894cb0..fdb1288b8f 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -333,16 +333,6 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa return nil, func() error { return nil }, err } - // TODO: Seems like we either need to fix this test or change this behavior to - // not start a transaction until right before we need one. - //// Starting transaction at the start of the server to load all events to - //// the event scheduler is causing read replica test to fail as it tries - //// pulling from remotes for branches that its remoteRef does not exist yet. - //err = sql.SystemVariables.SetGlobal(dsess.SkipReplicationErrors, true) - //if err != nil { - // return nil, func() error { return nil }, err - //} - ts, ok := newCtx.Session.(sql.TransactionSession) if !ok { return nil, func() error { return nil }, nil diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index f9c80f1809..1cc4cd1392 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1336,7 +1336,7 @@ func (db Database) createEventDefinitionFromFragment(ctx *sql.Context, frag sche // we convert them back to the caller's session timezone. // Here we are loading the events from disk, so they are already in UTC and don't need any other // timezone applied, so we specify "+00:00". - event, err := eventPlan.GetParsedEventDefinition(ctx, frag.created, frag.created, frag.created, "+00:00") + event, err := eventPlan.GetEventDefinition(ctx, frag.created, frag.created, frag.created, "+00:00") if err != nil { return nil, err } From 45a9494f92eac9a72b2b5aa91a404a9d5f1bdb93 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 7 Sep 2023 16:09:20 -0700 Subject: [PATCH 28/45] Bumping to latest GMS version from jennifer/event-execution dev branch --- go/go.mod | 2 +- go/go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index a632cb34c8..b2d8a82013 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20230907200408-0eb4c4f9f397 + github.com/dolthub/go-mysql-server v0.17.1-0.20230907230543-b06702cdd83d github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index ea04937490..9691c66cb1 100644 --- a/go/go.sum +++ b/go/go.sum @@ -186,6 +186,10 @@ github.com/dolthub/go-mysql-server v0.17.1-0.20230901220317-567487a88a38 h1:yQjT github.com/dolthub/go-mysql-server v0.17.1-0.20230901220317-567487a88a38/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= github.com/dolthub/go-mysql-server v0.17.1-0.20230907200408-0eb4c4f9f397 h1:UyWI4XDfk/N8Vrqa9fBOE2VVN4dFRBXz67j7Yh9JKuA= github.com/dolthub/go-mysql-server v0.17.1-0.20230907200408-0eb4c4f9f397/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= +github.com/dolthub/go-mysql-server v0.17.1-0.20230907225617-2e3175e17510 h1:r3x4T/K+Ogeyy9X3QXNdskQAeONy+w7M95LpLKTzr2I= +github.com/dolthub/go-mysql-server v0.17.1-0.20230907225617-2e3175e17510/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= +github.com/dolthub/go-mysql-server v0.17.1-0.20230907230543-b06702cdd83d h1:gPmfG+tTYAUF9H4lCxqM3UeDiwM2jKA3a35klFBm64U= +github.com/dolthub/go-mysql-server v0.17.1-0.20230907230543-b06702cdd83d/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= From 461938c907cda06a3832cb1d9ef0e0b08989eb8c Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 13 Sep 2023 10:09:28 -0700 Subject: [PATCH 29/45] Adding comments to explain code more --- go/libraries/doltcore/sqle/database.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 1cc4cd1392..742eaa9e1a 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1399,9 +1399,13 @@ func (db Database) UpdateLastExecuted(ctx *sql.Context, eventName string, lastEx // updateEventStatusTemporarilyForNonDefaultBranch updates the event status from ENABLE to DISABLE if it's not default branch. // The event status metadata is not updated in storage, but only for display purposes we return event status as 'DISABLE'. // This function is used temporarily to implement logic of only allowing enabled events to be executed on default branch. -func updateEventStatusTemporarilyForNonDefaultBranch(defaultBranch, createStmt string) string { +func updateEventStatusTemporarilyForNonDefaultBranch(revision, createStmt string) string { // TODO: need better way to determine the default branch; currently it checks only 'main' - if defaultBranch == env.DefaultInitBranch { + + // TODO: We currently rely on having the dolt_show_branch_databases flag turned on for this event scheduler's + // session in order to identify the events from the main branch. This is a bit inefficient (e.g. for databases + // with many, many branches), but this code should only run at startup, so may be okay for first version. + if revision == env.DefaultInitBranch { return createStmt } return strings.Replace(createStmt, "ENABLE", "DISABLE", 1) From 9a1a830aafc6fbf85078481e7d292ce07f47cd74 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 13 Sep 2023 14:45:34 -0700 Subject: [PATCH 30/45] Adding a new BATS test for multi-statement event bodies --- integration-tests/bats/events.bats | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index 64a5d7e423..9804ac7916 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -130,3 +130,36 @@ teardown() { [ $status -eq 0 ] [[ ! $output =~ "| 1 |" ]] || false } + +# Test that events with multiple statements in nested BEGIN/END blocks work correctly +@test "events: multiple statements in nested BEGIN END blocks in event body" { + # Use dolt sql to pipe in a HEREDOC; Note that this will connect to the running sql-server + cd repo1 + dolt sql << SQL +delimiter // +CREATE EVENT event1234 +ON SCHEDULE AT CURRENT_TIMESTAMP +DO +BEGIN +INSERT INTO totals (int_col) VALUES (111); +BEGIN +INSERT INTO totals (int_col) VALUES (222); +INSERT INTO totals (int_col) VALUES (333); +END; +END; +// +delimiter ; +SQL + + # Verify that our event ran correctly and inserted three rows + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT * FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 1 | 111 |" ]] || false + [[ $output =~ "| 2 | 222 |" ]] || false + [[ $output =~ "| 3 | 333 |" ]] || false + + # Verify that the event did not persist after execution + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM information_schema.events;" + [ $status -eq 0 ] + [[ $output =~ "| 0 |" ]] || false +} From a2e294889971c2380ebffeea0a3ce0bf76ce274c Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 13 Sep 2023 16:47:09 -0700 Subject: [PATCH 31/45] Re-enabling ShowBranchDatabases for the event executor's session in order to correctly load/enable events on sql-server startup --- go/cmd/dolt/commands/engine/sqlengine.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index fdb1288b8f..209e380e08 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -338,6 +338,16 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa return nil, func() error { return nil }, nil } + // set dolt_show_branch_databases to 'true' to get all revision databases + // when accessing all databases. When retrieving events from each database, + // only the default branch database will return events that have 'ENABLE' status. + // All other databases will return events with 'DISABLE' status regardless of the + // stored status. + err = sess.SetSessionVariable(newCtx, dsess.ShowBranchDatabases, true) + if err != nil { + return nil, func() error { return nil }, err + } + tr, err := sess.StartTransaction(newCtx, sql.ReadWrite) if err != nil { return nil, func() error { return nil }, err From 131aa5bdaba807c20876df9fde9673a22001690e Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 13 Sep 2023 17:04:00 -0700 Subject: [PATCH 32/45] Adding a test for an event that calls a stored procedure --- integration-tests/bats/events.bats | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index 9804ac7916..05b9b0a22e 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -163,3 +163,43 @@ SQL [ $status -eq 0 ] [[ $output =~ "| 0 |" ]] || false } + +# Test that events containing procedure calls work correctly +@test "events: procedure calls in events" { + # Create a procedure + cd repo1 + dolt sql << SQL +DELIMITER // +CREATE PROCEDURE InsertIntoTotals() +BEGIN + INSERT INTO totals (int_col) VALUES (42); +END // +DELIMITER ; +SQL + + # Use dolt sql to pipe in a HEREDOC; Note that this will connect to the running sql-server + dolt sql << SQL +delimiter // +CREATE EVENT event1234 +ON SCHEDULE AT CURRENT_TIMESTAMP +DO +BEGIN + CALL InsertIntoTotals(); +END; +// +delimiter ; +SQL + + # Verify that our event ran correctly and inserted one row + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT * FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 1 | 42 |" ]] || false + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 1 " ]] || false + + # Verify that the event did not persist after execution + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM information_schema.events;" + [ $status -eq 0 ] + [[ $output =~ "| 0 |" ]] || false +} \ No newline at end of file From 187649790fceafa6bae938e3094be83978c548f9 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 14 Sep 2023 11:35:31 -0700 Subject: [PATCH 33/45] Removing unnecessary newline in status output --- go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go b/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go index 39c375027b..72f7ebbcdc 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_checkout.go @@ -109,7 +109,7 @@ func doDoltCheckout(ctx *sql.Context, args []string) (statusCode int, successMes return 1, "", err } if !isModification { - return 0, fmt.Sprintf("Already on branch '%s'\n", branchName), nil + return 0, fmt.Sprintf("Already on branch '%s'", branchName), nil } // Check if user wants to checkout branch. From e71e43537b7cadc98fb5ddd6db13b5e5d3a8c164 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 14 Sep 2023 16:28:33 -0700 Subject: [PATCH 34/45] Adding a new BATS test for out-of-band updates to event definitions (e.g. merges, reverts, resets) --- integration-tests/bats/events.bats | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index 05b9b0a22e..90570559f6 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -202,4 +202,35 @@ SQL run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM information_schema.events;" [ $status -eq 0 ] [[ $output =~ "| 0 |" ]] || false +} + +# Test that out-of-band event definition changes (e.g. merges, reverts, or anything else that doesn't go through +# CREATE EVENT statements) are reflected correctly. +@test "events: out-of-band event changes are detected" { + # Use dolt sql to pipe in a HEREDOC; Note that this will connect to the running sql-server + dolt sql << SQL +call dolt_checkout('-b', 'other'); +CREATE EVENT event12345 +ON SCHEDULE EVERY 0.5 SECOND STARTS CURRENT_TIMESTAMP +DO INSERT INTO totals (int_col) VALUES (42); +call dolt_commit('-Am', 'Adding a new recurring event'); +SQL + + # Verify that our event IS NOT executing (on a non-main branch) + sleep 1 + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT COUNT(*) FROM totals;" + [ $status -eq 0 ] + [[ $output =~ "| 0 " ]] || false + + # Merge our event from other back to main + dolt sql << SQL +call dolt_checkout('main'); +call dolt_merge('other'); +SQL + + # Verify that the new event starts executing on main after we merge it over + sleep 1 + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT (SELECT COUNT(*) FROM totals) > 0;" + [ $status -eq 0 ] + [[ $output =~ "| 1 " ]] || false } \ No newline at end of file From be38ef36e46d6be4877a0b6b172d310c916ee1c7 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Fri, 15 Sep 2023 10:26:07 -0700 Subject: [PATCH 35/45] Minor comment tweak --- go/cmd/dolt/commands/engine/sqlengine.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 209e380e08..8909bdaa96 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -315,7 +315,8 @@ func configureBinlogReplicaController(config *SqlEngineConfig, engine *gms.Engin return nil } -// configureEventScheduler configures the event scheduler with the |engine|. +// configureEventScheduler configures the event scheduler with the |engine| for executing events, a |sessFactory| +// for creating sessions, and a DoltDatabaseProvider, |pro|. func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFactory sessionFactory, pro dsqle.DoltDatabaseProvider) error { // need to give correct user, use the definer as user to run the event definition queries ctxFactory := sqlContextFactory() From 108c81b594e17e9fd18ef278f70d5bf37bf6bff8 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Mon, 18 Sep 2023 16:40:34 -0700 Subject: [PATCH 36/45] Updating BATS test for out-of-band event changes --- integration-tests/bats/events.bats | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index 90570559f6..8ae73b3f39 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -211,7 +211,7 @@ SQL dolt sql << SQL call dolt_checkout('-b', 'other'); CREATE EVENT event12345 -ON SCHEDULE EVERY 0.5 SECOND STARTS CURRENT_TIMESTAMP +ON SCHEDULE EVERY 1 SECOND STARTS CURRENT_TIMESTAMP DO INSERT INTO totals (int_col) VALUES (42); call dolt_commit('-Am', 'Adding a new recurring event'); SQL @@ -222,15 +222,17 @@ SQL [ $status -eq 0 ] [[ $output =~ "| 0 " ]] || false - # Merge our event from other back to main + # Merge our event from other back to main and enable it dolt sql << SQL call dolt_checkout('main'); call dolt_merge('other'); +ALTER EVENT event12345 ENABLE; +call dolt_commit('-am', 'committing enabled event'); SQL # Verify that the new event starts executing on main after we merge it over - sleep 1 + sleep 2 run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT (SELECT COUNT(*) FROM totals) > 0;" [ $status -eq 0 ] [[ $output =~ "| 1 " ]] || false -} \ No newline at end of file +} From a7afaf3e1c2427be2b98a2053327d80cd7dda116 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 21 Sep 2023 15:58:45 -0700 Subject: [PATCH 37/45] Updating EventDatabase interfaces from changes to out-of-band change reloading in GMS --- go/libraries/doltcore/sqle/database.go | 50 ++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 742eaa9e1a..233086e7d8 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1292,29 +1292,65 @@ func (db Database) GetEvent(ctx *sql.Context, name string) (sql.EventDefinition, } // GetEvents implements sql.EventDatabase. -func (db Database) GetEvents(ctx *sql.Context) ([]sql.EventDefinition, error) { +func (db Database) GetEvents(ctx *sql.Context) (events []sql.EventDefinition, token interface{}, err error) { tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName) if err != nil { - return nil, err + return nil, nil, err } if !ok { - return nil, nil + // If the dolt_schemas table doesn't exist, it's not an error, just no events + return nil, nil, nil } frags, err := getSchemaFragmentsOfType(ctx, tbl.(*WritableDoltTable), eventFragment) if err != nil { - return nil, err + return nil, nil, err } - var events []sql.EventDefinition for _, frag := range frags { event, err := db.createEventDefinitionFromFragment(ctx, frag) if err != nil { - return nil, err + return nil, nil, err } events = append(events, *event) } - return events, nil + + // Grab a hash of the dolt_schemas table to use as the identifying token + // to track if events need to be reloaded. + tableHash, err := db.doltSchemaTableHash(ctx) + if err != nil { + return nil, nil, err + } + + return events, tableHash, nil +} + +// NeedsToReloadEvents implements sql.EventDatabase. +func (db Database) NeedsToReloadEvents(ctx *sql.Context, token interface{}) (bool, error) { + hash, ok := token.(hash.Hash) + if !ok { + return false, fmt.Errorf("expected token to be hash.Hash, but received %T", token) + } + + tableHash, err := db.doltSchemaTableHash(ctx) + if err != nil { + return false, err + } + + // If the current hash doesn't match what we last loaded, then we + // need to reload event definitions + return !tableHash.Equal(hash), nil +} + +// doltSchemaTableHash returns the hash of the dolt_schemas table, or any error encountered along the way. +func (db Database) doltSchemaTableHash(ctx *sql.Context) (hash.Hash, error) { + root, err := db.GetRoot(ctx) + if err != nil { + return hash.Hash{}, err + } + + tableHash, _, err := root.GetTableHash(ctx, doltdb.SchemasTableName) + return tableHash, err } // createEventDefinitionFromFragment creates an EventDefinition instance from the schema fragment |frag|. From f3b1fccba668dd662f1661598d1466f51a4fc418 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 21 Sep 2023 16:01:48 -0700 Subject: [PATCH 38/45] Bumping GMS dep to latest build from jennifer/event-execution dev branch (aecdeeda6873c91b0cf891ccf3da700cde5b118e) --- go/go.mod | 2 +- go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index b2d8a82013..5e299c8c3c 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20230907230543-b06702cdd83d + github.com/dolthub/go-mysql-server v0.17.1-0.20230921225706-aecdeeda6873 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index 9691c66cb1..0b5bf64f7e 100644 --- a/go/go.sum +++ b/go/go.sum @@ -190,6 +190,8 @@ github.com/dolthub/go-mysql-server v0.17.1-0.20230907225617-2e3175e17510 h1:r3x4 github.com/dolthub/go-mysql-server v0.17.1-0.20230907225617-2e3175e17510/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= github.com/dolthub/go-mysql-server v0.17.1-0.20230907230543-b06702cdd83d h1:gPmfG+tTYAUF9H4lCxqM3UeDiwM2jKA3a35klFBm64U= github.com/dolthub/go-mysql-server v0.17.1-0.20230907230543-b06702cdd83d/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= +github.com/dolthub/go-mysql-server v0.17.1-0.20230921225706-aecdeeda6873 h1:1J2Q4+2FtRDs+fmjV0F6x1MKJTgbkOjmBTQ3CROBLU8= +github.com/dolthub/go-mysql-server v0.17.1-0.20230921225706-aecdeeda6873/go.mod h1:vSQ47leaIPTtvSLKo89D1FdYdypU5OH6VBV63B2MS8Y= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= From 8b03936f369484eced93b7682142035ea5686399 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Thu, 21 Sep 2023 16:11:03 -0700 Subject: [PATCH 39/45] Bumping GMS dep to latest build from jennifer/event-execution dev branch (30e6a6114f41a46b9cd4f6b6266e4a20f1eefb12) --- go/go.mod | 2 +- go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index e553f62f88..52d764e205 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20230920214618-75692cb811e7 + github.com/dolthub/go-mysql-server v0.17.1-0.20230921230607-30e6a6114f41 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index 776bd37c4b..29dee4b3b5 100644 --- a/go/go.sum +++ b/go/go.sum @@ -182,6 +182,8 @@ github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= github.com/dolthub/go-mysql-server v0.17.1-0.20230920214618-75692cb811e7 h1:QWo34tf44CmwZh27GMqgPrNU0cr+Fc9ULoX6o40PwMQ= github.com/dolthub/go-mysql-server v0.17.1-0.20230920214618-75692cb811e7/go.mod h1:9xeYTIAFZLOdDZzuOf1HQF5Td6OYqMt24IRSJM0ayjM= +github.com/dolthub/go-mysql-server v0.17.1-0.20230921230607-30e6a6114f41 h1:B0+AvPqBHYDXytF3UxeuWfcN+tFpp+ya4ikJZ3agmQY= +github.com/dolthub/go-mysql-server v0.17.1-0.20230921230607-30e6a6114f41/go.mod h1:9xeYTIAFZLOdDZzuOf1HQF5Td6OYqMt24IRSJM0ayjM= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= From 267c3531b62ac3608847049960325c2a7f854454 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Mon, 25 Sep 2023 13:39:44 -0700 Subject: [PATCH 40/45] Preventing the need to run on dolt_show_branch_databases so that we don't have to poll every branch all the time. --- go/cmd/dolt/commands/engine/sqlengine.go | 10 ---------- go/libraries/doltcore/sqle/database.go | 5 +---- integration-tests/bats/events.bats | 23 +++++++++++++++++++++++ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 633dcb709f..57fb2a7cd4 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -342,16 +342,6 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa return nil, func() error { return nil }, nil } - // set dolt_show_branch_databases to 'true' to get all revision databases - // when accessing all databases. When retrieving events from each database, - // only the default branch database will return events that have 'ENABLE' status. - // All other databases will return events with 'DISABLE' status regardless of the - // stored status. - err = sess.SetSessionVariable(newCtx, dsess.ShowBranchDatabases, true) - if err != nil { - return nil, func() error { return nil }, err - } - tr, err := sess.StartTransaction(newCtx, sql.ReadWrite) if err != nil { return nil, func() error { return nil }, err diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index 233086e7d8..86e7b05435 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1438,10 +1438,7 @@ func (db Database) UpdateLastExecuted(ctx *sql.Context, eventName string, lastEx func updateEventStatusTemporarilyForNonDefaultBranch(revision, createStmt string) string { // TODO: need better way to determine the default branch; currently it checks only 'main' - // TODO: We currently rely on having the dolt_show_branch_databases flag turned on for this event scheduler's - // session in order to identify the events from the main branch. This is a bit inefficient (e.g. for databases - // with many, many branches), but this code should only run at startup, so may be okay for first version. - if revision == env.DefaultInitBranch { + if revision == "" || revision == env.DefaultInitBranch { return createStmt } return strings.Replace(createStmt, "ENABLE", "DISABLE", 1) diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index 8ae73b3f39..dfb7f61fa6 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -236,3 +236,26 @@ SQL [ $status -eq 0 ] [[ $output =~ "| 1 " ]] || false } + +@test "events: restarting a sql-server correctly schedules existing events" { + # Create the recurring event and make sure it runs at least once + dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "CREATE EVENT eventTest1 ON SCHEDULE EVERY 1 SECOND STARTS CURRENT_TIMESTAMP DO INSERT INTO totals (int_col) VALUES (111);" + sleep 1 + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT (SELECT COUNT(*) FROM totals) > 0;" + [ $status -eq 0 ] + [[ $output =~ "| 1 " ]] || false + + # Stop the sql-server, truncate the totals table, and assert it's empty + stop_sql_server 1 + dolt sql -q "truncate totals;" + run dolt sql -q "SELECT (SELECT COUNT(*) FROM totals) > 0;" + [ $status -eq 0 ] + [[ $output =~ "| false " ]] || false + + # Restart the server and assert that the event gets scheduled and executed again + start_sql_server + sleep 1 + run dolt sql-client -P $PORT -u dolt --use-db 'repo1' -q "SELECT (SELECT COUNT(*) FROM totals) > 0;" + [ $status -eq 0 ] + [[ $output =~ "| 1 " ]] || false +} From ab44211a85ea4c1eeff75cd65aed43ed2657f15e Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Mon, 25 Sep 2023 14:07:34 -0700 Subject: [PATCH 41/45] Bumping to latest GMS dep from jennifer/event-execution dev branch (39a057f69d89b528938fb27cdd4a0baa85ba8f59) --- go/go.mod | 4 ++-- go/go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go/go.mod b/go/go.mod index e86874cea0..944b1fe3e9 100644 --- a/go/go.mod +++ b/go/go.mod @@ -15,7 +15,7 @@ require ( github.com/dolthub/fslock v0.0.3 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 - github.com/dolthub/vitess v0.0.0-20230920212116-eb645ef178c2 + github.com/dolthub/vitess v0.0.0-20230925173309-ea0c9a6095af github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.13.0 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20230922201405-4d39c69a8090 + github.com/dolthub/go-mysql-server v0.17.1-0.20230925210315-39a057f69d89 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index f6a96e0bf0..498dc09192 100644 --- a/go/go.sum +++ b/go/go.sum @@ -182,6 +182,8 @@ github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= github.com/dolthub/go-mysql-server v0.17.1-0.20230922201405-4d39c69a8090 h1:FdsEiybuY6SSurAk41gd6DGvpqcQ8pm/XucTk2Ssm2o= github.com/dolthub/go-mysql-server v0.17.1-0.20230922201405-4d39c69a8090/go.mod h1:9xeYTIAFZLOdDZzuOf1HQF5Td6OYqMt24IRSJM0ayjM= +github.com/dolthub/go-mysql-server v0.17.1-0.20230925210315-39a057f69d89 h1:s5iL+4gW4DnoKg04TINx514xxeD3vbx+XOqk0mTOJ7k= +github.com/dolthub/go-mysql-server v0.17.1-0.20230925210315-39a057f69d89/go.mod h1:C0Q8nJKIEnCYXx+ot9F2V7bg7xz5wj+Z1o1BDyNnP7E= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= @@ -194,6 +196,8 @@ github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc= github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ= github.com/dolthub/vitess v0.0.0-20230920212116-eb645ef178c2 h1:L2ST5YNistuYVkVfZcTp5ItVpTNQ0i6oGGM7l6If9ME= github.com/dolthub/vitess v0.0.0-20230920212116-eb645ef178c2/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw= +github.com/dolthub/vitess v0.0.0-20230925173309-ea0c9a6095af h1:t4Fn1+oxtfdpW0tIJTlGFkuKiuLjbwNH0W72EiC9ueA= +github.com/dolthub/vitess v0.0.0-20230925173309-ea0c9a6095af/go.mod h1:IwjNXSQPymrja5pVqmfnYdcy7Uv7eNJNBPK/MEh9OOw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= From c9d2c73a4dcf5ed1ffa423061540f98b40690c8f Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 26 Sep 2023 15:40:12 -0700 Subject: [PATCH 42/45] Allowing a period parameter to be passed into event executor to override the default loop period (30s) and using for BATS tests --- go/cmd/dolt/commands/engine/sqlengine.go | 18 +++++++++++++++++- integration-tests/bats/events.bats | 5 +++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 57fb2a7cd4..706c152f73 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -16,8 +16,10 @@ package engine import ( "context" + "github.com/sirupsen/logrus" "os" "runtime" + "strconv" "strings" gms "github.com/dolthub/go-mysql-server" @@ -354,7 +356,21 @@ func configureEventScheduler(config *SqlEngineConfig, engine *gms.Engine, sessFa }, nil } - return engine.InitializeEventScheduler(getCtxFunc, config.EventSchedulerStatus) + // A hidden env var allows overriding the event scheduler period for testing. This option is not + // exposed via configuration because we do not want to encourage customers to use it. If the value + // is equal to or less than 0, then the period is ignored and the default period, 30s, is used. + eventSchedulerPeriod := 0 + eventSchedulerPeriodEnvVar := "DOLT_EVENT_SCHEDULER_PERIOD" + if s, ok := os.LookupEnv(eventSchedulerPeriodEnvVar); ok { + i, err := strconv.Atoi(s) + if err != nil { + logrus.Warnf("unable to parse value '%s' from env var '%s' as an integer", s, eventSchedulerPeriodEnvVar) + } else { + logrus.Warnf("overriding Dolt event scheduler period to %d seconds", i) + eventSchedulerPeriod = i + } + } + return engine.InitializeEventScheduler(getCtxFunc, config.EventSchedulerStatus, eventSchedulerPeriod) } // sqlContextFactory returns a contextFactory that creates a new sql.Context with the initial database provided diff --git a/integration-tests/bats/events.bats b/integration-tests/bats/events.bats index dfb7f61fa6..57e6a130b9 100644 --- a/integration-tests/bats/events.bats +++ b/integration-tests/bats/events.bats @@ -6,7 +6,12 @@ make_test_repo_and_start_server() { rm -rf ./"$1" mkdir "$1" cd "$1" + + # Override the default event scheduler period (30s) and set it to 1s so that we can run + # tests faster, without having to wait for the default 30s period to elapse several times. + export DOLT_EVENT_SCHEDULER_PERIOD=1 start_sql_server + dolt sql-client -P $PORT -u dolt --use-db information_schema -q "CREATE DATABASE repo1;" dolt sql-client -P $PORT -u dolt --use-db repo1 -q "CREATE TABLE totals (id int PRIMARY KEY AUTO_INCREMENT, int_col int);" dolt sql-client -P $PORT -u dolt --use-db repo1 -q "call dolt_commit('-Am', 'creating table');" From f41ef3e6c442985981b528b92a59fe2515920844 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 26 Sep 2023 15:51:58 -0700 Subject: [PATCH 43/45] Updating GMS dep to latest build from jennifer/event-execution dev branch (e3fb240e19db8be348428d9ee32ec9d257dc0058) --- go/go.mod | 2 +- go/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index 21d42677e4..fdb30ad9cd 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20230925200702-97cdc9c2cf91 + github.com/dolthub/go-mysql-server v0.17.1-0.20230926224224-e3fb240e19db github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index e50a7c4043..4824eca0f6 100644 --- a/go/go.sum +++ b/go/go.sum @@ -182,6 +182,8 @@ github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= github.com/dolthub/go-mysql-server v0.17.1-0.20230925200702-97cdc9c2cf91 h1:6iBYeUupd44zQW0CjIAMHTNRXNTS3sM2LuXI6z1EScc= github.com/dolthub/go-mysql-server v0.17.1-0.20230925200702-97cdc9c2cf91/go.mod h1:C0Q8nJKIEnCYXx+ot9F2V7bg7xz5wj+Z1o1BDyNnP7E= +github.com/dolthub/go-mysql-server v0.17.1-0.20230926224224-e3fb240e19db h1:iewMoOvBKOrWTbT4dmjTnMexyKyEkMutcAB1Pfql9iw= +github.com/dolthub/go-mysql-server v0.17.1-0.20230926224224-e3fb240e19db/go.mod h1:C0Q8nJKIEnCYXx+ot9F2V7bg7xz5wj+Z1o1BDyNnP7E= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ= From 7065185391b79db112b9db5dadc082eeddf12bd1 Mon Sep 17 00:00:00 2001 From: fulghum Date: Tue, 26 Sep 2023 23:02:33 +0000 Subject: [PATCH 44/45] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/cmd/dolt/commands/engine/sqlengine.go | 2 +- go/go.sum | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 706c152f73..0abdb3f23b 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -16,7 +16,6 @@ package engine import ( "context" - "github.com/sirupsen/logrus" "os" "runtime" "strconv" @@ -30,6 +29,7 @@ import ( "github.com/dolthub/go-mysql-server/sql/mysql_db" "github.com/dolthub/go-mysql-server/sql/rowexec" _ "github.com/dolthub/go-mysql-server/sql/variables" + "github.com/sirupsen/logrus" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/libraries/doltcore/branch_control" diff --git a/go/go.sum b/go/go.sum index 4824eca0f6..0ef53917f8 100644 --- a/go/go.sum +++ b/go/go.sum @@ -180,8 +180,6 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.17.1-0.20230925200702-97cdc9c2cf91 h1:6iBYeUupd44zQW0CjIAMHTNRXNTS3sM2LuXI6z1EScc= -github.com/dolthub/go-mysql-server v0.17.1-0.20230925200702-97cdc9c2cf91/go.mod h1:C0Q8nJKIEnCYXx+ot9F2V7bg7xz5wj+Z1o1BDyNnP7E= github.com/dolthub/go-mysql-server v0.17.1-0.20230926224224-e3fb240e19db h1:iewMoOvBKOrWTbT4dmjTnMexyKyEkMutcAB1Pfql9iw= github.com/dolthub/go-mysql-server v0.17.1-0.20230926224224-e3fb240e19db/go.mod h1:C0Q8nJKIEnCYXx+ot9F2V7bg7xz5wj+Z1o1BDyNnP7E= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= From 609296224f849378438356dd923783d95da9315c Mon Sep 17 00:00:00 2001 From: fulghum Date: Wed, 27 Sep 2023 00:32:39 +0000 Subject: [PATCH 45/45] [ga-bump-dep] Bump dependency in Dolt by fulghum --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index fdb30ad9cd..af7ce69e0e 100644 --- a/go/go.mod +++ b/go/go.mod @@ -59,7 +59,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.17.1-0.20230926224224-e3fb240e19db + github.com/dolthub/go-mysql-server v0.17.1-0.20230927003048-44ab355c2753 github.com/dolthub/swiss v0.1.0 github.com/goccy/go-json v0.10.2 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 diff --git a/go/go.sum b/go/go.sum index 0ef53917f8..30c946e733 100644 --- a/go/go.sum +++ b/go/go.sum @@ -180,8 +180,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y= github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168= -github.com/dolthub/go-mysql-server v0.17.1-0.20230926224224-e3fb240e19db h1:iewMoOvBKOrWTbT4dmjTnMexyKyEkMutcAB1Pfql9iw= -github.com/dolthub/go-mysql-server v0.17.1-0.20230926224224-e3fb240e19db/go.mod h1:C0Q8nJKIEnCYXx+ot9F2V7bg7xz5wj+Z1o1BDyNnP7E= +github.com/dolthub/go-mysql-server v0.17.1-0.20230927003048-44ab355c2753 h1:68B9w6CNK+fQQ73rCqkrLaPuWg69TGCE8Kyv9hsZLiY= +github.com/dolthub/go-mysql-server v0.17.1-0.20230927003048-44ab355c2753/go.mod h1:C0Q8nJKIEnCYXx+ot9F2V7bg7xz5wj+Z1o1BDyNnP7E= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514= github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488/go.mod h1:ehexgi1mPxRTk0Mok/pADALuHbvATulTh6gzr7NzZto= github.com/dolthub/jsonpath v0.0.2-0.20230525180605-8dc13778fd72 h1:NfWmngMi1CYUWU4Ix8wM+USEhjc+mhPlT9JUR/anvbQ=