diff --git a/docker/Dockerfile b/docker/Dockerfile index 8df6347664..5f42bcedba 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,6 +6,7 @@ ARG DOLT_VERSION RUN apt update -y && \ apt install -y \ curl \ + git \ tini \ ca-certificates && \ apt clean && \ diff --git a/docker/serverDockerfile b/docker/serverDockerfile index a794fc048f..50a3b7ae7d 100644 --- a/docker/serverDockerfile +++ b/docker/serverDockerfile @@ -4,7 +4,7 @@ FROM debian:bookworm-slim AS base ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ - curl tini ca-certificates && \ + curl git tini ca-certificates && \ rm -rf /var/lib/apt/lists/* diff --git a/go/cmd/dolt/cli/arg_parser_helpers.go b/go/cmd/dolt/cli/arg_parser_helpers.go index 4551cfa254..3f345dba40 100644 --- a/go/cmd/dolt/cli/arg_parser_helpers.go +++ b/go/cmd/dolt/cli/arg_parser_helpers.go @@ -61,6 +61,7 @@ func CreateCommitArgParser(supportsBranchFlag bool) *argparser.ArgParser { ap.SupportsFlag(UpperCaseAllFlag, "A", "Adds all tables and databases (including new tables) in the working set to the staged set.") ap.SupportsFlag(AmendFlag, "", "Amend previous commit") ap.SupportsOptionalString(SignFlag, "S", "key-id", "Sign the commit using GPG. If no key-id is provided the key-id is taken from 'user.signingkey' the in the configuration") + ap.SupportsFlag(SkipVerificationFlag, "", "Skip commit verification") if supportsBranchFlag { ap.SupportsString(BranchParam, "", "branch", "Commit to the specified branch instead of the current branch.") } @@ -96,6 +97,7 @@ func CreateMergeArgParser() *argparser.ArgParser { ap.SupportsFlag(NoCommitFlag, "", "Perform the merge and stop just before creating a merge commit. Note this will not prevent a fast-forward merge; use the --no-ff arg together with the --no-commit arg to prevent both fast-forwards and merge commits.") ap.SupportsFlag(NoEditFlag, "", "Use an auto-generated commit message when creating a merge commit. The default for interactive CLI sessions is to open an editor.") ap.SupportsString(AuthorParam, "", "author", "Specify an explicit author using the standard A U Thor {{.LessThan}}author@example.com{{.GreaterThan}} format.") + ap.SupportsFlag(SkipVerificationFlag, "", "Skip commit verification before merge") return ap } @@ -116,6 +118,7 @@ func CreateRebaseArgParser() *argparser.ArgParser { ap.SupportsFlag(AbortParam, "", "Abort an interactive rebase and return the working set to the pre-rebase state") ap.SupportsFlag(ContinueFlag, "", "Continue an interactive rebase after adjusting the rebase plan") ap.SupportsFlag(InteractiveFlag, "i", "Start an interactive rebase") + ap.SupportsFlag(SkipVerificationFlag, "", "Skip commit verification before rebase") return ap } @@ -174,6 +177,7 @@ func CreateRemoteArgParser() *argparser.ArgParser { func CreateCleanArgParser() *argparser.ArgParser { ap := argparser.NewArgParserWithVariableArgs("clean") ap.SupportsFlag(DryRunFlag, "", "Tests removing untracked tables without modifying the working set.") + ap.SupportsFlag(ExcludeIgnoreRulesFlag, "x", "Do not respect dolt_ignore; remove untracked tables that match dolt_ignore. dolt_nonlocal_tables is always respected.") return ap } @@ -192,6 +196,7 @@ func CreateCherryPickArgParser() *argparser.ArgParser { ap.SupportsFlag(AllowEmptyFlag, "", "Allow empty commits to be cherry-picked. "+ "Note that use of this option only keeps commits that were initially empty. "+ "Commits which become empty, due to a previous commit, will cause cherry-pick to fail.") + ap.SupportsFlag(SkipVerificationFlag, "", "Skip commit verification before cherry-pick") ap.TooManyArgsErrorFunc = func(receivedArgs []string) error { return errors.New("cherry-picking multiple commits is not supported yet.") } @@ -229,6 +234,7 @@ func CreatePullArgParser() *argparser.ArgParser { ap.SupportsString(UserFlag, "", "user", "User name to use when authenticating with the remote. Gets password from the environment variable {{.EmphasisLeft}}DOLT_REMOTE_PASSWORD{{.EmphasisRight}}.") ap.SupportsFlag(PruneFlag, "p", "After fetching, remove any remote-tracking references that don't exist on the remote.") ap.SupportsFlag(SilentFlag, "", "Suppress progress information.") + ap.SupportsFlag(SkipVerificationFlag, "", "Skip commit verification before merge") return ap } diff --git a/go/cmd/dolt/cli/flags.go b/go/cmd/dolt/cli/flags.go index 737ea9fc7c..c3e66ae0c2 100644 --- a/go/cmd/dolt/cli/flags.go +++ b/go/cmd/dolt/cli/flags.go @@ -17,77 +17,79 @@ package cli // Constants for command line flags names. These tend to be used in multiple places, so defining // them low in the package dependency tree makes sense. const ( - AbortParam = "abort" - AllFlag = "all" - AllowEmptyFlag = "allow-empty" - AmendFlag = "amend" - AuthorParam = "author" - ArchiveLevelParam = "archive-level" - BranchParam = "branch" - CachedFlag = "cached" - CheckoutCreateBranch = "b" - CreateResetBranch = "B" - CommitFlag = "commit" - ContinueFlag = "continue" - CopyFlag = "copy" - DateParam = "date" - DecorateFlag = "decorate" - DeleteFlag = "delete" - DeleteForceFlag = "D" - DepthFlag = "depth" - DryRunFlag = "dry-run" - EmptyParam = "empty" - ForceFlag = "force" - FullFlag = "full" - GraphFlag = "graph" - HardResetParam = "hard" - HostFlag = "host" - IncludeUntrackedFlag = "include-untracked" - InteractiveFlag = "interactive" - JobFlag = "job" - ListFlag = "list" - MergesFlag = "merges" - MessageArg = "message" - MinParentsFlag = "min-parents" - MoveFlag = "move" - NoCommitFlag = "no-commit" - NoEditFlag = "no-edit" - NoFFParam = "no-ff" - FFOnlyParam = "ff-only" - NoPrettyFlag = "no-pretty" - NoTLSFlag = "no-tls" - NoJsonMergeFlag = "dont-merge-json" - NotFlag = "not" - NumberFlag = "number" - OneLineFlag = "oneline" - OursFlag = "ours" - OutputOnlyFlag = "output-only" - ParentsFlag = "parents" - PatchFlag = "patch" - PasswordFlag = "password" - PortFlag = "port" - PruneFlag = "prune" - QuietFlag = "quiet" - RemoteParam = "remote" - SetUpstreamFlag = "set-upstream" - SetUpstreamToFlag = "set-upstream-to" - ShallowFlag = "shallow" - ShowIgnoredFlag = "ignored" - ShowSignatureFlag = "show-signature" - SignFlag = "gpg-sign" - SilentFlag = "silent" - SingleBranchFlag = "single-branch" - SkipEmptyFlag = "skip-empty" - SoftResetParam = "soft" - SquashParam = "squash" - StagedFlag = "staged" - StatFlag = "stat" - SystemFlag = "system" - TablesFlag = "tables" - TheirsFlag = "theirs" - TrackFlag = "track" - UpperCaseAllFlag = "ALL" - UserFlag = "user" + AbortParam = "abort" + AllFlag = "all" + AllowEmptyFlag = "allow-empty" + AmendFlag = "amend" + AuthorParam = "author" + ArchiveLevelParam = "archive-level" + BranchParam = "branch" + CachedFlag = "cached" + CheckoutCreateBranch = "b" + CreateResetBranch = "B" + CommitFlag = "commit" + ContinueFlag = "continue" + CopyFlag = "copy" + DateParam = "date" + DecorateFlag = "decorate" + DeleteFlag = "delete" + DeleteForceFlag = "D" + DepthFlag = "depth" + DryRunFlag = "dry-run" + EmptyParam = "empty" + ExcludeIgnoreRulesFlag = "x" + ForceFlag = "force" + FullFlag = "full" + GraphFlag = "graph" + HardResetParam = "hard" + HostFlag = "host" + IncludeUntrackedFlag = "include-untracked" + InteractiveFlag = "interactive" + JobFlag = "job" + ListFlag = "list" + MergesFlag = "merges" + MessageArg = "message" + MinParentsFlag = "min-parents" + MoveFlag = "move" + NoCommitFlag = "no-commit" + NoEditFlag = "no-edit" + NoFFParam = "no-ff" + FFOnlyParam = "ff-only" + NoPrettyFlag = "no-pretty" + NoTLSFlag = "no-tls" + NoJsonMergeFlag = "dont-merge-json" + NotFlag = "not" + NumberFlag = "number" + OneLineFlag = "oneline" + OursFlag = "ours" + OutputOnlyFlag = "output-only" + ParentsFlag = "parents" + PatchFlag = "patch" + PasswordFlag = "password" + PortFlag = "port" + PruneFlag = "prune" + QuietFlag = "quiet" + RemoteParam = "remote" + SetUpstreamFlag = "set-upstream" + SetUpstreamToFlag = "set-upstream-to" + ShallowFlag = "shallow" + ShowIgnoredFlag = "ignored" + ShowSignatureFlag = "show-signature" + SignFlag = "gpg-sign" + SilentFlag = "silent" + SingleBranchFlag = "single-branch" + SkipEmptyFlag = "skip-empty" + SkipVerificationFlag = "skip-verification" + SoftResetParam = "soft" + SquashParam = "squash" + StagedFlag = "staged" + StatFlag = "stat" + SystemFlag = "system" + TablesFlag = "tables" + TheirsFlag = "theirs" + TrackFlag = "track" + UpperCaseAllFlag = "ALL" + UserFlag = "user" ) // Flags used by `dolt diff` command and `dolt_diff()` table function. diff --git a/go/cmd/dolt/commands/clean.go b/go/cmd/dolt/commands/clean.go index 73e0fd0291..885e4fe08b 100644 --- a/go/cmd/dolt/commands/clean.go +++ b/go/cmd/dolt/commands/clean.go @@ -32,16 +32,17 @@ const ( var cleanDocContent = cli.CommandDocumentationContent{ ShortDesc: "Deletes untracked working tables", - LongDesc: "{{.EmphasisLeft}}dolt clean [--dry-run]{{.EmphasisRight}}\n\n" + + LongDesc: "{{.EmphasisLeft}}dolt clean [--dry-run] [-x]{{.EmphasisRight}}\n\n" + "The default (parameterless) form clears the values for all untracked working {{.LessThan}}tables{{.GreaterThan}} ." + - "This command permanently deletes unstaged or uncommitted tables.\n\n" + + "This command permanently deletes unstaged or uncommitted tables. By default, tables matching dolt_ignore or dolt_nonlocal_tables are not removed.\n\n" + "The {{.EmphasisLeft}}--dry-run{{.EmphasisRight}} flag can be used to test whether the clean can succeed without " + "deleting any tables from the current working set.\n\n" + - "{{.EmphasisLeft}}dolt clean [--dry-run] {{.LessThan}}tables{{.GreaterThan}}...{{.EmphasisRight}}\n\n" + + "The {{.EmphasisLeft}}-x{{.EmphasisRight}} flag causes dolt_ignore to be ignored so that untracked tables matching dolt_ignore are removed; dolt_nonlocal_tables is always respected (similar to git clean -x).\n\n" + + "{{.EmphasisLeft}}dolt clean [--dry-run] [-x] {{.LessThan}}tables{{.GreaterThan}}...{{.EmphasisRight}}\n\n" + "If {{.LessThan}}tables{{.GreaterThan}} is specified, only those table names are considered for deleting.\n\n", Synopsis: []string{ - "[--dry-run]", - "[--dry-run] {{.LessThan}}tables{{.GreaterThan}}...", + "[--dry-run] [-x]", + "[--dry-run] [-x] {{.LessThan}}tables{{.GreaterThan}}...", }, } @@ -87,6 +88,13 @@ func (cmd CleanCmd) Exec(ctx context.Context, commandStr string, args []string, buffer.WriteString("\"--dry-run\"") firstParamDone = true } + if apr.Contains(cli.ExcludeIgnoreRulesFlag) { + if firstParamDone { + buffer.WriteString(", ") + } + buffer.WriteString("\"-x\"") + firstParamDone = true + } if apr.NArg() > 0 { // loop over apr.Args() and add them to the buffer for i := 0; i < apr.NArg(); i++ { diff --git a/go/cmd/dolt/commands/commit.go b/go/cmd/dolt/commands/commit.go index 738bc54e62..23258c148e 100644 --- a/go/cmd/dolt/commands/commit.go +++ b/go/cmd/dolt/commands/commit.go @@ -266,6 +266,10 @@ func constructParametrizedDoltCommitQuery(msg string, apr *argparser.ArgParseRes writeToBuffer("--skip-empty") } + if apr.Contains(cli.SkipVerificationFlag) { + writeToBuffer("--skip-verification") + } + cfgSign := cliCtx.Config().GetStringOrDefault("sqlserver.global.gpgsign", "") if apr.Contains(cli.SignFlag) || strings.ToLower(cfgSign) == "true" { writeToBuffer("--gpg-sign") diff --git a/go/cmd/dolt/commands/diff.go b/go/cmd/dolt/commands/diff.go index 44a5b3e891..6a863a889c 100644 --- a/go/cmd/dolt/commands/diff.go +++ b/go/cmd/dolt/commands/diff.go @@ -26,7 +26,6 @@ import ( "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/vitess/go/sqltypes" ast "github.com/dolthub/vitess/go/vt/sqlparser" - "github.com/go-sql-driver/mysql" "github.com/gocraft/dbr/v2" "github.com/gocraft/dbr/v2/dialect" @@ -549,20 +548,6 @@ func getTableNamesAtRef(queryist cli.Queryist, sqlCtx *sql.Context, ref string) return tableNames, nil } -func isTableNotFoundError(err error) bool { - if sql.ErrTableNotFound.Is(err) { - return true - } - mse, ok := err.(*mysql.MySQLError) - if ok { - if strings.HasPrefix(mse.Message, "table not found:") { - return true - } - } - - return false -} - // applyDiffRoots applies the appropriate |from| and |to| root values to the receiver and returns the table names // (if any) given to the command. func (dArgs *diffArgs) applyDiffRoots(queryist cli.Queryist, sqlCtx *sql.Context, args []string, isCached, useMergeBase bool) ([]string, error) { diff --git a/go/cmd/dolt/commands/dump.go b/go/cmd/dolt/commands/dump.go index 2acdfa8d3e..d7a0c771ce 100644 --- a/go/cmd/dolt/commands/dump.go +++ b/go/cmd/dolt/commands/dump.go @@ -623,16 +623,6 @@ func dumpTable(ctx *sql.Context, dEnv *env.DoltEnv, engine *sqle.Engine, root do } func getTableWriter(ctx context.Context, dEnv *env.DoltEnv, tblOpts *tableOptions, outSch schema.Schema, filePath string) (table.SqlRowWriter, errhand.VerboseError) { - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, errhand.BuildDError("error: ").AddCause(err).Build() - } - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return nil, errhand.BuildDError("error: ").AddCause(err).Build() - } - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - writer, err := dEnv.FS.OpenForWriteAppend(filePath, os.ModePerm) if err != nil { return nil, errhand.BuildDError("Error opening writer for %s.", tblOpts.DestName()).AddCause(err).Build() @@ -643,7 +633,7 @@ func getTableWriter(ctx context.Context, dEnv *env.DoltEnv, tblOpts *tableOption return nil, errhand.BuildDError("Could not create table writer for %s", tblOpts.tableName).AddCause(err).Build() } - wr, err := tblOpts.dest.NewCreatingWriter(ctx, tblOpts, root, outSch, opts, writer) + wr, err := tblOpts.dest.NewCreatingWriter(ctx, tblOpts, root, outSch, editor.Options{}, writer) if err != nil { return nil, errhand.BuildDError("Could not create table writer for %s", tblOpts.tableName).AddCause(err).Build() } diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index 64c45e60c3..5eccafdd95 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -143,7 +143,7 @@ func NewSqlEngine( }) } - dbs, locations, err := CollectDBs(ctx, mrEnv, config.Bulk) + dbs, locations, err := CollectDBs(ctx, mrEnv) if err != nil { return nil, err } diff --git a/go/cmd/dolt/commands/engine/utils.go b/go/cmd/dolt/commands/engine/utils.go index 425e6aa01d..b775e18d5a 100644 --- a/go/cmd/dolt/commands/engine/utils.go +++ b/go/cmd/dolt/commands/engine/utils.go @@ -26,13 +26,13 @@ import ( // CollectDBs takes a MultiRepoEnv and creates Database objects from each environment and returns a slice of these // objects. -func CollectDBs(ctx context.Context, mrEnv *env.MultiRepoEnv, useBulkEditor bool) ([]dsess.SqlDatabase, []filesys.Filesys, error) { +func CollectDBs(ctx context.Context, mrEnv *env.MultiRepoEnv) ([]dsess.SqlDatabase, []filesys.Filesys, error) { var dbs []dsess.SqlDatabase var locations []filesys.Filesys var db dsess.SqlDatabase err := mrEnv.Iter(func(name string, dEnv *env.DoltEnv) (stop bool, err error) { - db, err = newDatabase(ctx, name, dEnv, useBulkEditor) + db, err = newDatabase(ctx, name, dEnv) if err != nil { return false, err } @@ -50,25 +50,7 @@ func CollectDBs(ctx context.Context, mrEnv *env.MultiRepoEnv, useBulkEditor bool return dbs, locations, nil } -func newDatabase(ctx context.Context, name string, dEnv *env.DoltEnv, useBulkEditor bool) (sqle.Database, error) { - var deaf editor.DbEaFactory - var err error - if useBulkEditor { - deaf, err = dEnv.BulkDbEaFactory(ctx) - } else { - deaf, err = dEnv.DbEaFactory(ctx) - } - if err != nil { - return sqle.Database{}, err - } - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return sqle.Database{}, err - } - opts := editor.Options{ - Deaf: deaf, - Tempdir: tmpDir, - } +func newDatabase(ctx context.Context, name string, dEnv *env.DoltEnv) (sqle.Database, error) { dbdata := dEnv.DbData(ctx) // Databases registered with the SQL engine are always // configured for FatalBehaviorCrash. These are local @@ -81,5 +63,5 @@ func newDatabase(ctx context.Context, name string, dEnv *env.DoltEnv, useBulkEdi // See also sqle/database_provider.go, where we do this when // creating new databases as well. dbdata.Ddb.SetCrashOnFatalError() - return sqle.NewDatabase(ctx, name, dbdata, opts) + return sqle.NewDatabase(ctx, name, dbdata, editor.Options{}) } diff --git a/go/cmd/dolt/commands/filter-branch.go b/go/cmd/dolt/commands/filter-branch.go index ba09c13e1a..c8b12e5e3a 100644 --- a/go/cmd/dolt/commands/filter-branch.go +++ b/go/cmd/dolt/commands/filter-branch.go @@ -335,16 +335,7 @@ func processFilterQuery(ctx context.Context, dEnv *env.DoltEnv, root doltdb.Root // we set manually with the one at the working set of the HEAD being rebased. // Some functionality will not work on this kind of engine, e.g. many DOLT_ functions. func rebaseSqlEngine(ctx context.Context, dEnv *env.DoltEnv, root doltdb.RootValue) (*sql.Context, *engine.SqlEngine, error) { - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, nil, err - } - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return nil, nil, err - } - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := dsqle.NewDatabase(ctx, filterDbName, dEnv.DbData(ctx), opts) + db, err := dsqle.NewDatabase(ctx, filterDbName, dEnv.DbData(ctx), editor.Options{}) if err != nil { return nil, nil, err } diff --git a/go/cmd/dolt/commands/indexcmds/rebuild.go b/go/cmd/dolt/commands/indexcmds/rebuild.go index b2f1e1b2f1..df4d711876 100644 --- a/go/cmd/dolt/commands/indexcmds/rebuild.go +++ b/go/cmd/dolt/commands/indexcmds/rebuild.go @@ -87,15 +87,7 @@ func (cmd RebuildCmd) Exec(ctx context.Context, commandStr string, args []string if !ok { return HandleErr(errhand.BuildDError("The table `%s` does not exist.", tableName).Build(), nil) } - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return HandleErr(errhand.BuildDError("error: ").AddCause(err).Build(), nil) - } - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return HandleErr(errhand.BuildDError("error: ").AddCause(err).Build(), nil) - } - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} + sch, err := table.GetSchema(ctx) if err != nil { return HandleErr(errhand.BuildDError("could not get table schema").AddCause(err).Build(), nil) @@ -104,7 +96,7 @@ func (cmd RebuildCmd) Exec(ctx context.Context, commandStr string, args []string if idxSch == nil { return HandleErr(errhand.BuildDError("the index `%s` does not exist on table `%s`", indexName, tableName).Build(), nil) } - indexRowData, err := creation.BuildSecondaryIndex(sql.NewContext(ctx), table, idxSch, tableName, opts) + indexRowData, err := creation.BuildSecondaryIndex(sql.NewContext(ctx), table, idxSch, tableName, editor.Options{}) if err != nil { return HandleErr(errhand.BuildDError("Unable to rebuild index `%s` on table `%s`.", indexName, tableName).AddCause(err).Build(), nil) } diff --git a/go/cmd/dolt/commands/merge.go b/go/cmd/dolt/commands/merge.go index d904657fe0..c28e9c65a7 100644 --- a/go/cmd/dolt/commands/merge.go +++ b/go/cmd/dolt/commands/merge.go @@ -318,6 +318,10 @@ func constructInterpolatedDoltMergeQuery(apr *argparser.ArgParseResults, cliCtx params = append(params, msg) } + if apr.Contains(cli.SkipVerificationFlag) { + writeToBuffer("--skip-verification", false) + } + if !apr.Contains(cli.AbortParam) && !apr.Contains(cli.SquashParam) { writeToBuffer("?", true) params = append(params, apr.Arg(0)) diff --git a/go/cmd/dolt/commands/schcmds/export.go b/go/cmd/dolt/commands/schcmds/export.go index 955b66a72f..075c9fa0b1 100644 --- a/go/cmd/dolt/commands/schcmds/export.go +++ b/go/cmd/dolt/commands/schcmds/export.go @@ -134,16 +134,7 @@ func exportSchemas(ctx context.Context, apr *argparser.ArgParseResults, root dol } for _, tn := range tablesToExport { - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return errhand.BuildDError("error: ").AddCause(err).Build() - } - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return errhand.BuildDError("error: ").AddCause(err).Build() - } - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - verr := exportTblSchema(ctx, tn, root, wr, opts) + verr := exportTblSchema(ctx, tn, root, wr, editor.Options{}) if verr != nil { return verr } diff --git a/go/cmd/dolt/commands/schcmds/show.go b/go/cmd/dolt/commands/schcmds/show.go index d2a31892f0..cef6aca02b 100644 --- a/go/cmd/dolt/commands/schcmds/show.go +++ b/go/cmd/dolt/commands/schcmds/show.go @@ -133,16 +133,7 @@ func printSchemas(ctx context.Context, apr *argparser.ArgParseResults, dEnv *env } } - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return errhand.BuildDError("error: ").AddCause(err).Build() - } - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return errhand.BuildDError("error: ").AddCause(err).Build() - } - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - sqlCtx, engine, _ := dsqle.PrepareCreateTableStmt(ctx, dsqle.NewUserSpaceDatabase(root, opts)) + sqlCtx, engine, _ := dsqle.PrepareCreateTableStmt(ctx, dsqle.NewUserSpaceDatabase(root, editor.Options{})) var notFound []string for _, tblName := range tables { diff --git a/go/cmd/dolt/commands/sqlserver/mcp.go b/go/cmd/dolt/commands/sqlserver/mcp.go index c7b90f9538..c5bea10536 100644 --- a/go/cmd/dolt/commands/sqlserver/mcp.go +++ b/go/cmd/dolt/commands/sqlserver/mcp.go @@ -156,6 +156,9 @@ func mcpRun(cfg *Config, lgr *logrus.Logger, state *svcs.ServiceState, cancelPtr logger, dbConf, *cfg.MCP.Port, + nil, // jwkClaimsMap + "", // jwkUrl + nil, // tlsConfig toolsets.WithToolSet(&toolsets.PrimitiveToolSetV1{}), ) if err != nil { diff --git a/go/cmd/dolt/commands/tblcmds/export.go b/go/cmd/dolt/commands/tblcmds/export.go index 89de84c5e9..a4e4d76b96 100644 --- a/go/cmd/dolt/commands/tblcmds/export.go +++ b/go/cmd/dolt/commands/tblcmds/export.go @@ -258,11 +258,7 @@ func getTableWriter(ctx context.Context, root doltdb.RootValue, dEnv *env.DoltEn return nil, errhand.BuildDError("Error opening writer for %s.", exOpts.DestName()).AddCause(err).Build() } - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return nil, errhand.BuildDError("Error opening writer for %s.", exOpts.DestName()).AddCause(err).Build() - } - wr, err := exOpts.dest.NewCreatingWriter(ctx, exOpts, root, rdSchema, editor.Options{Deaf: deaf}, writer) + wr, err := exOpts.dest.NewCreatingWriter(ctx, exOpts, root, rdSchema, editor.Options{}, writer) if err != nil { return nil, errhand.BuildDError("Error opening writer for %s.", exOpts.DestName()).AddCause(err).Build() } diff --git a/go/cmd/dolt/doltversion/version.go b/go/cmd/dolt/doltversion/version.go index 5d69ded2b4..f75dc9232e 100644 --- a/go/cmd/dolt/doltversion/version.go +++ b/go/cmd/dolt/doltversion/version.go @@ -15,5 +15,5 @@ package doltversion const ( - Version = "1.81.8" + Version = "1.81.10" ) diff --git a/go/gen/fb/serial/workingset.go b/go/gen/fb/serial/workingset.go index ec71849a2a..baea81dc77 100644 --- a/go/gen/fb/serial/workingset.go +++ b/go/gen/fb/serial/workingset.go @@ -579,7 +579,19 @@ func (rcv *RebaseState) MutateRebasingStarted(n bool) bool { return rcv._tab.MutateBoolSlot(16, n) } -const RebaseStateNumFields = 7 +func (rcv *RebaseState) SkipVerification() bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(18)) + if o != 0 { + return rcv._tab.GetBool(o + rcv._tab.Pos) + } + return false +} + +func (rcv *RebaseState) MutateSkipVerification(n bool) bool { + return rcv._tab.MutateBoolSlot(18, n) +} + +const RebaseStateNumFields = 8 func RebaseStateStart(builder *flatbuffers.Builder) { builder.StartObject(RebaseStateNumFields) @@ -614,6 +626,9 @@ func RebaseStateAddLastAttemptedStep(builder *flatbuffers.Builder, lastAttempted func RebaseStateAddRebasingStarted(builder *flatbuffers.Builder, rebasingStarted bool) { builder.PrependBoolSlot(6, rebasingStarted, false) } +func RebaseStateAddSkipVerification(builder *flatbuffers.Builder, skipVerification bool) { + builder.PrependBoolSlot(7, skipVerification, false) +} func RebaseStateEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } diff --git a/go/go.mod b/go/go.mod index be773f81ad..42e92cf45c 100644 --- a/go/go.mod +++ b/go/go.mod @@ -58,10 +58,10 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 - github.com/dolthub/dolt-mcp v0.2.2 + github.com/dolthub/dolt-mcp v0.3.4 github.com/dolthub/eventsapi_schema v0.0.0-20260205214132-a7a3c84c84a1 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.20.1-0.20260211220532-85072e590dc7 + github.com/dolthub/go-mysql-server v0.20.1-0.20260212215527-0cb492ad7051 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/edsrzf/mmap-go v1.2.0 github.com/esote/minmaxheap v1.0.0 diff --git a/go/go.sum b/go/go.sum index b0a87f228e..e025102fc0 100644 --- a/go/go.sum +++ b/go/go.sum @@ -186,8 +186,8 @@ github.com/denisenkom/go-mssqldb v0.10.0 h1:QykgLZBorFE95+gO3u9esLd0BmbvpWp0/waN github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 h1:IdqX7J8vi/Kn3T3Ee0VzqnLqwFmgA2hr8WZETPcQjfM= github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12/go.mod h1:rN7X8BHwkjPcfMQQ2QTAq/xM3leUSGLfb+1Js7Y6TVo= -github.com/dolthub/dolt-mcp v0.2.2 h1:bpROmam74n95uU4EA3BpOIVlTDT0pzeFMBwe/YRq2mI= -github.com/dolthub/dolt-mcp v0.2.2/go.mod h1:S++DJ4QWTAXq+0TNzFa7Oq3IhoT456DJHwAINFAHgDQ= +github.com/dolthub/dolt-mcp v0.3.4 h1:AyG5cw+fNWXDHXujtQnqUPZrpWtPg6FN6yYtjv1pP44= +github.com/dolthub/dolt-mcp v0.3.4/go.mod h1:bCZ7KHvDYs+M0e+ySgmGiNvLhcwsN7bbf5YCyillLrk= github.com/dolthub/eventsapi_schema v0.0.0-20260205214132-a7a3c84c84a1 h1:QePoMpa5qlquwUqRVyF9KAHsJAlYbE2+eZkMPAxeBXc= github.com/dolthub/eventsapi_schema v0.0.0-20260205214132-a7a3c84c84a1/go.mod h1:evuptFmr/0/j0X/g+3cveHEEOM5tqyRA15FNgirtOY0= github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 h1:u3PMzfF8RkKd3lB9pZ2bfn0qEG+1Gms9599cr0REMww= @@ -196,8 +196,8 @@ github.com/dolthub/fslock v0.0.0-20251215194149-ef20baba2318 h1:n+vdH5G5Db+1qnDC github.com/dolthub/fslock v0.0.0-20251215194149-ef20baba2318/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20250916051405-78a38d478790 h1:zxMsH7RLiG+dlZ/y0LgJHTV26XoiSJcuWq+em6t6VVc= github.com/dolthub/go-icu-regex v0.0.0-20250916051405-78a38d478790/go.mod h1:F3cnm+vMRK1HaU6+rNqQrOCyR03HHhR1GWG2gnPOqaE= -github.com/dolthub/go-mysql-server v0.20.1-0.20260211220532-85072e590dc7 h1:9xC+/i949mi2wwsu6BKgvnDnuRcYy4KysrIb2x7DaSo= -github.com/dolthub/go-mysql-server v0.20.1-0.20260211220532-85072e590dc7/go.mod h1:LEWdXw6LKjdonOv2X808RpUc8wZVtQx4ZEPvmDWkvY4= +github.com/dolthub/go-mysql-server v0.20.1-0.20260212215527-0cb492ad7051 h1:7vNnl/Z2HhFFUTdXNOySd8KFODBztPlmCITrRIKDgTw= +github.com/dolthub/go-mysql-server v0.20.1-0.20260212215527-0cb492ad7051/go.mod h1:LEWdXw6LKjdonOv2X808RpUc8wZVtQx4ZEPvmDWkvY4= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= diff --git a/go/libraries/doltcore/cherry_pick/cherry_pick.go b/go/libraries/doltcore/cherry_pick/cherry_pick.go index c8012218a2..e7fd3e6405 100644 --- a/go/libraries/doltcore/cherry_pick/cherry_pick.go +++ b/go/libraries/doltcore/cherry_pick/cherry_pick.go @@ -52,6 +52,9 @@ type CherryPickOptions struct { // and Dolt cherry-pick implementations, the default action is to fail when an empty commit is specified. In Git // and Dolt rebase implementations, the default action is to keep commits that start off as empty. EmptyCommitHandling doltdb.EmptyCommitHandling + + // SkipVerification controls whether test validation should be skipped before creating commits. + SkipVerification bool } // NewCherryPickOptions creates a new CherryPickOptions instance, filled out with default values for cherry-pick. @@ -61,6 +64,7 @@ func NewCherryPickOptions() CherryPickOptions { CommitMessage: "", CommitBecomesEmptyHandling: doltdb.ErrorOnEmptyCommit, EmptyCommitHandling: doltdb.ErrorOnEmptyCommit, + SkipVerification: false, } } @@ -159,9 +163,10 @@ func CreateCommitStagedPropsFromCherryPickOptions(ctx *sql.Context, options Cher } commitProps := actions.CommitStagedProps{ - Date: originalMeta.Time(), - Name: originalMeta.Name, - Email: originalMeta.Email, + Date: originalMeta.Time(), + Name: originalMeta.Name, + Email: originalMeta.Email, + SkipVerification: options.SkipVerification, } if options.CommitMessage != "" { diff --git a/go/libraries/doltcore/conflict/conflict.go b/go/libraries/doltcore/conflict/conflict.go deleted file mode 100644 index 9e793b54d6..0000000000 --- a/go/libraries/doltcore/conflict/conflict.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2019 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package conflict - -import ( - "context" - "errors" - - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding" - "github.com/dolthub/dolt/go/store/types" -) - -type ConflictSchema struct { - Base schema.Schema - Schema schema.Schema - MergeSchema schema.Schema -} - -func NewConflictSchema(base, sch, mergeSch schema.Schema) ConflictSchema { - return ConflictSchema{ - Base: base, - Schema: sch, - MergeSchema: mergeSch, - } -} - -func ValueFromConflictSchema(ctx context.Context, vrw types.ValueReadWriter, cs ConflictSchema) (types.Value, error) { - b, err := serializeSchema(ctx, vrw, cs.Base) - if err != nil { - return nil, err - } - - s, err := serializeSchema(ctx, vrw, cs.Schema) - if err != nil { - return nil, err - } - - m, err := serializeSchema(ctx, vrw, cs.MergeSchema) - if err != nil { - return nil, err - } - - return types.NewTuple(vrw.Format(), b, s, m) -} - -func ConflictSchemaFromValue(ctx context.Context, vrw types.ValueReadWriter, v types.Value) (cs ConflictSchema, err error) { - tup, ok := v.(types.Tuple) - if !ok { - err = errors.New("conflict schema value must be types.Struct") - return ConflictSchema{}, err - } - - b, err := tup.Get(0) - if err != nil { - return ConflictSchema{}, err - } - cs.Base, err = deserializeSchema(ctx, vrw, b) - if err != nil { - return ConflictSchema{}, err - } - - s, err := tup.Get(1) - if err != nil { - return ConflictSchema{}, err - } - cs.Schema, err = deserializeSchema(ctx, vrw, s) - if err != nil { - return ConflictSchema{}, err - } - - m, err := tup.Get(2) - if err != nil { - return ConflictSchema{}, err - } - cs.MergeSchema, err = deserializeSchema(ctx, vrw, m) - if err != nil { - return ConflictSchema{}, err - } - - return cs, nil -} - -func serializeSchema(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema) (types.Ref, error) { - st, err := encoding.MarshalSchema(ctx, vrw, sch) - if err != nil { - return types.Ref{}, err - } - - return vrw.WriteValue(ctx, st) -} - -func deserializeSchema(ctx context.Context, vrw types.ValueReadWriter, v types.Value) (schema.Schema, error) { - r, ok := v.(types.Ref) - if !ok { - return nil, errors.New("conflict schemas field value is unexpected type") - } - - return encoding.UnmarshalSchemaAtAddr(ctx, vrw, r.TargetHash()) -} - -type Conflict struct { - Base types.Value - Value types.Value - MergeValue types.Value -} - -func NewConflict(base, value, mergeValue types.Value) Conflict { - if base == nil { - base = types.NullValue - } - if value == nil { - value = types.NullValue - } - if mergeValue == nil { - mergeValue = types.NullValue - } - return Conflict{base, value, mergeValue} -} - -func ConflictFromTuple(tpl types.Tuple) (Conflict, error) { - base, err := tpl.Get(0) - - if err != nil { - return Conflict{}, err - } - - val, err := tpl.Get(1) - - if err != nil { - return Conflict{}, err - } - - mv, err := tpl.Get(2) - - if err != nil { - return Conflict{}, err - } - return Conflict{base, val, mv}, nil -} - -func (c Conflict) ToNomsList(vrw types.ValueReadWriter) (types.Tuple, error) { - return types.NewTuple(vrw.Format(), c.Base, c.Value, c.MergeValue) -} diff --git a/go/libraries/doltcore/diff/async_differ.go b/go/libraries/doltcore/diff/async_differ.go deleted file mode 100644 index 046feba32c..0000000000 --- a/go/libraries/doltcore/diff/async_differ.go +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright 2019 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package diff - -import ( - "context" - "fmt" - "time" - - "golang.org/x/sync/errgroup" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/utils/async" - "github.com/dolthub/dolt/go/store/diff" - "github.com/dolthub/dolt/go/store/types" -) - -// todo: make package private -type AsyncDiffer struct { - diffChan chan diff.Difference - bufferSize int - - eg *errgroup.Group - egCtx context.Context - egCancel func() - - diffStats map[types.DiffChangeType]uint64 -} - -var _ RowDiffer = &AsyncDiffer{} - -// todo: make package private once dolthub is migrated -func NewAsyncDiffer(bufferedDiffs int) *AsyncDiffer { - return &AsyncDiffer{ - diffChan: make(chan diff.Difference, bufferedDiffs), - bufferSize: bufferedDiffs, - egCtx: context.Background(), - egCancel: func() {}, - diffStats: make(map[types.DiffChangeType]uint64), - } -} - -func tableDontDescendLists(v1, v2 types.Value) bool { - kind := v1.Kind() - return !types.IsPrimitiveKind(kind) && kind != types.TupleKind && kind == v2.Kind() && kind != types.RefKind -} - -func (ad *AsyncDiffer) Start(ctx context.Context, from, to types.Map) { - ad.start(ctx, func(ctx context.Context) error { - return diff.Diff(ctx, from, to, ad.diffChan, true, tableDontDescendLists) - }) -} - -func (ad *AsyncDiffer) StartWithRange(ctx context.Context, from, to types.Map, start types.Value, inRange types.ValueInRange) { - ad.start(ctx, func(ctx context.Context) error { - return diff.DiffMapRange(ctx, from, to, start, inRange, ad.diffChan, true, tableDontDescendLists) - }) -} - -func (ad *AsyncDiffer) start(ctx context.Context, diffFunc func(ctx context.Context) error) { - ad.eg, ad.egCtx = errgroup.WithContext(ctx) - ad.egCancel = async.GoWithCancel(ad.egCtx, ad.eg, func(ctx context.Context) (err error) { - defer close(ad.diffChan) - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic in diff.Diff: %v", r) - } - }() - return diffFunc(ctx) - }) -} - -func (ad *AsyncDiffer) Close() error { - ad.egCancel() - return ad.eg.Wait() -} - -func (ad *AsyncDiffer) getDiffs(numDiffs int, timeoutChan <-chan time.Time, pred diffPredicate) ([]*diff.Difference, bool, error) { - diffs := make([]*diff.Difference, 0, numDiffs) - for { - select { - case d, more := <-ad.diffChan: - if more { - if pred(&d) { - ad.diffStats[d.ChangeType]++ - diffs = append(diffs, &d) - } - if numDiffs != 0 && numDiffs == len(diffs) { - return diffs, true, nil - } - } else { - return diffs, false, ad.eg.Wait() - } - case <-timeoutChan: - return diffs, true, nil - case <-ad.egCtx.Done(): - return nil, false, ad.eg.Wait() - } - } -} - -var forever <-chan time.Time = make(chan time.Time) - -type diffPredicate func(*diff.Difference) bool - -var alwaysTruePredicate diffPredicate = func(*diff.Difference) bool { - return true -} - -func hasChangeTypePredicate(changeType types.DiffChangeType) diffPredicate { - return func(d *diff.Difference) bool { - return d.ChangeType == changeType - } -} - -func (ad *AsyncDiffer) GetDiffs(numDiffs int, timeout time.Duration) ([]*diff.Difference, bool, error) { - if timeout < 0 { - return ad.GetDiffsWithoutTimeout(numDiffs) - } - return ad.getDiffs(numDiffs, time.After(timeout), alwaysTruePredicate) -} - -func (ad *AsyncDiffer) GetDiffsWithFilter(numDiffs int, timeout time.Duration, filterByChangeType types.DiffChangeType) ([]*diff.Difference, bool, error) { - if timeout < 0 { - return ad.GetDiffsWithoutTimeoutWithFilter(numDiffs, filterByChangeType) - } - return ad.getDiffs(numDiffs, time.After(timeout), hasChangeTypePredicate(filterByChangeType)) -} - -func (ad *AsyncDiffer) GetDiffsWithoutTimeoutWithFilter(numDiffs int, filterByChangeType types.DiffChangeType) ([]*diff.Difference, bool, error) { - return ad.getDiffs(numDiffs, forever, hasChangeTypePredicate(filterByChangeType)) -} - -func (ad *AsyncDiffer) GetDiffsWithoutTimeout(numDiffs int) ([]*diff.Difference, bool, error) { - return ad.getDiffs(numDiffs, forever, alwaysTruePredicate) -} - -type keylessDiffer struct { - *AsyncDiffer - - df diff.Difference - copiesLeft uint64 -} - -var _ RowDiffer = &keylessDiffer{} - -func (kd *keylessDiffer) getDiffs(numDiffs int, timeoutChan <-chan time.Time, pred diffPredicate) ([]*diff.Difference, bool, error) { - diffs := make([]*diff.Difference, numDiffs) - idx := 0 - - for { - // first populate |diffs| with copies of |kd.df| - - cpy := kd.df // save a copy of kd.df to reference - for (idx < numDiffs) && (kd.copiesLeft > 0) { - diffs[idx] = &cpy - idx++ - kd.copiesLeft-- - } - if idx == numDiffs { - return diffs, true, nil - } - - // then find the next Difference the satisfies |pred| - match := false - for !match { - select { - case <-timeoutChan: - return diffs, true, nil - - case <-kd.egCtx.Done(): - return nil, false, kd.eg.Wait() - - case d, more := <-kd.diffChan: - if !more { - return diffs[:idx], more, nil - } - - var err error - kd.df, kd.copiesLeft, err = convertDiff(d) - if err != nil { - return nil, false, err - } - - match = pred(&kd.df) - } - } - } -} - -func (kd *keylessDiffer) GetDiffs(numDiffs int, timeout time.Duration) ([]*diff.Difference, bool, error) { - if timeout < 0 { - return kd.getDiffs(numDiffs, forever, alwaysTruePredicate) - } - return kd.getDiffs(numDiffs, time.After(timeout), alwaysTruePredicate) -} - -func (kd *keylessDiffer) GetDiffsWithFilter(numDiffs int, timeout time.Duration, filterByChangeType types.DiffChangeType) ([]*diff.Difference, bool, error) { - if timeout < 0 { - return kd.getDiffs(numDiffs, forever, hasChangeTypePredicate(filterByChangeType)) - } - return kd.getDiffs(numDiffs, time.After(timeout), hasChangeTypePredicate(filterByChangeType)) -} - -// convertDiff reports the cardinality of a change, -// and converts updates to adds or deletes -func convertDiff(df diff.Difference) (diff.Difference, uint64, error) { - var oldCard uint64 - if df.OldValue != nil { - v, err := df.OldValue.(types.Tuple).Get(row.KeylessCardinalityValIdx) - if err != nil { - return df, 0, err - } - oldCard = uint64(v.(types.Uint)) - } - - var newCard uint64 - if df.NewValue != nil { - v, err := df.NewValue.(types.Tuple).Get(row.KeylessCardinalityValIdx) - if err != nil { - return df, 0, err - } - newCard = uint64(v.(types.Uint)) - } - - switch df.ChangeType { - case types.DiffChangeRemoved: - return df, oldCard, nil - - case types.DiffChangeAdded: - return df, newCard, nil - - case types.DiffChangeModified: - delta := int64(newCard) - int64(oldCard) - if delta > 0 { - df.ChangeType = types.DiffChangeAdded - df.OldValue = nil - return df, uint64(delta), nil - } else if delta < 0 { - df.ChangeType = types.DiffChangeRemoved - df.NewValue = nil - return df, uint64(-delta), nil - } else { - panic(fmt.Sprintf("diff with delta = 0 for key: %s", df.KeyValue.HumanReadableString())) - } - default: - return df, 0, fmt.Errorf("unexpected DiffChange type %d", df.ChangeType) - } -} - -type EmptyRowDiffer struct { -} - -var _ RowDiffer = &EmptyRowDiffer{} - -func (e EmptyRowDiffer) Start(ctx context.Context, from, to types.Map) { -} - -func (e EmptyRowDiffer) StartWithRange(ctx context.Context, from, to types.Map, start types.Value, inRange types.ValueInRange) { - -} - -func (e EmptyRowDiffer) GetDiffs(numDiffs int, timeout time.Duration) ([]*diff.Difference, bool, error) { - return nil, false, nil -} - -func (e EmptyRowDiffer) GetDiffsWithFilter(numDiffs int, timeout time.Duration, filterByChangeType types.DiffChangeType) ([]*diff.Difference, bool, error) { - return nil, false, nil -} - -func (e EmptyRowDiffer) Close() error { - return nil -} diff --git a/go/libraries/doltcore/diff/async_differ_test.go b/go/libraries/doltcore/diff/async_differ_test.go deleted file mode 100644 index 87a7d24303..0000000000 --- a/go/libraries/doltcore/diff/async_differ_test.go +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package diff - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/chunks" - "github.com/dolthub/dolt/go/store/constants" - "github.com/dolthub/dolt/go/store/types" -) - -func TestAsyncDiffer(t *testing.T) { - ctx := context.Background() - storage := &chunks.MemoryStorage{} - vrw := types.NewValueStore(storage.NewView()) - - vals := []types.Value{ - types.Uint(0), types.String("a"), - types.Uint(1), types.String("b"), - types.Uint(3), types.String("d"), - types.Uint(4), types.String("e"), - types.Uint(6), types.String("g"), - types.Uint(7), types.String("h"), - types.Uint(9), types.String("j"), - types.Uint(10), types.String("k"), - types.Uint(12), types.String("m"), - types.Uint(13), types.String("n"), - types.Uint(15), types.String("p"), - types.Uint(16), types.String("q"), - types.Uint(18), types.String("s"), - types.Uint(19), types.String("t"), - types.Uint(21), types.String("v"), - types.Uint(22), types.String("w"), - types.Uint(24), types.String("y"), - types.Uint(25), types.String("z"), - } - - m1, err := types.NewMap(ctx, vrw, vals...) - require.NoError(t, err) - - vals = []types.Value{ - types.Uint(0), types.String("a"), // unchanged - //types.Uint(1), types.String("b"), // deleted - types.Uint(2), types.String("c"), // added - types.Uint(3), types.String("d"), // unchanged - //types.Uint(4), types.String("e"), // deleted - types.Uint(5), types.String("f"), // added - types.Uint(6), types.String("g"), // unchanged - //types.Uint(7), types.String("h"), // deleted - types.Uint(8), types.String("i"), // added - types.Uint(9), types.String("j"), // unchanged - //types.Uint(10), types.String("k"), // deleted - types.Uint(11), types.String("l"), // added - types.Uint(12), types.String("m2"), // changed - //types.Uint(13), types.String("n"), // deleted - types.Uint(14), types.String("o"), // added - types.Uint(15), types.String("p2"), // changed - //types.Uint(16), types.String("q"), // deleted - types.Uint(17), types.String("r"), // added - types.Uint(18), types.String("s2"), // changed - //types.Uint(19), types.String("t"), // deleted - types.Uint(20), types.String("u"), // added - types.Uint(21), types.String("v2"), // changed - //types.Uint(22), types.String("w"), // deleted - types.Uint(23), types.String("x"), // added - types.Uint(24), types.String("y2"), // changed - //types.Uint(25), types.String("z"), // deleted - } - m2, err := types.NewMap(ctx, vrw, vals...) - require.NoError(t, err) - - tests := []struct { - name string - createdStarted func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer - expectedStats map[types.DiffChangeType]uint64 - }{ - { - name: "iter all", - createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer { - ad := NewAsyncDiffer(4) - ad.Start(ctx, m1, m2) - return ad - }, - expectedStats: map[types.DiffChangeType]uint64{ - types.DiffChangeModified: 5, - types.DiffChangeAdded: 8, - types.DiffChangeRemoved: 9, - }, - }, - - { - name: "iter range starting with nil", - createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer { - ad := NewAsyncDiffer(4) - ad.StartWithRange(ctx, m1, m2, nil, func(ctx context.Context, value types.Value) (bool, bool, error) { - return true, false, nil - }) - return ad - }, - expectedStats: map[types.DiffChangeType]uint64{ - types.DiffChangeModified: 5, - types.DiffChangeAdded: 8, - types.DiffChangeRemoved: 9, - }, - }, - - { - name: "iter range staring with Null Value", - createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer { - ad := NewAsyncDiffer(4) - ad.StartWithRange(ctx, m1, m2, types.NullValue, func(ctx context.Context, value types.Value) (bool, bool, error) { - return true, false, nil - }) - return ad - }, - expectedStats: map[types.DiffChangeType]uint64{ - types.DiffChangeModified: 5, - types.DiffChangeAdded: 8, - types.DiffChangeRemoved: 9, - }, - }, - - { - name: "iter range less than 17", - createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer { - ad := NewAsyncDiffer(4) - end := types.Uint(27) - ad.StartWithRange(ctx, m1, m2, types.NullValue, func(ctx context.Context, value types.Value) (bool, bool, error) { - valid, err := value.Less(ctx, vrw.Format(), end) - return valid, false, err - }) - return ad - }, - expectedStats: map[types.DiffChangeType]uint64{ - types.DiffChangeModified: 5, - types.DiffChangeAdded: 8, - types.DiffChangeRemoved: 9, - }, - }, - - { - name: "iter range less than 15", - createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer { - ad := NewAsyncDiffer(4) - end := types.Uint(15) - ad.StartWithRange(ctx, m1, m2, types.NullValue, func(ctx context.Context, value types.Value) (bool, bool, error) { - valid, err := value.Less(ctx, vrw.Format(), end) - return valid, false, err - }) - return ad - }, - expectedStats: map[types.DiffChangeType]uint64{ - types.DiffChangeModified: 1, - types.DiffChangeAdded: 5, - types.DiffChangeRemoved: 5, - }, - }, - - { - name: "iter range 10 < 15", - createdStarted: func(ctx context.Context, m1, m2 types.Map) *AsyncDiffer { - ad := NewAsyncDiffer(4) - start := types.Uint(10) - end := types.Uint(15) - ad.StartWithRange(ctx, m1, m2, start, func(ctx context.Context, value types.Value) (bool, bool, error) { - valid, err := value.Less(ctx, vrw.Format(), end) - return valid, false, err - }) - return ad - }, - expectedStats: map[types.DiffChangeType]uint64{ - types.DiffChangeModified: 1, - types.DiffChangeAdded: 2, - types.DiffChangeRemoved: 2, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctx := context.Background() - ad := test.createdStarted(ctx, m1, m2) - err := readAll(ad) - require.NoError(t, err) - require.Equal(t, test.expectedStats, ad.diffStats) - }) - } - - t.Run("can close without reading all", func(t *testing.T) { - ad := NewAsyncDiffer(1) - ad.Start(ctx, m1, m2) - res, more, err := ad.GetDiffs(1, -1) - require.NoError(t, err) - assert.True(t, more) - assert.Len(t, res, 1) - err = ad.Close() - assert.NoError(t, err) - }) - - t.Run("can filter based on change type", func(t *testing.T) { - ad := NewAsyncDiffer(20) - ad.Start(ctx, m1, m2) - res, more, err := ad.GetDiffs(10, -1) - require.NoError(t, err) - assert.True(t, more) - assert.Len(t, res, 10) - err = ad.Close() - assert.NoError(t, err) - - ad = NewAsyncDiffer(20) - ad.Start(ctx, m1, m2) - res, more, err = ad.GetDiffsWithFilter(10, 20*time.Second, types.DiffChangeModified) - require.NoError(t, err) - assert.False(t, more) - assert.Len(t, res, 5) - err = ad.Close() - assert.NoError(t, err) - - ad = NewAsyncDiffer(20) - ad.Start(ctx, m1, m2) - res, more, err = ad.GetDiffsWithFilter(6, -1, types.DiffChangeAdded) - require.NoError(t, err) - assert.True(t, more) - assert.Len(t, res, 6) - err = ad.Close() - assert.NoError(t, err) - }) - - k1Row1Vals := []types.Value{c1Tag, types.Uint(3), c2Tag, types.String("d")} - k1Vals, err := getKeylessRow(ctx, k1Row1Vals) - assert.NoError(t, err) - k1, err := types.NewMap(ctx, vrw, k1Vals...) - assert.NoError(t, err) - - // Delete one row, add two rows - k2Row1Vals := []types.Value{c1Tag, types.Uint(4), c2Tag, types.String("d")} - k2Vals1, err := getKeylessRow(ctx, k2Row1Vals) - assert.NoError(t, err) - k2Row2Vals := []types.Value{c1Tag, types.Uint(1), c2Tag, types.String("e")} - k2Vals2, err := getKeylessRow(ctx, k2Row2Vals) - assert.NoError(t, err) - k2Vals := append(k2Vals1, k2Vals2...) - k2, err := types.NewMap(ctx, vrw, k2Vals...) - require.NoError(t, err) - - t.Run("can diff and filter keyless tables", func(t *testing.T) { - kd := &keylessDiffer{AsyncDiffer: NewAsyncDiffer(20)} - kd.Start(ctx, k1, k2) - res, more, err := kd.GetDiffs(10, 20*time.Second) - require.NoError(t, err) - assert.False(t, more) - assert.Len(t, res, 3) - err = kd.Close() - assert.NoError(t, err) - - kd = &keylessDiffer{AsyncDiffer: NewAsyncDiffer(20)} - kd.Start(ctx, k1, k2) - res, more, err = kd.GetDiffsWithFilter(10, 20*time.Second, types.DiffChangeModified) - require.NoError(t, err) - assert.False(t, more) - assert.Len(t, res, 0) - err = kd.Close() - assert.NoError(t, err) - - kd = &keylessDiffer{AsyncDiffer: NewAsyncDiffer(20)} - kd.Start(ctx, k1, k2) - res, more, err = kd.GetDiffsWithFilter(6, -1, types.DiffChangeAdded) - require.NoError(t, err) - assert.False(t, more) - assert.Len(t, res, 2) - err = kd.Close() - assert.NoError(t, err) - }) -} - -func readAll(ad *AsyncDiffer) error { - for { - _, more, err := ad.GetDiffs(10, -1) - - if err != nil { - return err - } - - if !more { - break - } - } - - return nil -} - -var c1Tag = types.Uint(1) -var c2Tag = types.Uint(2) -var cardTag = types.Uint(schema.KeylessRowCardinalityTag) -var rowIdTag = types.Uint(schema.KeylessRowIdTag) - -func getKeylessRow(ctx context.Context, vals []types.Value) ([]types.Value, error) { - nbf, err := types.GetFormatForVersionString(constants.FormatDefaultString) - if err != nil { - return []types.Value{}, err - } - - id1, err := types.UUIDHashedFromValues(nbf, vals...) - if err != nil { - return []types.Value{}, err - } - - prefix := []types.Value{ - cardTag, - types.Uint(1), - } - vals = append(prefix, vals...) - - return []types.Value{ - mustTuple(rowIdTag, id1), - mustTuple(vals...), - }, nil -} - -func mustTuple(vals ...types.Value) types.Tuple { - tup, err := types.NewTuple(types.Format_Default, vals...) - if err != nil { - panic(err) - } - return tup -} diff --git a/go/libraries/doltcore/diff/diff_stat.go b/go/libraries/doltcore/diff/diff_stat.go index be4b88ac3a..82c2129daa 100644 --- a/go/libraries/doltcore/diff/diff_stat.go +++ b/go/libraries/doltcore/diff/diff_stat.go @@ -19,13 +19,10 @@ import ( "errors" "fmt" "io" - "time" "github.com/dolthub/dolt/go/cmd/dolt/errhand" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/diff" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" @@ -39,7 +36,6 @@ type DiffStatProgress struct { } type prollyReporter func(ctx context.Context, vMapping val.OrdinalMapping, fromD, toD *val.TupleDesc, change tree.Diff, ch chan<- DiffStatProgress) error -type nomsReporter func(ctx context.Context, change *diff.Difference, fromSch, toSch schema.Schema, ch chan<- DiffStatProgress) error // Stat reports a stat of diff changes between two values // todo: make package private once dolthub is migrated @@ -168,38 +164,6 @@ func diffProllyTrees(ctx context.Context, ch chan DiffStatProgress, keyless bool return nil } -func statWithReporter(ctx context.Context, ch chan DiffStatProgress, from, to types.Map, rpr nomsReporter, fromSch, toSch schema.Schema) (err error) { - ad := NewAsyncDiffer(1024) - ad.Start(ctx, from, to) - defer func() { - if cerr := ad.Close(); cerr != nil && err == nil { - err = cerr - } - }() - - var more bool - var diffs []*diff.Difference - for { - diffs, more, err = ad.GetDiffs(100, time.Millisecond) - if err != nil { - return err - } - - for _, df := range diffs { - err = rpr(ctx, df, fromSch, toSch, ch) - if err != nil { - return err - } - } - - if !more { - break - } - } - - return nil -} - func reportPkChanges(ctx context.Context, vMapping val.OrdinalMapping, fromD, toD *val.TupleDesc, change tree.Diff, ch chan<- DiffStatProgress) error { var stat DiffStatProgress switch change.Type { @@ -280,66 +244,3 @@ func prollyCountCellDiff(ctx context.Context, mapping val.OrdinalMapping, fromD, changed += newCols return changed } - -func reportNomsPkChanges(ctx context.Context, change *diff.Difference, fromSch, toSch schema.Schema, ch chan<- DiffStatProgress) error { - var stat DiffStatProgress - switch change.ChangeType { - case types.DiffChangeAdded: - stat = DiffStatProgress{Adds: 1} - case types.DiffChangeRemoved: - stat = DiffStatProgress{Removes: 1} - case types.DiffChangeModified: - oldTuple := change.OldValue.(types.Tuple) - newTuple := change.NewValue.(types.Tuple) - cellChanges, err := row.CountCellDiffs(oldTuple, newTuple, fromSch, toSch) - if err != nil { - return err - } - stat = DiffStatProgress{Changes: 1, CellChanges: cellChanges} - default: - return errors.New("unknown change type") - } - select { - case ch <- stat: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -func reportNomsKeylessChanges(ctx context.Context, change *diff.Difference, fromSch, toSch schema.Schema, ch chan<- DiffStatProgress) error { - var oldCard uint64 - if change.OldValue != nil { - v, err := change.OldValue.(types.Tuple).Get(row.KeylessCardinalityValIdx) - if err != nil { - return err - } - oldCard = uint64(v.(types.Uint)) - } - - var newCard uint64 - if change.NewValue != nil { - v, err := change.NewValue.(types.Tuple).Get(row.KeylessCardinalityValIdx) - if err != nil { - return err - } - newCard = uint64(v.(types.Uint)) - } - - var stat DiffStatProgress - delta := int64(newCard) - int64(oldCard) - if delta > 0 { - stat = DiffStatProgress{Adds: uint64(delta)} - } else if delta < 0 { - stat = DiffStatProgress{Removes: uint64(-delta)} - } else { - return fmt.Errorf("diff with delta = 0 for key: %s", change.KeyValue.HumanReadableString()) - } - - select { - case ch <- stat: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} diff --git a/go/libraries/doltcore/diff/diffsplitter.go b/go/libraries/doltcore/diff/diffsplitter.go index a3523f97e8..bc9ea57143 100644 --- a/go/libraries/doltcore/diff/diffsplitter.go +++ b/go/libraries/doltcore/diff/diffsplitter.go @@ -118,31 +118,6 @@ func mapQuerySchemaToTargetSchema(query, target sql.Schema) (mapping []int, err return } -func mapToAndFromColumns(query sql.Schema) (mapping []int, err error) { - last := query[len(query)-1] - if last.Name != "diff_type" { - return nil, errors.New("expected last diff column to be 'diff_type'") - } - query = query[:len(query)-1] - - mapping = make([]int, len(query)) - for i, col := range query { - if strings.HasPrefix(col.Name, fromPrefix) { - // map "from_..." column to "to_..." column - base := col.Name[len(fromPrefix):] - mapping[i] = query.IndexOfColName(toPrefix + base) - } else if strings.HasPrefix(col.Name, toPrefix) { - // map "to_..." column to "from_..." column - base := col.Name[len(toPrefix):] - mapping[i] = query.IndexOfColName(fromPrefix + base) - } else { - return nil, errors.New("expected column prefix of 'to_' or 'from_' (" + col.Name + ")") - } - } - // |mapping| will contain -1 for unmapped columns - return -} - func (ds DiffSplitter) SplitDiffResultRow(ctx *sql.Context, row sql.Row) (from, to RowDiff, err error) { from = RowDiff{ColDiffs: make([]ChangeType, len(ds.targetSch))} to = RowDiff{ColDiffs: make([]ChangeType, len(ds.targetSch))} diff --git a/go/libraries/doltcore/doltdb/commit.go b/go/libraries/doltcore/doltdb/commit.go index 83829d6918..c14671ef82 100644 --- a/go/libraries/doltcore/doltdb/commit.go +++ b/go/libraries/doltcore/doltdb/commit.go @@ -81,15 +81,6 @@ func NewCommit(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore return &Commit{vrw, ns, parents, commit}, nil } -// NewCommitFromValue generates a new Commit object that wraps a supplied types.Value. -func NewCommitFromValue(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, value types.Value) (*Commit, error) { - commit, err := datas.CommitFromValue(vrw.Format(), value) - if err != nil { - return nil, err - } - return NewCommit(ctx, vrw, ns, commit) -} - // HashOf returns the hash of the commit func (c *Commit) HashOf() (hash.Hash, error) { return c.dCommit.Addr(), nil diff --git a/go/libraries/doltcore/doltdb/errors.go b/go/libraries/doltcore/doltdb/errors.go index 2301df7cc5..c2088e11ae 100644 --- a/go/libraries/doltcore/doltdb/errors.go +++ b/go/libraries/doltcore/doltdb/errors.go @@ -27,7 +27,6 @@ var ErrInvTableName = errors.New("not a valid table name") var ErrInvHash = errors.New("not a valid hash") var ErrInvalidAncestorSpec = errors.New("invalid ancestor spec") var ErrInvalidBranchOrHash = errors.New("string is not a valid branch or hash") -var ErrInvalidHash = errors.New("string is not a valid hash") var ErrFoundHashNotACommit = errors.New("the value retrieved for this hash is not a commit") var ErrHashNotFound = errors.New("could not find a value for this hash") @@ -39,7 +38,6 @@ var ErrWorkspaceNotFound = errors.New("workspace not found") var ErrTableNotFound = errors.New("table not found") var ErrTableExists = errors.New("table already exists") var ErrAlreadyOnBranch = errors.New("Already on branch") -var ErrAlreadyOnWorkspace = errors.New("Already on workspace") var ErrNomsIO = errors.New("error reading from or writing to noms") @@ -109,37 +107,6 @@ func (rt RootType) String() string { return "unknown" } -type RootTypeSet map[RootType]struct{} - -func NewRootTypeSet(rts ...RootType) RootTypeSet { - mp := make(map[RootType]struct{}) - - for _, rt := range rts { - mp[rt] = struct{}{} - } - - return RootTypeSet(mp) -} - -func (rts RootTypeSet) Contains(rt RootType) bool { - _, ok := rts[rt] - return ok -} - -func (rts RootTypeSet) First(rtList []RootType) RootType { - for _, rt := range rtList { - if _, ok := rts[rt]; ok { - return rt - } - } - - return InvalidRoot -} - -func (rts RootTypeSet) IsEmpty() bool { - return len(rts) == 0 -} - type RootValueUnreadable struct { RootType RootType Cause error diff --git a/go/libraries/doltcore/doltdb/nonlocal_tables.go b/go/libraries/doltcore/doltdb/nonlocal_tables.go index dc229b990f..749838977c 100644 --- a/go/libraries/doltcore/doltdb/nonlocal_tables.go +++ b/go/libraries/doltcore/doltdb/nonlocal_tables.go @@ -51,7 +51,8 @@ func getNonlocalTablesRef(_ context.Context, valDesc *val.TupleDesc, valTuple va return result } -func GetGlobalTablePatterns(ctx context.Context, root RootValue, schema string, cb func(string)) error { +// GetNonlocalTablePatterns invokes |cb| once for each table name pattern in dolt_nonlocal_tables on |root| and |schema|. +func GetNonlocalTablePatterns(ctx context.Context, root RootValue, schema string, cb func(string)) error { table_name := TableName{Name: NonlocalTableName, Schema: schema} table, found, err := root.GetTable(ctx, table_name) if err != nil { diff --git a/go/libraries/doltcore/doltdb/root_val_storage.go b/go/libraries/doltcore/doltdb/root_val_storage.go index 6a7deabe57..60dfc67d14 100755 --- a/go/libraries/doltcore/doltdb/root_val_storage.go +++ b/go/libraries/doltcore/doltdb/root_val_storage.go @@ -472,6 +472,10 @@ func encodeTableNameForSerialization(name TableName) string { // decodeTableNameFromSerialization decodes a table name from a serialized string. See notes on serialization in // |encodeTableNameForSerialization| func decodeTableNameFromSerialization(encodedName string) (TableName, bool) { + if len(encodedName) == 0 { + return TableName{}, false + } + if encodedName[0] != 0 { return TableName{Name: encodedName}, true } else if len(encodedName) >= 4 { // 2 null bytes plus at least one char for schema and table name diff --git a/go/libraries/doltcore/doltdb/table_name_patterns.go b/go/libraries/doltcore/doltdb/table_name_patterns.go index 58848c61a6..00ee8b5588 100644 --- a/go/libraries/doltcore/doltdb/table_name_patterns.go +++ b/go/libraries/doltcore/doltdb/table_name_patterns.go @@ -43,6 +43,35 @@ func MatchTablePattern(pattern string, table string) (bool, error) { return re.MatchString(table), nil } +// CompiledTablePatterns holds compiled table name patterns for reuse when matching many names without recompiling. +type CompiledTablePatterns []*regexp.Regexp + +// CompileTablePatterns compiles each of |patterns| once and returns them for use with TableMatchesAny. Returns (nil, nil) when |patterns| is empty. +func CompileTablePatterns(patterns []string) (CompiledTablePatterns, error) { + if len(patterns) == 0 { + return nil, nil + } + compiled := make(CompiledTablePatterns, 0, len(patterns)) + for _, p := range patterns { + re, err := compilePattern(p) + if err != nil { + return nil, err + } + compiled = append(compiled, re) + } + return compiled, nil +} + +// TableMatchesAny reports whether |table| matches any of the patterns in |c|. +func (c CompiledTablePatterns) TableMatchesAny(table string) bool { + for _, re := range c { + if re.MatchString(table) { + return true + } + } + return false +} + // GetMatchingTables returns all tables that match a pattern func GetMatchingTables(ctx *sql.Context, root RootValue, schemaName string, pattern string) (results []string, err error) { // If the pattern doesn't contain any special characters, look up that name. diff --git a/go/libraries/doltcore/doltdb/workingset.go b/go/libraries/doltcore/doltdb/workingset.go index f6529114fc..5498271679 100644 --- a/go/libraries/doltcore/doltdb/workingset.go +++ b/go/libraries/doltcore/doltdb/workingset.go @@ -75,6 +75,8 @@ type RebaseState struct { // rebasingStarted is true once the rebase plan has been started to execute. Once rebasingStarted is true, the // value in lastAttemptedStep has been initialized and is valid to read. rebasingStarted bool + // skipVerification indicates whether test validation should be skipped during rebase operations. + skipVerification bool } // Branch returns the name of the branch being actively rebased. This is the branch that will be updated to point @@ -120,6 +122,10 @@ func (rs RebaseState) WithRebasingStarted(rebasingStarted bool) *RebaseState { return &rs } +func (rs RebaseState) SkipVerification() bool { + return rs.skipVerification +} + type MergeState struct { // the source commit commit *Commit @@ -322,13 +328,14 @@ func (ws WorkingSet) StartMerge(commit *Commit, commitSpecStr string) *WorkingSe // the branch that is being rebased, and |previousRoot| is root value of the branch being rebased. The HEAD and STAGED // root values of the branch being rebased must match |previousRoot|; WORKING may be a different root value, but ONLY // if it contains only ignored tables. -func (ws WorkingSet) StartRebase(ctx *sql.Context, ontoCommit *Commit, branch string, previousRoot RootValue, commitBecomesEmptyHandling EmptyCommitHandling, emptyCommitHandling EmptyCommitHandling) (*WorkingSet, error) { +func (ws WorkingSet) StartRebase(ctx *sql.Context, ontoCommit *Commit, branch string, previousRoot RootValue, commitBecomesEmptyHandling EmptyCommitHandling, emptyCommitHandling EmptyCommitHandling, skipVerification bool) (*WorkingSet, error) { ws.rebaseState = &RebaseState{ ontoCommit: ontoCommit, preRebaseWorking: previousRoot, branch: branch, commitBecomesEmptyHandling: commitBecomesEmptyHandling, emptyCommitHandling: emptyCommitHandling, + skipVerification: skipVerification, } ontoRoot, err := ontoCommit.GetRootValue(ctx) @@ -549,6 +556,7 @@ func newWorkingSet(ctx context.Context, name string, vrw types.ValueReadWriter, emptyCommitHandling: EmptyCommitHandling(dsws.RebaseState.EmptyCommitHandling(ctx)), lastAttemptedStep: dsws.RebaseState.LastAttemptedStep(ctx), rebasingStarted: dsws.RebaseState.RebasingStarted(ctx), + skipVerification: dsws.RebaseState.SkipVerification(ctx), } } @@ -646,7 +654,7 @@ func (ws *WorkingSet) writeValues(ctx context.Context, db *DoltDB, meta *datas.W rebaseState = datas.NewRebaseState(preRebaseWorking.TargetHash(), dCommit.Addr(), ws.rebaseState.branch, uint8(ws.rebaseState.commitBecomesEmptyHandling), uint8(ws.rebaseState.emptyCommitHandling), - ws.rebaseState.lastAttemptedStep, ws.rebaseState.rebasingStarted) + ws.rebaseState.lastAttemptedStep, ws.rebaseState.rebasingStarted, ws.rebaseState.skipVerification) } return &datas.WorkingSetSpec{ diff --git a/go/libraries/doltcore/env/actions/checkout.go b/go/libraries/doltcore/env/actions/checkout.go index 83658f13aa..cf5ad58f55 100644 --- a/go/libraries/doltcore/env/actions/checkout.go +++ b/go/libraries/doltcore/env/actions/checkout.go @@ -216,7 +216,7 @@ func CleanOldWorkingSet( } // we also have to do a clean, because we the ResetHard won't touch any new tables (tables only in the working set) - newRoots, err := CleanUntracked(ctx, resetRoots, []string{}, false, true) + newRoots, err := CleanUntracked(ctx, resetRoots, []string{}, false, true, false) if err != nil { return err } diff --git a/go/libraries/doltcore/env/actions/commit.go b/go/libraries/doltcore/env/actions/commit.go index 61b2ede4ca..15b1a619e2 100644 --- a/go/libraries/doltcore/env/actions/commit.go +++ b/go/libraries/doltcore/env/actions/commit.go @@ -15,8 +15,12 @@ package actions import ( + "fmt" + "io" + "strings" "time" + gms "github.com/dolthub/go-mysql-server" "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/dolt/go/libraries/doltcore/diff" @@ -25,14 +29,42 @@ import ( ) type CommitStagedProps struct { - Message string - Date time.Time - AllowEmpty bool - SkipEmpty bool - Amend bool - Force bool - Name string - Email string + Message string + Date time.Time + AllowEmpty bool + SkipEmpty bool + Amend bool + Force bool + Name string + Email string + SkipVerification bool +} + +const ( + // System variable name, defined here to avoid circular imports + DoltCommitVerificationGroups = "dolt_commit_verification_groups" +) + +// GetCommitRunTestGroups returns the test groups to run for commit operations +// Returns empty slice if no tests should be run, ["*"] if all tests should be run, +// or specific group names if only those groups should be run +func GetCommitRunTestGroups() []string { + _, val, ok := sql.SystemVariables.GetGlobal(DoltCommitVerificationGroups) + if !ok { + return nil + } + if stringVal, ok := val.(string); ok && stringVal != "" { + if stringVal == "*" { + return []string{"*"} + } + // Split by comma and trim whitespace + groups := strings.Split(stringVal, ",") + for i, group := range groups { + groups[i] = strings.TrimSpace(group) + } + return groups + } + return nil } // GetCommitStaged returns a new pending commit with the roots and commit properties given. @@ -114,6 +146,16 @@ func GetCommitStaged( } } + if !props.SkipVerification { + testGroups := GetCommitRunTestGroups() + if len(testGroups) > 0 { + err := runCommitVerification(ctx, testGroups) + if err != nil { + return nil, err + } + } + } + meta, err := datas.NewCommitMetaWithUserTS(props.Name, props.Email, props.Message, props.Date) if err != nil { return nil, err @@ -121,3 +163,61 @@ func GetCommitStaged( return db.NewPendingCommit(ctx, roots, mergeParents, props.Amend, meta) } + +func runCommitVerification(ctx *sql.Context, testGroups []string) error { + type sessionInterface interface { + sql.Session + GenericProvider() sql.MutableDatabaseProvider + } + + session, ok := ctx.Session.(sessionInterface) + if !ok { + return fmt.Errorf("session does not provide database provider interface") + } + + provider := session.GenericProvider() + engine := gms.NewDefault(provider) + + return runTestsUsingDtablefunctions(ctx, engine, testGroups) +} + +// runTestsUsingDtablefunctions runs tests using the dtablefunctions package against the staged root +func runTestsUsingDtablefunctions(ctx *sql.Context, engine *gms.Engine, testGroups []string) error { + if len(testGroups) == 0 { + return nil + } + + var allFailures []string + + for _, group := range testGroups { + query := fmt.Sprintf("SELECT * FROM dolt_test_run('%s')", group) + _, iter, _, err := engine.Query(ctx, query) + if err != nil { + return fmt.Errorf("failed to run dolt_test_run for group %s: %w", group, err) + } + + for { + row, rErr := iter.Next(ctx) + if rErr == io.EOF { + break + } + if rErr != nil { + return fmt.Errorf("error reading test results: %w", rErr) + } + + // Extract status (column 3) + status := fmt.Sprintf("%v", row[3]) + if status != "PASS" { + testName := fmt.Sprintf("%v", row[0]) + message := fmt.Sprintf("%v", row[4]) + allFailures = append(allFailures, fmt.Sprintf("%s (%s)", testName, message)) + } + } + } + + if len(allFailures) > 0 { + return fmt.Errorf("commit verification failed: %s", strings.Join(allFailures, ", ")) + } + + return nil +} diff --git a/go/libraries/doltcore/env/actions/reset.go b/go/libraries/doltcore/env/actions/reset.go index c5ffa8c279..163cd4451f 100644 --- a/go/libraries/doltcore/env/actions/reset.go +++ b/go/libraries/doltcore/env/actions/reset.go @@ -270,60 +270,74 @@ func getUnionedTables(ctx context.Context, tables []doltdb.TableName, stagedRoot return tables, nil } -// CleanUntracked deletes untracked tables from the working root. -// Evaluates untracked tables as: all working tables - all staged tables. -func CleanUntracked(ctx *sql.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool) (doltdb.Roots, error) { +// CleanUntracked deletes from the working root the tables that are untracked (in working but not in staged/head). If +// |tables| is non-empty it uses only those names as candidates; otherwise it uses all working tables. Tables matching +// dolt_nonlocal_tables are always excluded. When |respectIgnoreRules| is true, tables matching dolt_ignore are also excluded. Does nothing when |dryrun| is true. +func CleanUntracked(ctx *sql.Context, roots doltdb.Roots, tables []string, dryrun bool, force bool, respectIgnoreRules bool) (doltdb.Roots, error) { untrackedTables := make(map[doltdb.TableName]struct{}) + for _, name := range tables { + resolvedName, tblExists, err := resolve.TableName(ctx, roots.Working, name) + if err != nil { + return doltdb.Roots{}, err + } + if !tblExists { + return doltdb.Roots{}, fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, name) + } + untrackedTables[resolvedName] = struct{}{} + } - var err error if len(tables) == 0 { allTableNames, err := roots.Working.GetAllTableNames(ctx, true) if err != nil { - return doltdb.Roots{}, nil + return doltdb.Roots{}, err } - for _, tableName := range allTableNames { - untrackedTables[tableName] = struct{}{} - } - } else { - for i := range tables { - name := tables[i] - resolvedName, tblExists, err := resolve.TableName(ctx, roots.Working, name) + var candidates []doltdb.TableName + if respectIgnoreRules { + candidates, err = doltdb.ExcludeIgnoredTables(ctx, roots, allTableNames) if err != nil { return doltdb.Roots{}, err } - if !tblExists { - return doltdb.Roots{}, fmt.Errorf("%w: '%s'", doltdb.ErrTableNotFound, name) + } else { + candidates = allTableNames + } + var nonlocalPatterns []string + err = doltdb.GetNonlocalTablePatterns(ctx, roots.Working, doltdb.DefaultSchemaName, func(p string) { + nonlocalPatterns = append(nonlocalPatterns, p) + }) + if err != nil { + return doltdb.Roots{}, err + } + compiled, err := doltdb.CompileTablePatterns(nonlocalPatterns) + if err != nil { + return doltdb.Roots{}, err + } + for _, tableName := range candidates { + if compiled.TableMatchesAny(tableName.Name) { + continue } - untrackedTables[resolvedName] = struct{}{} + untrackedTables[tableName] = struct{}{} } } - // untracked tables = working tables - staged tables headTblNames := GetAllTableNames(ctx, roots.Staged) - if err != nil { - return doltdb.Roots{}, err - } - for _, name := range headTblNames { delete(untrackedTables, name) } - newRoot := roots.Working - var toDelete []doltdb.TableName + toDelete := make([]doltdb.TableName, 0, len(untrackedTables)) for t := range untrackedTables { toDelete = append(toDelete, t) } - newRoot, err = newRoot.RemoveTables(ctx, force, force, toDelete...) - if err != nil { - return doltdb.Roots{}, fmt.Errorf("failed to remove tables; %w", err) - } - if dryrun { return roots, nil } - roots.Working = newRoot + newRoot, err := roots.Working.RemoveTables(ctx, force, force, toDelete...) + if err != nil { + return doltdb.Roots{}, fmt.Errorf("failed to remove tables; %w", err) + } + roots.Working = newRoot return roots, nil } diff --git a/go/libraries/doltcore/env/actions/test_table_helpers.go b/go/libraries/doltcore/env/actions/test_table_helpers.go deleted file mode 100644 index dc72230056..0000000000 --- a/go/libraries/doltcore/env/actions/test_table_helpers.go +++ /dev/null @@ -1,447 +0,0 @@ -// Copyright 2025 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package actions - -import ( - "fmt" - "io" - "strconv" - "time" - - "github.com/dolthub/go-mysql-server/sql" - "github.com/shopspring/decimal" - "golang.org/x/exp/constraints" - - "github.com/dolthub/dolt/go/store/val" -) - -const ( - AssertionExpectedRows = "expected_rows" - AssertionExpectedColumns = "expected_columns" - AssertionExpectedSingleValue = "expected_single_value" -) - -// AssertData parses an assertion, comparison, and value, then returns the status of the test. -// Valid comparison are: "==", "!=", "<", ">", "<=", and ">=". -// testPassed indicates whether the test was successful or not. -// message is a string used to indicate test failures, and will not halt the overall process. -// message will be empty if the test passed. -// err indicates runtime failures and will stop dolt_test_run from proceeding. -func AssertData(sqlCtx *sql.Context, assertion string, comparison string, value *string, queryResult sql.RowIter) (testPassed bool, message string, err error) { - switch assertion { - case AssertionExpectedRows: - message, err = expectRows(sqlCtx, comparison, value, queryResult) - case AssertionExpectedColumns: - message, err = expectColumns(sqlCtx, comparison, value, queryResult) - case AssertionExpectedSingleValue: - message, err = expectSingleValue(sqlCtx, comparison, value, queryResult) - default: - return false, fmt.Sprintf("%s is not a valid assertion type", assertion), nil - } - - if err != nil { - return false, "", err - } else if message != "" { - return false, message, nil - } - return true, "", nil -} - -func expectSingleValue(sqlCtx *sql.Context, comparison string, value *string, queryResult sql.RowIter) (message string, err error) { - row, err := queryResult.Next(sqlCtx) - if err == io.EOF { - return fmt.Sprintf("expected_single_value expects exactly one cell. Received 0 rows"), nil - } else if err != nil { - return "", err - } - - if len(row) != 1 { - return fmt.Sprintf("expected_single_value expects exactly one cell. Received multiple columns"), nil - } - _, err = queryResult.Next(sqlCtx) - if err == nil { //If multiple rows were given, we should error out - return fmt.Sprintf("expected_single_value expects exactly one cell. Received multiple rows"), nil - } else if err != io.EOF { // "True" error, so we should quit out - return "", err - } - - if value == nil { // If we're expecting a null value, we don't need to type switch - return compareNullValue(comparison, row[0], AssertionExpectedSingleValue), nil - } - - // Check if the expected value is a boolean string, and if so, coerce the actual value to boolean, with the exception - // of "0" and "1", which are valid integers and are covered below. - if *value != "0" && *value != "1" { - if expectedBool, err := strconv.ParseBool(*value); err == nil { - actualBool, boolErr := getInterfaceAsBool(row[0]) - if boolErr != nil { - return fmt.Sprintf("Could not convert value to boolean: %v", boolErr), nil - } - return compareBooleans(comparison, expectedBool, actualBool, AssertionExpectedSingleValue), nil - } - } - - switch actualValue := row[0].(type) { - case int8: - expectedInt, err := strconv.ParseInt(*value, 10, 64) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, int8(expectedInt), actualValue, AssertionExpectedSingleValue), nil - case int16: - expectedInt, err := strconv.ParseInt(*value, 10, 64) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, int16(expectedInt), actualValue, AssertionExpectedSingleValue), nil - case int32: - expectedInt, err := strconv.ParseInt(*value, 10, 64) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, int32(expectedInt), actualValue, AssertionExpectedSingleValue), nil - case int64: - expectedInt, err := strconv.ParseInt(*value, 10, 64) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, expectedInt, actualValue, AssertionExpectedSingleValue), nil - case int: - expectedInt, err := strconv.ParseInt(*value, 10, 64) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, int(expectedInt), actualValue, AssertionExpectedSingleValue), nil - case uint8: - expectedUint, err := strconv.ParseUint(*value, 10, 32) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, uint8(expectedUint), actualValue, AssertionExpectedSingleValue), nil - case uint16: - expectedUint, err := strconv.ParseUint(*value, 10, 32) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, uint16(expectedUint), actualValue, AssertionExpectedSingleValue), nil - case uint32: - expectedUint, err := strconv.ParseUint(*value, 10, 32) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, uint32(expectedUint), actualValue, AssertionExpectedSingleValue), nil - case uint64: - expectedUint, err := strconv.ParseUint(*value, 10, 64) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, expectedUint, actualValue, AssertionExpectedSingleValue), nil - case uint: - expectedUint, err := strconv.ParseUint(*value, 10, 64) - if err != nil { - return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil - } - return compareTestAssertion(comparison, uint(expectedUint), actualValue, AssertionExpectedSingleValue), nil - case float64: - expectedFloat, err := strconv.ParseFloat(*value, 64) - if err != nil { - return fmt.Sprintf("Could not compare non float value '%s', with %f", *value, actualValue), nil - } - return compareTestAssertion(comparison, expectedFloat, actualValue, AssertionExpectedSingleValue), nil - case float32: - expectedFloat, err := strconv.ParseFloat(*value, 32) - if err != nil { - return fmt.Sprintf("Could not compare non float value '%s', with %f", *value, actualValue), nil - } - return compareTestAssertion(comparison, float32(expectedFloat), actualValue, AssertionExpectedSingleValue), nil - case decimal.Decimal: - expectedDecimal, err := decimal.NewFromString(*value) - if err != nil { - return fmt.Sprintf("Could not compare non decimal value '%s', with %s", *value, actualValue), nil - } - return compareDecimals(comparison, expectedDecimal, actualValue, AssertionExpectedSingleValue), nil - case time.Time: - expectedTime, format, err := parseTestsDate(*value) - if err != nil { - return fmt.Sprintf("%s does not appear to be a valid date", *value), nil - } - return compareDates(comparison, expectedTime, actualValue, format, AssertionExpectedSingleValue), nil - case *val.TextStorage, string: - actualString, err := GetStringColAsString(sqlCtx, actualValue) - if err != nil { - return "", err - } - return compareTestAssertion(comparison, *value, *actualString, AssertionExpectedSingleValue), nil - default: - return fmt.Sprintf("Type %T is not supported. Open an issue at https://github.com/dolthub/dolt/issues to see it added", actualValue), nil - } -} - -func expectRows(sqlCtx *sql.Context, comparison string, value *string, queryResult sql.RowIter) (message string, err error) { - if value == nil { - return "null is not a valid assertion for expected_rows", nil - } - expectedRows, err := strconv.Atoi(*value) - if err != nil { - return fmt.Sprintf("cannot run assertion on non integer value: %s", *value), nil - } - - var numRows int - for { - _, err := queryResult.Next(sqlCtx) - if err == io.EOF { - break - } else if err != nil { - return "", err - } - numRows++ - } - return compareTestAssertion(comparison, expectedRows, numRows, AssertionExpectedRows), nil -} - -func expectColumns(sqlCtx *sql.Context, comparison string, value *string, queryResult sql.RowIter) (message string, err error) { - if value == nil { - return "null is not a valid assertion for expected_rows", nil - } - expectedColumns, err := strconv.Atoi(*value) - if err != nil { - return fmt.Sprintf("cannot run assertion on non integer value: %s", *value), nil - } - - var numColumns int - row, err := queryResult.Next(sqlCtx) - if err != nil && err != io.EOF { - return "", err - } - numColumns = len(row) - return compareTestAssertion(comparison, expectedColumns, numColumns, AssertionExpectedColumns), nil -} - -// compareTestAssertion is a generic function used for comparing string, ints, floats. -// It takes in a comparison string from one of: "==", "!=", "<", ">", "<=", ">=" -// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise -func compareTestAssertion[T constraints.Ordered](comparison string, expectedValue, actualValue T, assertionType string) string { - switch comparison { - case "==": - if actualValue != expectedValue { - return fmt.Sprintf("Assertion failed: %s equal to %v, got %v", assertionType, expectedValue, actualValue) - } - case "!=": - if actualValue == expectedValue { - return fmt.Sprintf("Assertion failed: %s not equal to %v, got %v", assertionType, expectedValue, actualValue) - } - case "<": - if actualValue >= expectedValue { - return fmt.Sprintf("Assertion failed: %s less than %v, got %v", assertionType, expectedValue, actualValue) - } - case "<=": - if actualValue > expectedValue { - return fmt.Sprintf("Assertion failed: %s less than or equal to %v, got %v", assertionType, expectedValue, actualValue) - } - case ">": - if actualValue <= expectedValue { - return fmt.Sprintf("Assertion failed: %s greater than %v, got %v", assertionType, expectedValue, actualValue) - } - case ">=": - if actualValue < expectedValue { - return fmt.Sprintf("Assertion failed: %s greater than or equal to %v, got %v", assertionType, expectedValue, actualValue) - } - default: - return fmt.Sprintf("%s is not a valid comparison type", comparison) - } - return "" -} - -// parseTestsDate is an internal function that parses the queried string according to allowed time formats for dolt_tests. -// It returns the parsed time, the format that succeeded, and an error if applicable. -func parseTestsDate(value string) (parsedTime time.Time, format string, err error) { - // List of valid formats - formats := []string{ - time.DateOnly, - time.DateTime, - time.TimeOnly, - time.RFC3339, - time.RFC1123Z, - } - - for _, format := range formats { - if parsedTime, parseErr := time.Parse(format, value); parseErr == nil { - return parsedTime, format, nil - } else { - err = parseErr - } - } - return time.Time{}, "", err -} - -// compareDates is a function used for comparing time values. -// It takes in a comparison string from one of: "==", "!=", "<", ">", "<=", ">=" -// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise -func compareDates(comparison string, expectedValue, realValue time.Time, format string, assertionType string) string { - expectedStr := expectedValue.Format(format) - realStr := realValue.Format(format) - switch comparison { - case "==": - if !expectedValue.Equal(realValue) { - return fmt.Sprintf("Assertion failed: %s equal to %s, got %s", assertionType, expectedStr, realStr) - } - case "!=": - if expectedValue.Equal(realValue) { - return fmt.Sprintf("Assertion failed: %s not equal to %s, got %s", assertionType, expectedStr, realStr) - } - case "<": - if realValue.Equal(expectedValue) || realValue.After(expectedValue) { - return fmt.Sprintf("Assertion failed: %s less than %s, got %s", assertionType, expectedStr, realStr) - } - case "<=": - if realValue.After(expectedValue) { - return fmt.Sprintf("Assertion failed: %s less than or equal to %s, got %s", assertionType, expectedStr, realStr) - } - case ">": - if realValue.Before(expectedValue) || realValue.Equal(expectedValue) { - return fmt.Sprintf("Assertion failed: %s greater than %s, got %s", assertionType, expectedStr, realStr) - } - case ">=": - if realValue.Before(expectedValue) { - return fmt.Sprintf("Assertion failed: %s greater than or equal to %s, got %s", assertionType, expectedStr, realStr) - } - default: - return fmt.Sprintf("%s is not a valid comparison type", comparison) - } - return "" -} - -// compareDecimals is a function used for comparing decimals. -// It takes in a comparison string from one of: "==", "!=", "<", ">", "<=", ">=" -// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise -func compareDecimals(comparison string, expectedValue, realValue decimal.Decimal, assertionType string) string { - switch comparison { - case "==": - if !expectedValue.Equal(realValue) { - return fmt.Sprintf("Assertion failed: %s equal to %v, got %v", assertionType, expectedValue, realValue) - } - case "!=": - if expectedValue.Equal(realValue) { - return fmt.Sprintf("Assertion failed: %s not equal to %v, got %v", assertionType, expectedValue, realValue) - } - case "<": - if realValue.GreaterThanOrEqual(expectedValue) { - return fmt.Sprintf("Assertion failed: %s less than %v, got %v", assertionType, expectedValue, realValue) - } - case "<=": - if realValue.GreaterThan(expectedValue) { - return fmt.Sprintf("Assertion failed: %s less than or equal to %v, got %v", assertionType, expectedValue, realValue) - } - case ">": - if realValue.LessThanOrEqual(expectedValue) { - return fmt.Sprintf("Assertion failed: %s greater than %v, got %v", assertionType, expectedValue, realValue) - } - case ">=": - if realValue.LessThan(expectedValue) { - return fmt.Sprintf("Assertion failed: %s greater than or equal to %v, got %v", assertionType, expectedValue, realValue) - } - default: - return fmt.Sprintf("%s is not a valid comparison type", comparison) - } - return "" -} - -// getTinyIntColAsBool returns the value interface{} as a bool -// This is necessary because the query engine may return a tinyint column as a bool, int, or other types. -// Based on GetTinyIntColAsBool from commands/utils.go, which we can't depend on here due to package cycles. -func getInterfaceAsBool(col interface{}) (bool, error) { - switch v := col.(type) { - case bool: - return v, nil - case int: - return v == 1, nil - case int8: - return v == 1, nil - case int16: - return v == 1, nil - case int32: - return v == 1, nil - case int64: - return v == 1, nil - case uint: - return v == 1, nil - case uint8: - return v == 1, nil - case uint16: - return v == 1, nil - case uint32: - return v == 1, nil - case uint64: - return v == 1, nil - case string: - return v == "1", nil - default: - return false, fmt.Errorf("unexpected type %T, was expecting bool, int, or string", v) - } -} - -// compareBooleans is a function used for comparing boolean values. -// It takes in a comparison string from one of: "==", "!=" -// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise -func compareBooleans(comparison string, expectedValue, realValue bool, assertionType string) string { - switch comparison { - case "==": - if expectedValue != realValue { - return fmt.Sprintf("Assertion failed: %s equal to %t, got %t", assertionType, expectedValue, realValue) - } - case "!=": - if expectedValue == realValue { - return fmt.Sprintf("Assertion failed: %s not equal to %t, got %t", assertionType, expectedValue, realValue) - } - default: - return fmt.Sprintf("%s is not a valid comparison for boolean values. Only '==' and '!=' are supported", comparison) - } - return "" -} - -// compareNullValue is a function used for comparing a null value. -// It takes in a comparison string from one of: "==", "!=" -// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise -func compareNullValue(comparison string, actualValue interface{}, assertionType string) string { - switch comparison { - case "==": - if actualValue != nil { - return fmt.Sprintf("Assertion failed: %s equal to NULL, got %v", assertionType, actualValue) - } - case "!=": - if actualValue == nil { - return fmt.Sprintf("Assertion failed: %s not equal to NULL, got NULL", assertionType) - } - default: - return fmt.Sprintf("%s is not a valid comparison for NULL values", comparison) - } - return "" -} - -// GetStringColAsString is a function that returns a text column as a string. -// This is necessary as the dolt_tests system table returns *val.TextStorage types under certain situations, -// so we use a special parser to get the correct string values -func GetStringColAsString(sqlCtx *sql.Context, tableValue interface{}) (*string, error) { - if ts, ok := tableValue.(*val.TextStorage); ok { - str, err := ts.Unwrap(sqlCtx) - return &str, err - } else if str, ok := tableValue.(string); ok { - return &str, nil - } else if tableValue == nil { - return nil, nil - } else { - return nil, fmt.Errorf("unexpected type %T, was expecting string", tableValue) - } -} diff --git a/go/libraries/doltcore/env/environment.go b/go/libraries/doltcore/env/environment.go index 441581f461..69a79bf2ec 100644 --- a/go/libraries/doltcore/env/environment.go +++ b/go/libraries/doltcore/env/environment.go @@ -33,7 +33,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/grpcendpoint" "github.com/dolthub/dolt/go/libraries/doltcore/ref" - "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" "github.com/dolthub/dolt/go/libraries/utils/concurrentmap" "github.com/dolthub/dolt/go/libraries/utils/config" "github.com/dolthub/dolt/go/libraries/utils/filesys" @@ -1350,40 +1349,6 @@ func (dEnv *DoltEnv) TempTableFilesDir() (string, error) { return absPath, nil } -func (dEnv *DoltEnv) DbEaFactory(ctx context.Context) (editor.DbEaFactory, error) { - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, err - } - - db := dEnv.DoltDB(ctx) - if db == nil { - if dEnv.DBLoadError != nil { - return nil, dEnv.DBLoadError - } - return nil, errors.New("DoltDB failed to initialize but no error was recorded") - } - - return editor.NewDbEaFactory(tmpDir, db.ValueReadWriter()), nil -} - -func (dEnv *DoltEnv) BulkDbEaFactory(ctx context.Context) (editor.DbEaFactory, error) { - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, err - } - - db := dEnv.DoltDB(ctx) - if db == nil { - if dEnv.DBLoadError != nil { - return nil, dEnv.DBLoadError - } - return nil, errors.New("DoltDB failed to initialize but no error was recorded") - } - - return editor.NewBulkImportTEAFactory(db.ValueReadWriter(), tmpDir), nil -} - func (dEnv *DoltEnv) IsAccessModeReadOnly(ctx context.Context) (bool, error) { db := dEnv.DoltDB(ctx) if db == nil { diff --git a/go/libraries/doltcore/env/memory.go b/go/libraries/doltcore/env/memory.go deleted file mode 100644 index a2d3d1736e..0000000000 --- a/go/libraries/doltcore/env/memory.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package env - -import ( - "context" - "fmt" - "os" - "time" - - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" - "github.com/dolthub/dolt/go/libraries/doltcore/ref" - "github.com/dolthub/dolt/go/libraries/utils/concurrentmap" - "github.com/dolthub/dolt/go/libraries/utils/config" - "github.com/dolthub/dolt/go/store/chunks" - "github.com/dolthub/dolt/go/store/datas" - "github.com/dolthub/dolt/go/store/hash" -) - -func NewMemoryDbData(ctx context.Context, cfg config.ReadableConfig) (DbData[context.Context], error) { - branchName := GetDefaultInitBranch(cfg) - - ddb, err := NewMemoryDoltDB(ctx, branchName) - if err != nil { - return DbData[context.Context]{}, err - } - - rs, err := NewMemoryRepoState(ctx, ddb, branchName) - if err != nil { - return DbData[context.Context]{}, err - } - - return DbData[context.Context]{ - Ddb: ddb, - Rsw: rs, - Rsr: rs, - }, nil -} - -func NewMemoryDoltDB(ctx context.Context, initBranch string) (*doltdb.DoltDB, error) { - ts := &chunks.TestStorage{} - cs := ts.NewViewWithDefaultFormat() - ddb, err := doltdb.DoltDBFromCS(cs, "") - if err != nil { - return nil, err - } - - m := "memory" - branchRef := ref.NewBranchRef(initBranch) - err = ddb.WriteEmptyRepoWithCommitTimeAndDefaultBranch(ctx, m, m, datas.CommitterDate(), branchRef) - if err != nil { - return nil, err - } - - return ddb, nil -} - -func NewMemoryRepoState(ctx context.Context, ddb *doltdb.DoltDB, initBranch string) (MemoryRepoState, error) { - head := ref.NewBranchRef(initBranch) - rs := MemoryRepoState{ - DoltDB: ddb, - Head: head, - } - - commit, err := ddb.ResolveCommitRef(ctx, head) - if err != nil { - return MemoryRepoState{}, err - } - - root, err := commit.GetRootValue(ctx) - if err != nil { - return MemoryRepoState{}, err - } - - err = rs.UpdateWorkingRoot(ctx, root) - if err != nil { - return MemoryRepoState{}, err - } - - err = rs.UpdateStagedRoot(ctx, root) - if err != nil { - return MemoryRepoState{}, err - } - - return rs, nil -} - -type MemoryRepoState struct { - DoltDB *doltdb.DoltDB - Head ref.DoltRef -} - -var _ RepoStateReader[context.Context] = MemoryRepoState{} -var _ RepoStateWriter = MemoryRepoState{} - -func (m MemoryRepoState) CWBHeadRef(context.Context) (ref.DoltRef, error) { - return m.Head, nil -} - -func (m MemoryRepoState) CWBHeadSpec(ctx context.Context) (*doltdb.CommitSpec, error) { - headRef, err := m.CWBHeadRef(ctx) - if err != nil { - return nil, err - } - spec, err := doltdb.NewCommitSpec(headRef.GetPath()) - if err != nil { - return nil, err - } - return spec, nil -} - -func (m MemoryRepoState) UpdateStagedRoot(ctx context.Context, newRoot doltdb.RootValue) error { - var h hash.Hash - var wsRef ref.WorkingSetRef - - ws, err := m.WorkingSet(ctx) - if err == doltdb.ErrWorkingSetNotFound { - // first time updating root - headRef, err := m.CWBHeadRef(ctx) - if err != nil { - return err - } - wsRef, err = ref.WorkingSetRefForHead(headRef) - if err != nil { - return err - } - ws = doltdb.EmptyWorkingSet(wsRef).WithWorkingRoot(newRoot).WithStagedRoot(newRoot) - } else if err != nil { - return err - } else { - h, err = ws.HashOf() - if err != nil { - return err - } - - wsRef = ws.Ref() - } - - return m.DoltDB.UpdateWorkingSet(ctx, wsRef, ws.WithStagedRoot(newRoot), h, m.workingSetMeta(), nil) -} - -func (m MemoryRepoState) UpdateWorkingRoot(ctx context.Context, newRoot doltdb.RootValue) error { - var h hash.Hash - var wsRef ref.WorkingSetRef - - ws, err := m.WorkingSet(ctx) - if err == doltdb.ErrWorkingSetNotFound { - // first time updating root - headRef, err := m.CWBHeadRef(ctx) - if err != nil { - return err - } - wsRef, err = ref.WorkingSetRefForHead(headRef) - if err != nil { - return err - } - ws = doltdb.EmptyWorkingSet(wsRef).WithWorkingRoot(newRoot).WithStagedRoot(newRoot) - } else if err != nil { - return err - } else { - h, err = ws.HashOf() - if err != nil { - return err - } - - wsRef = ws.Ref() - } - - return m.DoltDB.UpdateWorkingSet(ctx, wsRef, ws.WithWorkingRoot(newRoot), h, m.workingSetMeta(), nil) -} - -func (m MemoryRepoState) WorkingSet(ctx context.Context) (*doltdb.WorkingSet, error) { - headRef, err := m.CWBHeadRef(ctx) - if err != nil { - return nil, err - } - workingSetRef, err := ref.WorkingSetRefForHead(headRef) - if err != nil { - return nil, err - } - - workingSet, err := m.DoltDB.ResolveWorkingSet(ctx, workingSetRef) - if err != nil { - return nil, err - } - - return workingSet, nil -} - -func (m MemoryRepoState) workingSetMeta() *datas.WorkingSetMeta { - return &datas.WorkingSetMeta{ - Timestamp: uint64(time.Now().Unix()), - Description: "updated from dolt environment", - } -} - -func (m MemoryRepoState) SetCWBHeadRef(_ context.Context, r ref.MarshalableRef) (err error) { - m.Head = r.Ref - return -} - -func (m MemoryRepoState) GetRemotes() (*concurrentmap.Map[string, Remote], error) { - return concurrentmap.New[string, Remote](), nil -} - -func (m MemoryRepoState) AddRemote(r Remote) error { - return fmt.Errorf("cannot insert a remote in a memory database") -} - -func (m MemoryRepoState) GetBranches() (*concurrentmap.Map[string, BranchConfig], error) { - return concurrentmap.New[string, BranchConfig](), nil -} - -func (m MemoryRepoState) UpdateBranch(name string, new BranchConfig) error { - return nil -} - -func (m MemoryRepoState) RemoveRemote(ctx context.Context, name string) error { - return fmt.Errorf("cannot delete a remote from a memory database") -} - -func (m MemoryRepoState) TempTableFilesDir() (string, error) { - return os.TempDir(), nil -} - -func (m MemoryRepoState) GetBackups() (*concurrentmap.Map[string, Remote], error) { - panic("cannot get backups on in memory database") -} - -func (m MemoryRepoState) AddBackup(r Remote) error { - panic("cannot add backup to in memory database") -} - -func (m MemoryRepoState) RemoveBackup(ctx context.Context, name string) error { - panic("cannot remove backup from in memory database") -} diff --git a/go/libraries/doltcore/merge/data_merge_test.go b/go/libraries/doltcore/merge/data_merge_test.go index 8933b60a0b..a81f41143d 100644 --- a/go/libraries/doltcore/merge/data_merge_test.go +++ b/go/libraries/doltcore/merge/data_merge_test.go @@ -127,7 +127,6 @@ func testDataMergeHelper(t *testing.T, tests []dataMergeTest, flipSides bool) { var mo merge.MergeOpts var eo editor.Options - eo = eo.WithDeaf(editor.NewInMemDeaf(a.VRW())) // attempt merge before skipping to assert no panics result, err := merge.MergeRoots(sql.NewContext(ctx), doltdb.SimpleTableResolver{}, l, r, a, rootish{r}, rootish{a}, eo, mo) @@ -147,7 +146,6 @@ func testDataMergeHelper(t *testing.T, tests []dataMergeTest, flipSides bool) { func setupDataMergeTest(ctx context.Context, t *testing.T, schema namedSchema, test dataTest) (anc, left, right, merged doltdb.RootValue) { denv := dtestutils.CreateTestEnv() var eo editor.Options - eo = eo.WithDeaf(editor.NewInMemDeaf(denv.DoltDB(ctx).ValueReadWriter())) ancestorTable := tbl(schema, test.ancestor...) anc = makeRootWithTable(t, denv.DoltDB(ctx), eo, *ancestorTable) diff --git a/go/libraries/doltcore/merge/keyless_integration_test.go b/go/libraries/doltcore/merge/keyless_integration_test.go index 55fedc7971..05d3988fa3 100644 --- a/go/libraries/doltcore/merge/keyless_integration_test.go +++ b/go/libraries/doltcore/merge/keyless_integration_test.go @@ -458,18 +458,6 @@ type keylessEntry struct { c2 int } -func (e keylessEntries) toTupleSet() tupleSet { - tups := make([]types.Tuple, len(e)) - for i, t := range e { - tups[i] = t.ToNomsTuple() - } - return mustTupleSet(tups...) -} - -func (e keylessEntry) ToNomsTuple() types.Tuple { - return dtu.MustTuple(cardTag, types.Uint(e.card), c1Tag, types.Int(e.c1), c2Tag, types.Int(e.c2)) -} - func (e keylessEntry) HashAndValue() ([]byte, val.Tuple, error) { valBld.PutUint64(0, uint64(e.card)) valBld.PutInt64(1, int64(e.c1)) @@ -497,14 +485,6 @@ func (e conflictEntries) toConflictSet(t *testing.T) conflictSet { return s } -func (e conflictEntries) toTupleSet() tupleSet { - tups := make([]types.Tuple, len(e)) - for i, t := range e { - tups[i] = t.ToNomsTuple() - } - return mustTupleSet(tups...) -} - func (e conflictEntry) Key(t *testing.T) (h [16]byte) { if e.base != nil { h2, _, err := e.base.HashAndValue() @@ -528,34 +508,6 @@ func (e conflictEntry) Key(t *testing.T) (h [16]byte) { return } -func (e conflictEntry) ToNomsTuple() types.Tuple { - var b, o, t types.Value = types.NullValue, types.NullValue, types.NullValue - if e.base != nil { - b = e.base.ToNomsTuple() - } - if e.ours != nil { - o = e.ours.ToNomsTuple() - } - if e.theirs != nil { - t = e.theirs.ToNomsTuple() - } - return dtu.MustTuple(b, o, t) -} - -type tupleSet map[hash.Hash]types.Tuple - -func mustTupleSet(tt ...types.Tuple) (s tupleSet) { - s = make(tupleSet, len(tt)) - for _, tup := range tt { - h, err := tup.Hash(types.Format_Default) - if err != nil { - panic(err) - } - s[h] = tup - } - return -} - type hash128Set map[[16]byte]val.Tuple func mustHash128Set(entries ...keylessEntry) (s hash128Set) { diff --git a/go/libraries/doltcore/merge/merge.go b/go/libraries/doltcore/merge/merge.go index 573570beef..c072f3f0e6 100644 --- a/go/libraries/doltcore/merge/merge.go +++ b/go/libraries/doltcore/merge/merge.go @@ -413,14 +413,6 @@ type ArtifactStatus struct { ConstraintViolationsTables []string } -func (as ArtifactStatus) HasConflicts() bool { - return len(as.DataConflictTables) > 0 || len(as.SchemaConflictsTables) > 0 -} - -func (as ArtifactStatus) HasConstraintViolations() bool { - return len(as.ConstraintViolationsTables) > 0 -} - // MergeWouldStompChanges returns list of table names that are stomped and the diffs map between head and working set. func MergeWouldStompChanges(ctx context.Context, roots doltdb.Roots, mergeCommit *doltdb.Commit) ([]doltdb.TableName, map[doltdb.TableName]hash.Hash, error) { mergeRoot, err := mergeCommit.GetRootValue(ctx) diff --git a/go/libraries/doltcore/merge/merge_test.go b/go/libraries/doltcore/merge/merge_test.go index f965067511..6244e5be85 100644 --- a/go/libraries/doltcore/merge/merge_test.go +++ b/go/libraries/doltcore/merge/merge_test.go @@ -32,7 +32,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/table/editor" "github.com/dolthub/dolt/go/libraries/doltcore/table/editor/creation" filesys2 "github.com/dolthub/dolt/go/libraries/utils/filesys" - "github.com/dolthub/dolt/go/libraries/utils/valutil" "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/pool" "github.com/dolthub/dolt/go/store/prolly" @@ -76,10 +75,6 @@ func (v rowV) value() val.Tuple { return tup } -func (v rowV) nomsValue() types.Value { - return valsToTestTupleWithoutPks([]types.Value{types.Int(v.col1), types.Int(v.col2)}) -} - const ( NoopAction ActionType = iota InsertAction @@ -505,68 +500,6 @@ func rebuildAllProllyIndexes(ctx *sql.Context, tbl *doltdb.Table) (*doltdb.Table return tbl.SetIndexSet(ctx, indexes) } -func calcExpectedStats(t *testing.T) *MergeStats { - s := &MergeStats{Operation: TableModified} - for _, testCase := range testRows { - if (testCase.leftAction == InsertAction) != (testCase.rightAction == InsertAction) { - if testCase.leftAction == UpdateAction || testCase.rightAction == UpdateAction || - testCase.leftAction == DeleteAction || testCase.rightAction == DeleteAction { - // Either the row exists in the ancestor commit and we are - // deleting or updating it, or the row doesn't exist and we are - // inserting it. - t.Fatalf("it's impossible for an insert to be paired with an update or delete") - } - } - - if testCase.leftAction == NoopAction { - switch testCase.rightAction { - case NoopAction: - case DeleteAction: - s.Deletes++ - case InsertAction: - s.Adds++ - case UpdateAction: - s.Modifications++ - } - continue - } - - if testCase.rightAction == NoopAction { - switch testCase.leftAction { - case NoopAction: - case DeleteAction: - s.Deletes++ - case InsertAction: - s.Adds++ - case UpdateAction: - s.Modifications++ - } - continue - } - - if testCase.conflict { - // (UpdateAction, DeleteAction), - // (DeleteAction, UpdateAction), - // (UpdateAction, UpdateAction) with conflict, - // (InsertAction, InsertAction) with conflict - s.DataConflicts++ - continue - } - - if testCase.leftAction == InsertAction && testCase.rightAction == InsertAction { - // Equivalent inserts - continue - } - - if !valutil.NilSafeEqCheck(unwrapNoms(testCase.leftValue), unwrapNoms(testCase.rightValue)) { - s.Modifications++ - continue - } - } - - return s -} - func mustMakeEmptyRepo(t *testing.T) *doltdb.DoltDB { ddb, _ := doltdb.LoadDoltDB(context.Background(), types.Format_Default, doltdb.InMemDoltDB, filesys2.LocalFS) err := ddb.WriteEmptyRepo(context.Background(), env.DefaultInitBranch, name, email) @@ -645,40 +578,6 @@ func key(i int) val.Tuple { return tup } -func nomsKey(i int) types.Value { - return mustTuple(types.NewTuple(types.Format_Default, types.Uint(idTag), types.Int(i))) -} - -func unwrap(v *rowV) val.Tuple { - if v == nil { - return nil - } - return v.value() -} - -func unwrapNoms(v *rowV) types.Value { - if v == nil { - return nil - } - return v.nomsValue() -} - -func mustTuple(tpl types.Tuple, err error) types.Tuple { - if err != nil { - panic(err) - } - - return tpl -} - -func mustString(str string, err error) string { - if err != nil { - panic(err) - } - - return str -} - func MustDebugFormatProlly(t *testing.T, m prolly.Map) string { s, err := prolly.DebugFormat(context.Background(), m) require.NoError(t, err) diff --git a/go/libraries/doltcore/merge/row_merge_test.go b/go/libraries/doltcore/merge/row_merge_test.go index 2df992dbb8..3fd67cd0de 100644 --- a/go/libraries/doltcore/merge/row_merge_test.go +++ b/go/libraries/doltcore/merge/row_merge_test.go @@ -26,15 +26,6 @@ import ( "github.com/dolthub/dolt/go/store/val" ) -type nomsRowMergeTest struct { - name string - row, mergeRow, ancRow types.Value - sch schema.Schema - expectedResult types.Value - expectCellMerge bool - expectConflict bool -} - type rowMergeTest struct { name string row, mergeRow, ancRow val.Tuple @@ -69,39 +60,6 @@ func build(ints ...int) []*int { return out } -var convergentEditCases = []testCase{ - { - "add same row", - build(1, 2), - build(1, 2), - nil, - 2, 2, 2, - build(1, 2), - false, - false, - }, - { - "both delete row", - nil, - nil, - build(1, 2), - 2, 2, 2, - nil, - false, - false, - }, - { - "modify row to equal value", - build(2, 2), - build(2, 2), - build(1, 1), - 2, 2, 2, - build(2, 2), - false, - false, - }, -} - var testCases = []testCase{ { "insert different rows", @@ -221,35 +179,6 @@ func TestRowMerge(t *testing.T) { } } -func valsToTestTupleWithoutPks(vals []types.Value) types.Value { - return valsToTestTuple(vals, false) -} - -func valsToTestTupleWithPks(vals []types.Value) types.Value { - return valsToTestTuple(vals, true) -} - -func valsToTestTuple(vals []types.Value, includePrimaryKeys bool) types.Value { - if vals == nil { - return nil - } - - tplVals := make([]types.Value, 0, 2*len(vals)) - for i, val := range vals { - if !types.IsNull(val) { - tag := i - // Assume one primary key tag, add 1 to all other tags - if includePrimaryKeys { - tag++ - } - tplVals = append(tplVals, types.Uint(tag)) - tplVals = append(tplVals, val) - } - } - - return mustTuple(types.NewTuple(types.Format_Default, tplVals...)) -} - func createRowMergeStruct(t testCase) rowMergeTest { mergedSch := calcMergedSchema(t) leftSch := calcSchema(t.rowCnt) @@ -269,16 +198,6 @@ func createRowMergeStruct(t testCase) rowMergeTest { t.expectConflict} } -func createNomsRowMergeStruct(t testCase) nomsRowMergeTest { - sch := calcMergedSchema(t) - - tpl := valsToTestTupleWithPks(toVals(t.row)) - mergeTpl := valsToTestTupleWithPks(toVals(t.mergeRow)) - ancTpl := valsToTestTupleWithPks(toVals(t.ancRow)) - expectedTpl := valsToTestTupleWithPks(toVals(t.expectedResult)) - return nomsRowMergeTest{t.name, tpl, mergeTpl, ancTpl, sch, expectedTpl, t.expectCellMerge, t.expectConflict} -} - func calcMergedSchema(t testCase) schema.Schema { longest := t.rowCnt if t.mRowCnt > longest { @@ -323,20 +242,3 @@ func buildTup(sch schema.Schema, r []*int) val.Tuple { } return tup } - -func toVals(ints []*int) []types.Value { - if ints == nil { - return nil - } - - v := make([]types.Value, len(ints)) - for i, d := range ints { - if d == nil { - v[i] = types.NullValue - continue - } - - v[i] = types.Int(*d) - } - return v -} diff --git a/go/libraries/doltcore/merge/schema_merge_test.go b/go/libraries/doltcore/merge/schema_merge_test.go index 2e495c0d6e..674971215c 100644 --- a/go/libraries/doltcore/merge/schema_merge_test.go +++ b/go/libraries/doltcore/merge/schema_merge_test.go @@ -1650,7 +1650,6 @@ func testSchemaMergeHelper(t *testing.T, tests []schemaMergeTest, flipSides bool var mo merge.MergeOpts var eo editor.Options - eo = eo.WithDeaf(editor.NewInMemDeaf(a.VRW())) // attempt merge before skipping to assert no panics result, err := merge.MergeRoots(sql.NewContext(ctx), doltdb.SimpleTableResolver{}, l, r, a, rootish{r}, rootish{a}, eo, mo) maybeSkip(t, test, flipSides) @@ -1693,7 +1692,6 @@ func testSchemaMergeHelper(t *testing.T, tests []schemaMergeTest, flipSides bool func setupSchemaMergeTest(ctx context.Context, t *testing.T, test schemaMergeTest) (anc, left, right, merged doltdb.RootValue) { denv := dtestutils.CreateTestEnv() var eo editor.Options - eo = eo.WithDeaf(editor.NewInMemDeaf(denv.DoltDB(ctx).ValueReadWriter())) anc = makeRootWithTable(t, denv.DoltDB(ctx), eo, test.ancestor) assert.NotNil(t, anc) if test.left != nil { diff --git a/go/libraries/doltcore/merge/violations_fk.go b/go/libraries/doltcore/merge/violations_fk.go index 0b1dab5290..9cfece09a2 100644 --- a/go/libraries/doltcore/merge/violations_fk.go +++ b/go/libraries/doltcore/merge/violations_fk.go @@ -20,12 +20,10 @@ import ( "fmt" "github.com/dolthub/go-mysql-server/sql" - gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/schema" - json2 "github.com/dolthub/dolt/go/libraries/doltcore/sqle/json" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/types" @@ -543,16 +541,3 @@ func foreignKeyCVJson(foreignKey doltdb.ForeignKey, sch, refSch schema.Schema) ( return d, nil } - -func jsonDataToNomsValue(ctx context.Context, vrw types.ValueReadWriter, data []byte) (types.JSON, error) { - var doc interface{} - if err := json.Unmarshal(data, &doc); err != nil { - return types.JSON{}, err - } - sqlDoc := gmstypes.JSONDocument{Val: doc} - nomsJson, err := json2.NomsJSONFromJSONValue(ctx, vrw, sqlDoc) - if err != nil { - return types.JSON{}, err - } - return types.JSON(nomsJson), nil -} diff --git a/go/libraries/doltcore/mvdata/data_loc_test.go b/go/libraries/doltcore/mvdata/data_loc_test.go index 6393030f54..f49a5eb63f 100644 --- a/go/libraries/doltcore/mvdata/data_loc_test.go +++ b/go/libraries/doltcore/mvdata/data_loc_test.go @@ -86,7 +86,7 @@ func TestBasics(t *testing.T) { {NewDataLocation("file.csv", ""), CsvFile.ReadableStr() + ":file.csv", true}, {NewDataLocation("file.psv", ""), PsvFile.ReadableStr() + ":file.psv", true}, {NewDataLocation("file.json", ""), JsonFile.ReadableStr() + ":file.json", true}, - //{NewDataLocation("file.nbf", ""), NbfFile, "file.nbf", true}, + // {NewDataLocation("file.nbf", ""), NbfFile, "file.nbf", true}, } for _, test := range tests { @@ -133,7 +133,7 @@ func TestExists(t *testing.T) { NewDataLocation("file.csv", ""), NewDataLocation("file.psv", ""), NewDataLocation("file.json", ""), - //NewDataLocation("file.nbf", ""), + // NewDataLocation("file.nbf", ""), } ddb, root, fs := createRootAndFS() @@ -192,7 +192,7 @@ func TestCreateRdWr(t *testing.T) { {NewDataLocation("file.csv", ""), reflect.TypeOf((*csv.CSVReader)(nil)).Elem(), reflect.TypeOf((*csv.CSVWriter)(nil)).Elem()}, {NewDataLocation("file.psv", ""), reflect.TypeOf((*csv.CSVReader)(nil)).Elem(), reflect.TypeOf((*csv.CSVWriter)(nil)).Elem()}, {NewDataLocation("file.json", ""), reflect.TypeOf((*json.JSONReader)(nil)).Elem(), reflect.TypeOf((*json.RowWriter)(nil)).Elem()}, - //{NewDataLocation("file.nbf", ""), reflect.TypeOf((*nbf.NBFReader)(nil)).Elem(), reflect.TypeOf((*nbf.NBFWriter)(nil)).Elem()}, + // {NewDataLocation("file.nbf", ""), reflect.TypeOf((*nbf.NBFReader)(nil)).Elem(), reflect.TypeOf((*nbf.NBFWriter)(nil)).Elem()}, } ctx := context.Background() @@ -220,16 +220,6 @@ func TestCreateRdWr(t *testing.T) { loc := test.dl - tmpDir, tdErr := dEnv.TempTableFilesDir() - if tdErr != nil { - t.Fatal("Unexpected error accessing .dolt directory.", tdErr) - } - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - t.Fatal("Unexpected error accessing .dolt directory.", err) - } - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - filePath, fpErr := dEnv.FS.Abs(strings.Split(loc.String(), ":")[1]) if fpErr != nil { t.Fatal("Unexpected error getting filepath", fpErr) @@ -240,7 +230,7 @@ func TestCreateRdWr(t *testing.T) { t.Fatal("Unexpected error opening file for writer.", wrErr) } - wr, wErr := loc.NewCreatingWriter(context.Background(), mvOpts, root, fakeSchema, opts, writer) + wr, wErr := loc.NewCreatingWriter(context.Background(), mvOpts, root, fakeSchema, editor.Options{}, writer) if wErr != nil { t.Fatal("Unexpected error creating writer.", wErr) } diff --git a/go/libraries/doltcore/mvdata/engine_table_writer.go b/go/libraries/doltcore/mvdata/engine_table_writer.go index af7abce5eb..7ee5752d01 100644 --- a/go/libraries/doltcore/mvdata/engine_table_writer.go +++ b/go/libraries/doltcore/mvdata/engine_table_writer.go @@ -31,7 +31,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/overrides" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" "github.com/dolthub/dolt/go/store/types" ) @@ -40,6 +39,9 @@ const ( tableWriterStatUpdateRate = 64 * 1024 ) +// StatsCb is a callback for reporting stats about the rows that have been processed so far +type StatsCb func(types.AppliedEditStats) + // SqlEngineTableWriter is a utility for importing a set of rows through the sql engine. type SqlEngineTableWriter struct { se *sqle.Engine @@ -51,7 +53,7 @@ type SqlEngineTableWriter struct { force bool disableFks bool - statsCB noms.StatsCB + statsCB StatsCb stats types.AppliedEditStats statOps int32 @@ -60,7 +62,7 @@ type SqlEngineTableWriter struct { rowOperationSchema sql.PrimaryKeySchema } -func NewSqlEngineTableWriter(ctx *sql.Context, engine *sqle.Engine, createTableSchema, rowOperationSchema schema.Schema, options *MoverOptions, statsCB noms.StatsCB) (*SqlEngineTableWriter, error) { +func NewSqlEngineTableWriter(ctx *sql.Context, engine *sqle.Engine, createTableSchema, rowOperationSchema schema.Schema, options *MoverOptions, statsCB StatsCb) (*SqlEngineTableWriter, error) { if engine.IsReadOnly() { // SqlEngineTableWriter does not respect read only mode return nil, analyzererrors.ErrReadOnlyDatabase.New(ctx.GetCurrentDatabase()) diff --git a/go/libraries/doltcore/row/row.go b/go/libraries/doltcore/row/row.go index 28766fc3de..aec735c45f 100644 --- a/go/libraries/doltcore/row/row.go +++ b/go/libraries/doltcore/row/row.go @@ -15,8 +15,6 @@ package row import ( - "context" - "errors" "fmt" "github.com/dolthub/dolt/go/libraries/doltcore/schema" @@ -24,8 +22,6 @@ import ( "github.com/dolthub/dolt/go/store/types" ) -var ErrRowNotValid = errors.New("invalid row for current schema") - type Row interface { // Iterates over all the columns in the row. Columns that have no value set will not be visited. IterCols(cb func(tag uint64, val types.Value) (stop bool, err error)) (bool, error) @@ -82,21 +78,6 @@ func FromNoms(sch schema.Schema, nomsKey, nomsVal types.Tuple) (Row, error) { return pkRowFromNoms(sch, nomsKey, nomsVal) } -// ToNoms returns the storage-layer tuples corresponding to |r|. -func ToNoms(ctx context.Context, sch schema.Schema, r Row) (key, val types.Tuple, err error) { - k, err := r.NomsMapKey(sch).Value(ctx) - if err != nil { - return key, val, err - } - - v, err := r.NomsMapValue(sch).Value(ctx) - if err != nil { - return key, val, err - } - - return k.(types.Tuple), v.(types.Tuple), nil -} - func GetFieldByName(colName string, r Row, sch schema.Schema) (types.Value, bool) { col, ok := sch.GetAllCols().GetByName(colName) @@ -123,71 +104,6 @@ func GetFieldByNameWithDefault(colName string, defVal types.Value, r Row, sch sc } } -// ReduceToIndexKeysFromTagMap creates a full key and a partial key from the given map of tags (first tuple being the -// full key). Please refer to the note in the index editor for more information regarding partial keys. -func ReduceToIndexKeysFromTagMap(nbf *types.NomsBinFormat, idx schema.Index, tagToVal map[uint64]types.Value, tf *types.TupleFactory) (types.Tuple, types.Tuple, error) { - vals := make([]types.Value, 0, len(idx.AllTags())*2) - for _, tag := range idx.AllTags() { - val, ok := tagToVal[tag] - if !ok { - val = types.NullValue - } - vals = append(vals, types.Uint(tag), val) - } - - if tf == nil { - fullKey, err := types.NewTuple(nbf, vals...) - if err != nil { - return types.Tuple{}, types.Tuple{}, err - } - - partialKey, err := types.NewTuple(nbf, vals[:idx.Count()*2]...) - if err != nil { - return types.Tuple{}, types.Tuple{}, err - } - - return fullKey, partialKey, nil - } else { - fullKey, err := tf.Create(vals...) - if err != nil { - return types.Tuple{}, types.Tuple{}, err - } - - partialKey, err := tf.Create(vals[:idx.Count()*2]...) - if err != nil { - return types.Tuple{}, types.Tuple{}, err - } - - return fullKey, partialKey, nil - } -} - -// ReduceToIndexPartialKey creates an index record from a primary storage record. -func ReduceToIndexPartialKey(tags []uint64, idx schema.Index, r Row) (types.Tuple, error) { - var vals []types.Value - if idx.Name() != "" { - tags = idx.IndexedColumnTags() - } - for _, tag := range tags { - val, ok := r.GetColVal(tag) - if !ok { - val = types.NullValue - } - vals = append(vals, types.Uint(tag), val) - } - - return types.NewTuple(r.Format(), vals...) -} - -func IsEmpty(r Row) (b bool) { - b = true - _, _ = r.IterCols(func(_ uint64, _ types.Value) (stop bool, err error) { - b = false - return true, nil - }) - return b -} - // IsValid returns whether the row given matches the types and satisfies all the constraints of the schema given. func IsValid(r Row, sch schema.Schema) (bool, error) { column, constraint, err := findInvalidCol(r, sch) @@ -265,77 +181,3 @@ func AreEqual(row1, row2 Row, sch schema.Schema) bool { return true } - -func TaggedValsEqualForSch(tv, other TaggedValues, sch schema.Schema) bool { - if tv == nil && other == nil { - return true - } else if tv == nil || other == nil { - return false - } - - for _, tag := range sch.GetAllCols().Tags { - val1, _ := tv[tag] - val2, _ := other[tag] - - if !valutil.NilSafeEqCheck(val1, val2) { - return false - } - } - - return true -} - -func KeyAndTaggedValuesForRow(r Row, sch schema.Schema) (types.Tuple, TaggedValues, error) { - switch typed := r.(type) { - case nomsRow: - pkCols := sch.GetPKCols() - keyVals := make([]types.Value, 0, pkCols.Size()*2) - tv := make(TaggedValues) - err := pkCols.Iter(func(tag uint64, col schema.Column) (stop bool, err error) { - val, ok := typed.key[tag] - if !ok || types.IsNull(val) { - return false, errors.New("invalid key contains null values") - } - - tv[tag] = val - keyVals = append(keyVals, types.Uint(tag)) - keyVals = append(keyVals, val) - return false, nil - }) - - if err != nil { - return types.Tuple{}, nil, err - } - - nonPkCols := sch.GetNonPKCols() - _, err = typed.value.Iter(func(tag uint64, val types.Value) (stop bool, err error) { - if _, ok := nonPkCols.TagToIdx[tag]; ok { - tv[tag] = val - } - - return false, nil - }) - - if err != nil { - return types.Tuple{}, nil, err - } - - t, err := types.NewTuple(r.Format(), keyVals...) - if err != nil { - return types.Tuple{}, nil, err - } - - return t, tv, nil - - case keylessRow: - tv, err := typed.TaggedValues() - if err != nil { - return types.Tuple{}, nil, err - } - - return typed.key, tv, nil - - default: - panic("unknown row type") - } -} diff --git a/go/libraries/doltcore/row/tagged_values.go b/go/libraries/doltcore/row/tagged_values.go index 3dad2d6b55..6e0aad257a 100644 --- a/go/libraries/doltcore/row/tagged_values.go +++ b/go/libraries/doltcore/row/tagged_values.go @@ -191,60 +191,6 @@ func TaggedValuesFromTupleValueSlice(vals types.TupleValueSlice) (TaggedValues, return taggedTuple, nil } -func TaggedValuesFromTupleKeyAndValue(key, value types.Tuple) (TaggedValues, error) { - tv := make(TaggedValues) - err := AddToTaggedVals(tv, key) - - if err != nil { - return nil, err - } - - err = AddToTaggedVals(tv, value) - - if err != nil { - return nil, err - } - - return tv, nil -} - -func AddToTaggedVals(tv TaggedValues, t types.Tuple) error { - return IterDoltTuple(t, func(tag uint64, val types.Value) error { - tv[tag] = val - return nil - }) -} - -func IterDoltTuple(t types.Tuple, cb func(tag uint64, val types.Value) error) error { - itr, err := t.Iterator() - - if err != nil { - return err - } - - for itr.HasMore() { - _, tag, err := itr.NextUint64() - - if err != nil { - return err - } - - _, currVal, err := itr.Next() - - if err != nil { - return err - } - - err = cb(tag, currVal) - - if err != nil { - return err - } - } - - return nil -} - func (tt TaggedValues) String() string { str := "{" for k, v := range tt { @@ -260,36 +206,3 @@ func (tt TaggedValues) String() string { str += "\n}" return str } - -// CountCellDiffs returns the number of fields that are different between two -// tuples and does not panic if tuples are different lengths. -func CountCellDiffs(from, to types.Tuple, fromSch, toSch schema.Schema) (uint64, error) { - fromColLen := len(fromSch.GetAllCols().GetColumns()) - toColLen := len(toSch.GetAllCols().GetColumns()) - changed := 0 - f, err := ParseTaggedValues(from) - if err != nil { - return 0, err - } - - t, err := ParseTaggedValues(to) - if err != nil { - return 0, err - } - - for i, v := range f { - ov, ok := t[i] - // !ok means t[i] has NULL value, and it is not cell modify if it was from drop column or add column - if (!ok && fromColLen == toColLen) || (ok && !v.Equals(ov)) { - changed++ - } - } - - for i := range t { - if f[i] == nil { - changed++ - } - } - - return uint64(changed), nil -} diff --git a/go/libraries/doltcore/rowconv/row_converter.go b/go/libraries/doltcore/rowconv/row_converter.go index eb5edc5648..13ab76e145 100644 --- a/go/libraries/doltcore/rowconv/row_converter.go +++ b/go/libraries/doltcore/rowconv/row_converter.go @@ -18,12 +18,9 @@ import ( "context" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/store/types" - - "github.com/dolthub/go-mysql-server/sql" ) var IdentityConverter = &RowConverter{nil, true, nil} @@ -96,56 +93,6 @@ func panicOnDuplicateMappings(mapping *FieldMapping) { } } -// ConvertWithWarnings takes an input row, maps its columns to their destination columns, performing any type -// conversions needed to create a row of the expected destination schema, and uses the optional WarnFunction -// callback to let callers handle logging a warning when a field cannot be cleanly converted. -func (rc *RowConverter) ConvertWithWarnings(inRow row.Row, warnFn WarnFunction) (row.Row, error) { - return rc.convert(inRow, warnFn) -} - -// convert takes a row and maps its columns to their destination columns, automatically performing any type conversion -// needed, and using the optional WarnFunction to let callers log warnings on any type conversion errors. -func (rc *RowConverter) convert(inRow row.Row, warnFn WarnFunction) (row.Row, error) { - if rc.IdentityConverter { - return inRow, nil - } - - outTaggedVals := make(row.TaggedValues, len(rc.SrcToDest)) - _, err := inRow.IterCols(func(tag uint64, val types.Value) (stop bool, err error) { - convFunc, ok := rc.ConvFuncs[tag] - - if ok { - outTag := rc.SrcToDest[tag] - outVal, err := convFunc(val) - - if sql.ErrInvalidValue.Is(err) && warnFn != nil { - col, _ := rc.SrcSch.GetAllCols().GetByTag(tag) - warnFn(DatatypeCoercionFailureWarningCode, DatatypeCoercionFailureWarning, col.Name) - outVal = types.NullValue - err = nil - } - - if err != nil { - return false, err - } - - if types.IsNull(outVal) { - return false, nil - } - - outTaggedVals[outTag] = outVal - } - - return false, nil - }) - - if err != nil { - return nil, err - } - - return row.New(inRow.Format(), rc.DestSch, outTaggedVals) -} - func IsNecessary(srcSch, destSch schema.Schema, destToSrc map[uint64]uint64) (bool, error) { srcCols := srcSch.GetAllCols() destCols := destSch.GetAllCols() diff --git a/go/libraries/doltcore/schema/constraint.go b/go/libraries/doltcore/schema/constraint.go index 95b227315a..2d984d1b1e 100644 --- a/go/libraries/doltcore/schema/constraint.go +++ b/go/libraries/doltcore/schema/constraint.go @@ -42,18 +42,6 @@ const ( NotNullConstraintType = "not_null" ) -// ColConstraintFromTypeAndParams takes in a string representing the type of the constraint and a map of parameters -// that can be used to determine the behavior of the constraint. An example might be a constraint which validated -// a value is in a given range. For this the constraint type might by "in_range_constraint", and the parameters might -// be {"min": -10, "max": 10} -func ColConstraintFromTypeAndParams(colCnstType string, params map[string]string) ColConstraint { - switch colCnstType { - case NotNullConstraintType: - return NotNullConstraint{} - } - panic("Unknown column constraint type: " + colCnstType) -} - // NotNullConstraint validates that a value is not null. It does not restrict 0 length strings, or 0 valued ints, or // anything other than non nil values type NotNullConstraint struct{} diff --git a/go/libraries/doltcore/schema/typeinfo/blobstring.go b/go/libraries/doltcore/schema/typeinfo/blobstring.go index 59ffc758d7..97e0ade86d 100644 --- a/go/libraries/doltcore/schema/typeinfo/blobstring.go +++ b/go/libraries/doltcore/schema/typeinfo/blobstring.go @@ -24,7 +24,6 @@ import ( "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" - "github.com/dolthub/vitess/go/sqltypes" "github.com/dolthub/dolt/go/store/types" ) @@ -45,38 +44,10 @@ type blobStringType struct { var _ TypeInfo = (*blobStringType)(nil) var ( - TinyTextType TypeInfo = &blobStringType{sqlStringType: gmstypes.TinyText} - TextType TypeInfo = &blobStringType{sqlStringType: gmstypes.Text} - MediumTextType TypeInfo = &blobStringType{sqlStringType: gmstypes.MediumText} - LongTextType TypeInfo = &blobStringType{sqlStringType: gmstypes.LongText} + TextType TypeInfo = &blobStringType{sqlStringType: gmstypes.Text} + LongTextType TypeInfo = &blobStringType{sqlStringType: gmstypes.LongText} ) -func CreateBlobStringTypeFromParams(params map[string]string) (TypeInfo, error) { - collationStr, ok := params[blobStringTypeParam_Collate] - if !ok { - return nil, fmt.Errorf(`create blobstring type info is missing param "%v"`, blobStringTypeParam_Collate) - } - collation, err := sql.ParseCollation("", collationStr, false) - if err != nil { - return nil, err - } - - maxLengthStr, ok := params[blobStringTypeParam_Length] - if !ok { - return nil, fmt.Errorf(`create blobstring type info is missing param "%v"`, blobStringTypeParam_Length) - } - length, err := strconv.ParseInt(maxLengthStr, 10, 64) - if err != nil { - return nil, err - } - - sqlType, err := gmstypes.CreateString(sqltypes.Text, length, collation) - if err != nil { - return nil, err - } - return &blobStringType{sqlType}, nil -} - // ConvertNomsValueToValue implements TypeInfo interface. func (ti *blobStringType) ConvertNomsValueToValue(v types.Value) (interface{}, error) { if val, ok := v.(types.Blob); ok { diff --git a/go/libraries/doltcore/schema/typeinfo/common_test.go b/go/libraries/doltcore/schema/typeinfo/common_test.go index 3c3df0dac6..7ef3834054 100644 --- a/go/libraries/doltcore/schema/typeinfo/common_test.go +++ b/go/libraries/doltcore/schema/typeinfo/common_test.go @@ -116,29 +116,6 @@ func generateSetType(t *testing.T, numOfElements int) *setType { return &setType{gmstypes.MustCreateSetType(vals, sql.Collation_Default)} } -func generateInlineBlobTypes(t *testing.T, numOfTypes uint16) []TypeInfo { - var res []TypeInfo - loop(t, 1, 500, numOfTypes, func(i int64) { - pad := false - if i%2 == 0 { - pad = true - } - res = append(res, generateInlineBlobType(t, i, pad)) - }) - return res -} - -func generateInlineBlobType(t *testing.T, length int64, pad bool) *inlineBlobType { - require.True(t, length > 0) - if pad { - t, err := gmstypes.CreateBinary(sqltypes.Binary, length) - if err == nil { - return &inlineBlobType{t} - } - } - return &inlineBlobType{gmstypes.MustCreateBinary(sqltypes.VarBinary, length)} -} - func generateVarStringTypes(t *testing.T, numOfTypes uint16) []TypeInfo { var res []TypeInfo loop(t, 1, 500, numOfTypes, func(i int64) { diff --git a/go/libraries/doltcore/schema/typeinfo/datetime.go b/go/libraries/doltcore/schema/typeinfo/datetime.go index 88c69d5e80..b9be59452e 100644 --- a/go/libraries/doltcore/schema/typeinfo/datetime.go +++ b/go/libraries/doltcore/schema/typeinfo/datetime.go @@ -17,7 +17,6 @@ package typeinfo import ( "context" "fmt" - "strconv" "time" "github.com/dolthub/go-mysql-server/sql" @@ -50,39 +49,6 @@ func CreateDatetimeTypeFromSqlType(typ sql.DatetimeType) *datetimeType { return &datetimeType{typ} } -func CreateDatetimeTypeFromParams(params map[string]string) (TypeInfo, error) { - if sqlType, ok := params[datetimeTypeParam_SQL]; ok { - precision := 6 - if precisionParam, ok := params[datetimeTypeParam_Precision]; ok { - var err error - precision, err = strconv.Atoi(precisionParam) - if err != nil { - return nil, err - } - } - switch sqlType { - case datetimeTypeParam_SQL_Date: - return DateType, nil - case datetimeTypeParam_SQL_Datetime: - gmsType, err := gmstypes.CreateDatetimeType(sqltypes.Datetime, precision) - if err != nil { - return nil, err - } - return CreateDatetimeTypeFromSqlType(gmsType), nil - case datetimeTypeParam_SQL_Timestamp: - gmsType, err := gmstypes.CreateDatetimeType(sqltypes.Timestamp, precision) - if err != nil { - return nil, err - } - return CreateDatetimeTypeFromSqlType(gmsType), nil - default: - return nil, fmt.Errorf(`create datetime type info has invalid param "%v"`, sqlType) - } - } else { - return nil, fmt.Errorf(`create datetime type info is missing param "%v"`, datetimeTypeParam_SQL) - } -} - // ConvertNomsValueToValue implements TypeInfo interface. func (ti *datetimeType) ConvertNomsValueToValue(v types.Value) (interface{}, error) { if val, ok := v.(types.Timestamp); ok { @@ -121,7 +87,7 @@ func (ti *datetimeType) ReadFrom(_ *types.NomsBinFormat, reader types.CodecReade // ConvertValueToNomsValue implements TypeInfo interface. func (ti *datetimeType) ConvertValueToNomsValue(ctx context.Context, vrw types.ValueReadWriter, v interface{}) (types.Value, error) { - //TODO: handle the zero value as a special case that is valid for all ranges + // TODO: handle the zero value as a special case that is valid for all ranges if v == nil { return types.NullValue, nil } diff --git a/go/libraries/doltcore/schema/typeinfo/enum.go b/go/libraries/doltcore/schema/typeinfo/enum.go index b9bfffb8d6..21e653835a 100644 --- a/go/libraries/doltcore/schema/typeinfo/enum.go +++ b/go/libraries/doltcore/schema/typeinfo/enum.go @@ -16,12 +16,10 @@ package typeinfo import ( "context" - "encoding/gob" "fmt" "strings" "github.com/dolthub/go-mysql-server/sql" - gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/dolt/go/store/types" ) @@ -39,34 +37,6 @@ type enumType struct { var _ TypeInfo = (*enumType)(nil) -func CreateEnumTypeFromParams(params map[string]string) (TypeInfo, error) { - var collation sql.CollationID - var err error - if collationStr, ok := params[enumTypeParam_Collation]; ok { - collation, err = sql.ParseCollation("", collationStr, false) - if err != nil { - return nil, err - } - } else { - return nil, fmt.Errorf(`create enum type info is missing param "%v"`, enumTypeParam_Collation) - } - var values []string - if valuesStr, ok := params[enumTypeParam_Values]; ok { - dec := gob.NewDecoder(strings.NewReader(valuesStr)) - err = dec.Decode(&values) - if err != nil { - return nil, err - } - } else { - return nil, fmt.Errorf(`create enum type info is missing param "%v"`, enumTypeParam_Values) - } - sqlEnumType, err := gmstypes.CreateEnumType(values, collation) - if err != nil { - return nil, err - } - return CreateEnumTypeFromSqlEnumType(sqlEnumType), nil -} - func CreateEnumTypeFromSqlEnumType(sqlEnumType sql.EnumType) TypeInfo { return &enumType{sqlEnumType} } diff --git a/go/libraries/doltcore/schema/typeinfo/extended.go b/go/libraries/doltcore/schema/typeinfo/extended.go index 5b412e4147..913e24fc73 100644 --- a/go/libraries/doltcore/schema/typeinfo/extended.go +++ b/go/libraries/doltcore/schema/typeinfo/extended.go @@ -19,15 +19,10 @@ import ( "fmt" "github.com/dolthub/go-mysql-server/sql" - gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/dolt/go/store/types" ) -const ( - extendedTypeParams_string_encoded = "string_encoded" -) - // extendedType is a type that refers to an ExtendedType in GMS. These are only supported in the new format, and have many // more limitations than traditional types (for now). type extendedType struct { @@ -36,18 +31,6 @@ type extendedType struct { var _ TypeInfo = (*extendedType)(nil) -// CreateExtendedTypeFromParams creates a TypeInfo from the given parameter map. -func CreateExtendedTypeFromParams(params map[string]string) (TypeInfo, error) { - if encodedString, ok := params[extendedTypeParams_string_encoded]; ok { - t, err := gmstypes.DeserializeTypeFromString(encodedString) - if err != nil { - return nil, err - } - return &extendedType{t}, nil - } - return nil, fmt.Errorf(`create extended type info is missing "%v" param`, extendedTypeParams_string_encoded) -} - // CreateExtendedTypeFromSqlType creates a TypeInfo from the given extended type. func CreateExtendedTypeFromSqlType(typ sql.ExtendedType) TypeInfo { return &extendedType{typ} diff --git a/go/libraries/doltcore/schema/typeinfo/float.go b/go/libraries/doltcore/schema/typeinfo/float.go index 9e31ab8d61..421baeee10 100644 --- a/go/libraries/doltcore/schema/typeinfo/float.go +++ b/go/libraries/doltcore/schema/typeinfo/float.go @@ -30,12 +30,6 @@ import ( type FloatWidth int8 -const ( - floatTypeParam_Width = "width" - floatTypeParam_Width_32 = "32" - floatTypeParam_Width_64 = "64" -) - type floatType struct { sqlFloatType sql.NumberType } @@ -46,20 +40,6 @@ var ( Float64Type = &floatType{gmstypes.Float64} ) -func CreateFloatTypeFromParams(params map[string]string) (TypeInfo, error) { - if width, ok := params[floatTypeParam_Width]; ok { - switch width { - case floatTypeParam_Width_32: - return Float32Type, nil - case floatTypeParam_Width_64: - return Float64Type, nil - default: - return nil, fmt.Errorf(`create float type info has "%v" param with value "%v"`, floatTypeParam_Width, width) - } - } - return nil, fmt.Errorf(`create float type info is missing "%v" param`, floatTypeParam_Width) -} - // ConvertNomsValueToValue implements TypeInfo interface. func (ti *floatType) ConvertNomsValueToValue(v types.Value) (interface{}, error) { if val, ok := v.(types.Float); ok { diff --git a/go/libraries/doltcore/schema/typeinfo/geometry.go b/go/libraries/doltcore/schema/typeinfo/geometry.go index ca27072b8d..0bb36b3220 100644 --- a/go/libraries/doltcore/schema/typeinfo/geometry.go +++ b/go/libraries/doltcore/schema/typeinfo/geometry.go @@ -17,7 +17,6 @@ package typeinfo import ( "context" "fmt" - "strconv" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" @@ -289,28 +288,6 @@ func geometryTypeConverter(ctx context.Context, src *geometryType, destTi TypeIn } } -func CreateGeometryTypeFromParams(params map[string]string) (TypeInfo, error) { - var ( - err error - sridVal uint64 - def bool - ) - if s, ok := params["SRID"]; ok { - sridVal, err = strconv.ParseUint(s, 10, 32) - if err != nil { - return nil, err - } - } - if d, ok := params["DefinedSRID"]; ok { - def, err = strconv.ParseBool(d) - if err != nil { - return nil, err - } - } - - return CreateGeometryTypeFromSqlGeometryType(gmstypes.GeometryType{SRID: uint32(sridVal), DefinedSRID: def}), nil -} - func CreateGeometryTypeFromSqlGeometryType(sqlGeometryType gmstypes.GeometryType) TypeInfo { return &geometryType{sqlGeometryType: sqlGeometryType} } diff --git a/go/libraries/doltcore/schema/typeinfo/geometry_collection.go b/go/libraries/doltcore/schema/typeinfo/geometry_collection.go index 2390a91c31..7b8179d7d7 100644 --- a/go/libraries/doltcore/schema/typeinfo/geometry_collection.go +++ b/go/libraries/doltcore/schema/typeinfo/geometry_collection.go @@ -17,7 +17,6 @@ package typeinfo import ( "context" "fmt" - "strconv" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" @@ -197,25 +196,3 @@ func geomcollTypeConverter(ctx context.Context, src *geomcollType, destTi TypeIn return nil, false, UnhandledTypeConversion.New(src.String(), destTi.String()) } } - -func CreateGeomCollTypeFromParams(params map[string]string) (TypeInfo, error) { - var ( - err error - sridVal uint64 - def bool - ) - if s, ok := params["SRID"]; ok { - sridVal, err = strconv.ParseUint(s, 10, 32) - if err != nil { - return nil, err - } - } - if d, ok := params["DefinedSRID"]; ok { - def, err = strconv.ParseBool(d) - if err != nil { - return nil, err - } - } - - return &geomcollType{sqlGeomCollType: gmstypes.GeomCollType{SRID: uint32(sridVal), DefinedSRID: def}}, nil -} diff --git a/go/libraries/doltcore/schema/typeinfo/inlineblob.go b/go/libraries/doltcore/schema/typeinfo/inlineblob.go index b5777431db..ef556b391e 100644 --- a/go/libraries/doltcore/schema/typeinfo/inlineblob.go +++ b/go/libraries/doltcore/schema/typeinfo/inlineblob.go @@ -22,19 +22,11 @@ import ( "unsafe" "github.com/dolthub/go-mysql-server/sql" - gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/vitess/go/sqltypes" "github.com/dolthub/dolt/go/store/types" ) -const ( - inlineBlobTypeParam_Length = "length" - inlineBlobTypeParam_SQL = "sql" - inlineBlobTypeParam_SQL_Binary = "bin" - inlineBlobTypeParam_SQL_VarBinary = "varbin" -) - // inlineBlobType handles BINARY and VARBINARY. BLOB types are handled by varBinaryType. type inlineBlobType struct { sqlBinaryType sql.StringType @@ -42,40 +34,6 @@ type inlineBlobType struct { var _ TypeInfo = (*inlineBlobType)(nil) -var ( - VarbinaryDefaultType = &inlineBlobType{gmstypes.MustCreateBinary(sqltypes.VarBinary, 16383)} -) - -func CreateInlineBlobTypeFromParams(params map[string]string) (TypeInfo, error) { - var length int64 - var err error - if lengthStr, ok := params[inlineBlobTypeParam_Length]; ok { - length, err = strconv.ParseInt(lengthStr, 10, 64) - if err != nil { - return nil, err - } - } else { - return nil, fmt.Errorf(`create inlineblob type info is missing param "%v"`, inlineBlobTypeParam_Length) - } - if sqlStr, ok := params[inlineBlobTypeParam_SQL]; ok { - var sqlType sql.StringType - switch sqlStr { - case inlineBlobTypeParam_SQL_Binary: - sqlType, err = gmstypes.CreateBinary(sqltypes.Binary, length) - case inlineBlobTypeParam_SQL_VarBinary: - sqlType, err = gmstypes.CreateBinary(sqltypes.VarBinary, length) - default: - return nil, fmt.Errorf(`create inlineblob type info has "%v" param with value "%v"`, inlineBlobTypeParam_SQL, sqlStr) - } - if err != nil { - return nil, err - } - return &inlineBlobType{sqlType}, nil - } else { - return nil, fmt.Errorf(`create inlineblob type info is missing param "%v"`, inlineBlobTypeParam_SQL) - } -} - // ConvertNomsValueToValue implements TypeInfo interface. func (ti *inlineBlobType) ConvertNomsValueToValue(v types.Value) (interface{}, error) { if val, ok := v.(types.InlineBlob); ok { diff --git a/go/libraries/doltcore/schema/typeinfo/int.go b/go/libraries/doltcore/schema/typeinfo/int.go index 1fdfcf6922..37d5c7d668 100644 --- a/go/libraries/doltcore/schema/typeinfo/int.go +++ b/go/libraries/doltcore/schema/typeinfo/int.go @@ -26,15 +26,6 @@ import ( "github.com/dolthub/dolt/go/store/types" ) -const ( - intTypeParams_Width = "width" - intTypeParams_Width_8 = "8" - intTypeParams_Width_16 = "16" - intTypeParams_Width_24 = "24" - intTypeParams_Width_32 = "32" - intTypeParams_Width_64 = "64" -) - type intType struct { sqlIntType sql.NumberType } @@ -48,26 +39,6 @@ var ( Int64Type = &intType{gmstypes.Int64} ) -func CreateIntTypeFromParams(params map[string]string) (TypeInfo, error) { - if width, ok := params[intTypeParams_Width]; ok { - switch width { - case intTypeParams_Width_8: - return Int8Type, nil - case intTypeParams_Width_16: - return Int16Type, nil - case intTypeParams_Width_24: - return Int24Type, nil - case intTypeParams_Width_32: - return Int32Type, nil - case intTypeParams_Width_64: - return Int64Type, nil - default: - return nil, fmt.Errorf(`create int type info has "%v" param with value "%v"`, intTypeParams_Width, width) - } - } - return nil, fmt.Errorf(`create int type info is missing "%v" param`, intTypeParams_Width) -} - // ConvertNomsValueToValue implements TypeInfo interface. func (ti *intType) ConvertNomsValueToValue(v types.Value) (interface{}, error) { if val, ok := v.(types.Int); ok { diff --git a/go/libraries/doltcore/schema/typeinfo/linestring.go b/go/libraries/doltcore/schema/typeinfo/linestring.go index 4d046c27f8..3ea15a6b2e 100644 --- a/go/libraries/doltcore/schema/typeinfo/linestring.go +++ b/go/libraries/doltcore/schema/typeinfo/linestring.go @@ -17,7 +17,6 @@ package typeinfo import ( "context" "fmt" - "strconv" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" @@ -197,24 +196,3 @@ func linestringTypeConverter(ctx context.Context, src *linestringType, destTi Ty return nil, false, UnhandledTypeConversion.New(src.String(), destTi.String()) } } - -func CreateLineStringTypeFromParams(params map[string]string) (TypeInfo, error) { - var ( - err error - sridVal uint64 - def bool - ) - if s, ok := params["SRID"]; ok { - sridVal, err = strconv.ParseUint(s, 10, 32) - if err != nil { - return nil, err - } - } - if d, ok := params["DefinedSRID"]; ok { - def, err = strconv.ParseBool(d) - if err != nil { - return nil, err - } - } - return &linestringType{sqlLineStringType: gmstypes.LineStringType{SRID: uint32(sridVal), DefinedSRID: def}}, nil -} diff --git a/go/libraries/doltcore/schema/typeinfo/multilinestring.go b/go/libraries/doltcore/schema/typeinfo/multilinestring.go index 8fb88913b9..9af48ed569 100644 --- a/go/libraries/doltcore/schema/typeinfo/multilinestring.go +++ b/go/libraries/doltcore/schema/typeinfo/multilinestring.go @@ -17,7 +17,6 @@ package typeinfo import ( "context" "fmt" - "strconv" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" @@ -197,25 +196,3 @@ func multilinestringTypeConverter(ctx context.Context, src *multilinestringType, return nil, false, UnhandledTypeConversion.New(src.String(), destTi.String()) } } - -func CreateMultiLineStringTypeFromParams(params map[string]string) (TypeInfo, error) { - var ( - err error - sridVal uint64 - def bool - ) - if s, ok := params["SRID"]; ok { - sridVal, err = strconv.ParseUint(s, 10, 32) - if err != nil { - return nil, err - } - } - if d, ok := params["DefinedSRID"]; ok { - def, err = strconv.ParseBool(d) - if err != nil { - return nil, err - } - } - - return &multilinestringType{sqlMultiLineStringType: gmstypes.MultiLineStringType{SRID: uint32(sridVal), DefinedSRID: def}}, nil -} diff --git a/go/libraries/doltcore/schema/typeinfo/multipoint.go b/go/libraries/doltcore/schema/typeinfo/multipoint.go index 7c27ad7e4a..906c6b5a62 100644 --- a/go/libraries/doltcore/schema/typeinfo/multipoint.go +++ b/go/libraries/doltcore/schema/typeinfo/multipoint.go @@ -17,7 +17,6 @@ package typeinfo import ( "context" "fmt" - "strconv" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" @@ -197,24 +196,3 @@ func multipointTypeConverter(ctx context.Context, src *multipointType, destTi Ty return nil, false, UnhandledTypeConversion.New(src.String(), destTi.String()) } } - -func CreateMultiPointTypeFromParams(params map[string]string) (TypeInfo, error) { - var ( - err error - sridVal uint64 - def bool - ) - if s, ok := params["SRID"]; ok { - sridVal, err = strconv.ParseUint(s, 10, 32) - if err != nil { - return nil, err - } - } - if d, ok := params["DefinedSRID"]; ok { - def, err = strconv.ParseBool(d) - if err != nil { - return nil, err - } - } - return &multipointType{sqlMultiPointType: gmstypes.MultiPointType{SRID: uint32(sridVal), DefinedSRID: def}}, nil -} diff --git a/go/libraries/doltcore/schema/typeinfo/multipolygon.go b/go/libraries/doltcore/schema/typeinfo/multipolygon.go index e2c79a9563..1ef0e48dcd 100644 --- a/go/libraries/doltcore/schema/typeinfo/multipolygon.go +++ b/go/libraries/doltcore/schema/typeinfo/multipolygon.go @@ -17,7 +17,6 @@ package typeinfo import ( "context" "fmt" - "strconv" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" @@ -197,25 +196,3 @@ func multipolygonTypeConverter(ctx context.Context, src *multipolygonType, destT return nil, false, UnhandledTypeConversion.New(src.String(), destTi.String()) } } - -func CreateMultiPolygonTypeFromParams(params map[string]string) (TypeInfo, error) { - var ( - err error - sridVal uint64 - def bool - ) - if s, ok := params["SRID"]; ok { - sridVal, err = strconv.ParseUint(s, 10, 32) - if err != nil { - return nil, err - } - } - if d, ok := params["DefinedSRID"]; ok { - def, err = strconv.ParseBool(d) - if err != nil { - return nil, err - } - } - - return &multipolygonType{sqlMultiPolygonType: gmstypes.MultiPolygonType{SRID: uint32(sridVal), DefinedSRID: def}}, nil -} diff --git a/go/libraries/doltcore/schema/typeinfo/point.go b/go/libraries/doltcore/schema/typeinfo/point.go index 2846f4319d..e30a96e10e 100644 --- a/go/libraries/doltcore/schema/typeinfo/point.go +++ b/go/libraries/doltcore/schema/typeinfo/point.go @@ -17,7 +17,6 @@ package typeinfo import ( "context" "fmt" - "strconv" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" @@ -198,28 +197,6 @@ func pointTypeConverter(ctx context.Context, src *pointType, destTi TypeInfo) (t } } -func CreatePointTypeFromParams(params map[string]string) (TypeInfo, error) { - var ( - err error - sridVal uint64 - def bool - ) - if s, ok := params["SRID"]; ok { - sridVal, err = strconv.ParseUint(s, 10, 32) - if err != nil { - return nil, err - } - } - if d, ok := params["DefinedSRID"]; ok { - def, err = strconv.ParseBool(d) - if err != nil { - return nil, err - } - } - - return CreatePointTypeFromSqlPointType(gmstypes.PointType{SRID: uint32(sridVal), DefinedSRID: def}), nil -} - func CreatePointTypeFromSqlPointType(sqlPointType gmstypes.PointType) TypeInfo { return &pointType{sqlPointType: sqlPointType} } diff --git a/go/libraries/doltcore/schema/typeinfo/polygon.go b/go/libraries/doltcore/schema/typeinfo/polygon.go index 59a47bf9ed..5b73f792e0 100644 --- a/go/libraries/doltcore/schema/typeinfo/polygon.go +++ b/go/libraries/doltcore/schema/typeinfo/polygon.go @@ -17,7 +17,6 @@ package typeinfo import ( "context" "fmt" - "strconv" "github.com/dolthub/go-mysql-server/sql" gmstypes "github.com/dolthub/go-mysql-server/sql/types" @@ -197,25 +196,3 @@ func polygonTypeConverter(ctx context.Context, src *polygonType, destTi TypeInfo return nil, false, UnhandledTypeConversion.New(src.String(), destTi.String()) } } - -func CreatePolygonTypeFromParams(params map[string]string) (TypeInfo, error) { - var ( - err error - sridVal uint64 - def bool - ) - if s, ok := params["SRID"]; ok { - sridVal, err = strconv.ParseUint(s, 10, 32) - if err != nil { - return nil, err - } - } - if d, ok := params["DefinedSRID"]; ok { - def, err = strconv.ParseBool(d) - if err != nil { - return nil, err - } - } - - return &polygonType{sqlPolygonType: gmstypes.PolygonType{SRID: uint32(sridVal), DefinedSRID: def}}, nil -} diff --git a/go/libraries/doltcore/schema/typeinfo/set.go b/go/libraries/doltcore/schema/typeinfo/set.go index d3cccbc097..01b97f10e2 100644 --- a/go/libraries/doltcore/schema/typeinfo/set.go +++ b/go/libraries/doltcore/schema/typeinfo/set.go @@ -16,21 +16,14 @@ package typeinfo import ( "context" - "encoding/gob" "fmt" "strings" "github.com/dolthub/go-mysql-server/sql" - gmstypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/dolt/go/store/types" ) -const ( - setTypeParam_Collation = "collate" - setTypeParam_Values = "vals" -) - // This is a dolt implementation of the MySQL type Set, thus most of the functionality // within is directly reliant on the go-mysql-server implementation. type setType struct { @@ -39,33 +32,6 @@ type setType struct { var _ TypeInfo = (*setType)(nil) -func CreateSetTypeFromParams(params map[string]string) (TypeInfo, error) { - collationStr, ok := params[setTypeParam_Collation] - if !ok { - return nil, fmt.Errorf(`create set type info is missing param "%v"`, setTypeParam_Collation) - } - collation, err := sql.ParseCollation("", collationStr, false) - if err != nil { - return nil, err - } - - valuesStr, ok := params[setTypeParam_Values] - if !ok { - return nil, fmt.Errorf(`create set type info is missing param "%v"`, setTypeParam_Values) - } - var values []string - dec := gob.NewDecoder(strings.NewReader(valuesStr)) - if err = dec.Decode(&values); err != nil { - return nil, err - } - - sqlSetType, err := gmstypes.CreateSetType(values, collation) - if err != nil { - return nil, err - } - return CreateSetTypeFromSqlSetType(sqlSetType), nil -} - func CreateSetTypeFromSqlSetType(sqlSetType sql.SetType) TypeInfo { return &setType{sqlSetType} } diff --git a/go/libraries/doltcore/schema/typeinfo/uint.go b/go/libraries/doltcore/schema/typeinfo/uint.go index 1389640757..bdaa7500f2 100644 --- a/go/libraries/doltcore/schema/typeinfo/uint.go +++ b/go/libraries/doltcore/schema/typeinfo/uint.go @@ -26,15 +26,6 @@ import ( "github.com/dolthub/dolt/go/store/types" ) -const ( - uintTypeParam_Width = "width" - uintTypeParam_Width_8 = "8" - uintTypeParam_Width_16 = "16" - uintTypeParam_Width_24 = "24" - uintTypeParam_Width_32 = "32" - uintTypeParam_Width_64 = "64" -) - type uintType struct { sqlUintType sql.NumberType } @@ -48,26 +39,6 @@ var ( Uint64Type = &uintType{gmstypes.Uint64} ) -func CreateUintTypeFromParams(params map[string]string) (TypeInfo, error) { - if width, ok := params[uintTypeParam_Width]; ok { - switch width { - case uintTypeParam_Width_8: - return Uint8Type, nil - case uintTypeParam_Width_16: - return Uint16Type, nil - case uintTypeParam_Width_24: - return Uint24Type, nil - case uintTypeParam_Width_32: - return Uint32Type, nil - case uintTypeParam_Width_64: - return Uint64Type, nil - default: - return nil, fmt.Errorf(`create uint type info has "%v" param with value "%v"`, uintTypeParam_Width, width) - } - } - return nil, fmt.Errorf(`create uint type info is missing "%v" param`, uintTypeParam_Width) -} - // ConvertNomsValueToValue implements TypeInfo interface. func (ti *uintType) ConvertNomsValueToValue(v types.Value) (interface{}, error) { if val, ok := v.(types.Uint); ok { diff --git a/go/libraries/doltcore/schema/typeinfo/varbinary.go b/go/libraries/doltcore/schema/typeinfo/varbinary.go index c66991070e..4ec1eb1563 100644 --- a/go/libraries/doltcore/schema/typeinfo/varbinary.go +++ b/go/libraries/doltcore/schema/typeinfo/varbinary.go @@ -16,7 +16,6 @@ package typeinfo import ( "context" - "encoding/binary" "fmt" "io" "strconv" @@ -24,16 +23,10 @@ import ( "unsafe" "github.com/dolthub/go-mysql-server/sql" - gmstypes "github.com/dolthub/go-mysql-server/sql/types" - "github.com/dolthub/vitess/go/sqltypes" "github.com/dolthub/dolt/go/store/types" ) -const ( - varBinaryTypeParam_Length = "length" -) - // As a type, this is modeled more after MySQL's story for binary data. There, it's treated // as a string that is interpreted as raw bytes, rather than as a bespoke data structure, // and thus this is mirrored here in its implementation. This will minimize any differences @@ -46,31 +39,6 @@ type varBinaryType struct { var _ TypeInfo = (*varBinaryType)(nil) -var ( - TinyBlobType TypeInfo = &varBinaryType{sqlBinaryType: gmstypes.TinyBlob} - BlobType TypeInfo = &varBinaryType{sqlBinaryType: gmstypes.Blob} - MediumBlobType TypeInfo = &varBinaryType{sqlBinaryType: gmstypes.MediumBlob} - LongBlobType TypeInfo = &varBinaryType{sqlBinaryType: gmstypes.LongBlob} -) - -func CreateVarBinaryTypeFromParams(params map[string]string) (TypeInfo, error) { - var length int64 - var err error - if lengthStr, ok := params[varBinaryTypeParam_Length]; ok { - length, err = strconv.ParseInt(lengthStr, 10, 64) - if err != nil { - return nil, err - } - } else { - return nil, fmt.Errorf(`create varbinary type info is missing param "%v"`, varBinaryTypeParam_Length) - } - sqlType, err := gmstypes.CreateBinary(sqltypes.Blob, length) - if err != nil { - return nil, err - } - return &varBinaryType{sqlType}, nil -} - // ConvertNomsValueToValue implements TypeInfo interface. func (ti *varBinaryType) ConvertNomsValueToValue(v types.Value) (interface{}, error) { if val, ok := v.(types.Blob); ok { @@ -199,29 +167,6 @@ func fromBlob(b types.Blob) ([]byte, error) { return str, nil } -// hasPrefix finds out if a Blob has a prefixed integer. Initially blobs for varBinary prepended an integer indicating -// the length, which was unnecessary (as the underlying sequence tracks the total size). It's been removed, but this -// may be used to see if a Blob is one of those older Blobs. A false positive is possible, but EXTREMELY unlikely. -func hasPrefix(b types.Blob, ctx context.Context) (bool, error) { - blobLength := b.Len() - if blobLength < 8 { - return false, nil - } - countBytes := make([]byte, 8) - n, err := b.ReadAt(ctx, countBytes, 0) - if err != nil { - return false, err - } - if n != 8 { - return false, fmt.Errorf("wanted 8 bytes from blob for count, got %d", n) - } - prefixedLength := binary.LittleEndian.Uint64(countBytes) - if prefixedLength == blobLength-8 { - return true, nil - } - return false, nil -} - // varBinaryTypeConverter is an internal function for GetTypeConverter that handles the specific type as the source TypeInfo. func varBinaryTypeConverter(ctx context.Context, src *varBinaryType, destTi TypeInfo) (tc TypeConverter, needsConversion bool, err error) { switch dest := destTi.(type) { diff --git a/go/libraries/doltcore/schema/typeinfo/varstring.go b/go/libraries/doltcore/schema/typeinfo/varstring.go index 15f618479f..6a19dd2e4a 100644 --- a/go/libraries/doltcore/schema/typeinfo/varstring.go +++ b/go/libraries/doltcore/schema/typeinfo/varstring.go @@ -28,15 +28,6 @@ import ( "github.com/dolthub/dolt/go/store/types" ) -const ( - varStringTypeParam_Collate = "collate" - varStringTypeParam_Length = "length" - varStringTypeParam_SQL = "sql" - varStringTypeParam_SQL_Char = "char" - varStringTypeParam_SQL_VarChar = "varchar" - varStringTypeParam_SQL_Text = "text" -) - // varStringType handles CHAR and VARCHAR. The TEXT types are handled by blobStringType. For any repositories that were // created before the introduction of blobStringType, they will use varStringType for TEXT types. As varStringType makes // use of the String Value type, it does not actually support all viable lengths of a TEXT string, meaning all such @@ -60,48 +51,6 @@ func CreateVarStringTypeFromSqlType(stringType sql.StringType) TypeInfo { return &varStringType{stringType} } -func CreateVarStringTypeFromParams(params map[string]string) (TypeInfo, error) { - var length int64 - var collation sql.CollationID - var err error - if collationStr, ok := params[varStringTypeParam_Collate]; ok { - collation, err = sql.ParseCollation("", collationStr, false) - if err != nil { - return nil, err - } - } else { - return nil, fmt.Errorf(`create varstring type info is missing param "%v"`, varStringTypeParam_Collate) - } - if maxLengthStr, ok := params[varStringTypeParam_Length]; ok { - length, err = strconv.ParseInt(maxLengthStr, 10, 64) - if err != nil { - return nil, err - } - - } else { - return nil, fmt.Errorf(`create varstring type info is missing param "%v"`, varStringTypeParam_Length) - } - if sqlStr, ok := params[varStringTypeParam_SQL]; ok { - var sqlType sql.StringType - switch sqlStr { - case varStringTypeParam_SQL_Char: - sqlType, err = gmstypes.CreateString(sqltypes.Char, length, collation) - case varStringTypeParam_SQL_VarChar: - sqlType, err = gmstypes.CreateString(sqltypes.VarChar, length, collation) - case varStringTypeParam_SQL_Text: - sqlType, err = gmstypes.CreateString(sqltypes.Text, length, collation) - default: - return nil, fmt.Errorf(`create varstring type info has "%v" param with value "%v"`, varStringTypeParam_SQL, sqlStr) - } - if err != nil { - return nil, err - } - return &varStringType{sqlType}, nil - } else { - return nil, fmt.Errorf(`create varstring type info is missing param "%v"`, varStringTypeParam_Length) - } -} - // ConvertNomsValueToValue implements TypeInfo interface. func (ti *varStringType) ConvertNomsValueToValue(v types.Value) (interface{}, error) { if val, ok := v.(types.String); ok { diff --git a/go/libraries/doltcore/sqle/alterschema_test.go b/go/libraries/doltcore/sqle/alterschema_test.go index 8af46caac6..e67d037c20 100644 --- a/go/libraries/doltcore/sqle/alterschema_test.go +++ b/go/libraries/doltcore/sqle/alterschema_test.go @@ -434,12 +434,8 @@ func TestDropPks(t *testing.T) { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() defer dEnv.DoltDB(ctx).Close() - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := NewDatabase(ctx, "dolt", dEnv.DbData(ctx), opts) + + db, err := NewDatabase(ctx, "dolt", dEnv.DbData(ctx), editor.Options{}) require.NoError(t, err) root, _ := dEnv.WorkingRoot(ctx) diff --git a/go/libraries/doltcore/sqle/binlogreplication/binlog_replica_applier.go b/go/libraries/doltcore/sqle/binlogreplication/binlog_replica_applier.go index 100103b3ee..dd96907a79 100644 --- a/go/libraries/doltcore/sqle/binlogreplication/binlog_replica_applier.go +++ b/go/libraries/doltcore/sqle/binlogreplication/binlog_replica_applier.go @@ -841,9 +841,7 @@ func getTableWriter(ctx *sql.Context, engine *gms.Engine, tableName, databaseNam return nil, nil, err } - options := sqlDatabase.EditOptions() - options.ForeignKeyChecksDisabled = foreignKeyChecksDisabled - writeSession := writer.NewWriteSession(binFormat, ws, tracker, options) + writeSession := writer.NewWriteSession(binFormat, ws, tracker, sqlDatabase.EditOptions()) ds := dsess.DSessFromSess(ctx.Session) setter := ds.SetWorkingRoot diff --git a/go/libraries/doltcore/sqle/common_test.go b/go/libraries/doltcore/sqle/common_test.go index 6b4979c109..e2cfe821a3 100644 --- a/go/libraries/doltcore/sqle/common_test.go +++ b/go/libraries/doltcore/sqle/common_test.go @@ -39,12 +39,7 @@ type SetupFn func(t *testing.T, dEnv *env.DoltEnv) // Runs the query given and returns the result. The schema result of the query's execution is currently ignored, and // the targetSchema given is used to prepare all rows. func executeSelect(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, root doltdb.RootValue, query string) ([]sql.Row, sql.Schema, error) { - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := NewDatabase(ctx, "dolt", dEnv.DbData(ctx), opts) + db, err := NewDatabase(ctx, "dolt", dEnv.DbData(ctx), editor.Options{}) require.NoError(t, err) engine, sqlCtx, err := NewTestEngine(dEnv, ctx, db) @@ -72,12 +67,7 @@ func executeSelect(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, root do // Runs the query given and returns the error (if any). func executeModify(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, root doltdb.RootValue, query string) (doltdb.RootValue, error) { - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := NewDatabase(ctx, "dolt", dEnv.DbData(ctx), opts) + db, err := NewDatabase(ctx, "dolt", dEnv.DbData(ctx), editor.Options{}) require.NoError(t, err) engine, sqlCtx, err := NewTestEngine(dEnv, ctx, db) diff --git a/go/libraries/doltcore/sqle/database.go b/go/libraries/doltcore/sqle/database.go index d541bf197e..b21284ec34 100644 --- a/go/libraries/doltcore/sqle/database.go +++ b/go/libraries/doltcore/sqle/database.go @@ -1955,7 +1955,10 @@ func (db Database) CreateTable(ctx *sql.Context, tableName string, sch sql.Prima return err } - if doltdb.IsSystemTable(doltdb.TableName{Name: tableName, Schema: db.schemaName}) && !doltdb.IsFullTextTable(tableName) && !doltdb.HasDoltCIPrefix(tableName) { + if doltdb.IsSystemTable(doltdb.TableName{Name: tableName, Schema: db.schemaName}) && + !doltdb.IsFullTextTable(tableName) && + !doltdb.HasDoltCIPrefix(tableName) && + tableName != doltdb.TestsTableName { // NM4 - determine why this is required now. return ErrReservedTableName.New(tableName) } diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 69a4401be7..a53dbfc72a 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -990,23 +990,7 @@ func (p *DoltDatabaseProvider) registerNewDatabase(ctx *sql.Context, name string // Ensure any provider-supplied DB load params are applied before any lazy DB load occurs. p.applyDBLoadParamsToEnv(newEnv) - fkChecks, err := ctx.GetSessionVariable(ctx, "foreign_key_checks") - if err != nil { - return err - } - - deaf, err := newEnv.DbEaFactory(ctx) - if err != nil { - return err - } - - opts := editor.Options{ - Deaf: deaf, - // TODO: this doesn't seem right, why is this getting set in the constructor to the DB - ForeignKeyChecksDisabled: fkChecks.(int8) == 0, - } - - db, err := NewDatabase(ctx, name, newEnv.DbData(ctx), opts) + db, err := NewDatabase(ctx, name, newEnv.DbData(ctx), editor.Options{}) if err != nil { return err } diff --git a/go/libraries/doltcore/sqle/database_provider_test.go b/go/libraries/doltcore/sqle/database_provider_test.go index f0c5d025f1..160a86a960 100644 --- a/go/libraries/doltcore/sqle/database_provider_test.go +++ b/go/libraries/doltcore/sqle/database_provider_test.go @@ -44,12 +44,8 @@ func TestDatabaseProvider(t *testing.T) { setup := func(t *testing.T) (*sqle.Engine, *sql.Context, *DoltDatabaseProvider) { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), opts) + + db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), editor.Options{}) require.NoError(t, err) engine, sqlCtx, err := NewTestEngine(dEnv, context.Background(), db) diff --git a/go/libraries/doltcore/sqle/database_test.go b/go/libraries/doltcore/sqle/database_test.go index fe190f5677..6d28b6aba8 100644 --- a/go/libraries/doltcore/sqle/database_test.go +++ b/go/libraries/doltcore/sqle/database_test.go @@ -45,15 +45,9 @@ func TestIsKeyFuncs(t *testing.T) { func TestNeedsToReloadEvents(t *testing.T) { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - timestamp := time.Now().Truncate(time.Minute).UTC() - db, err := NewDatabase(ctx, "dolt", dEnv.DbData(ctx), opts) + db, err := NewDatabase(ctx, "dolt", dEnv.DbData(ctx), editor.Options{}) require.NoError(t, err) _, sqlCtx, err := NewTestEngine(dEnv, ctx, db) diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_cherry_pick.go b/go/libraries/doltcore/sqle/dprocedures/dolt_cherry_pick.go index f1c215cfe3..58cbe5c364 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_cherry_pick.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_cherry_pick.go @@ -103,6 +103,8 @@ func doDoltCherryPick(ctx *sql.Context, args []string) (string, int, int, int, e cherryPickOptions.EmptyCommitHandling = doltdb.KeepEmptyCommit } + cherryPickOptions.SkipVerification = apr.Contains(cli.SkipVerificationFlag) + commit, mergeResult, err := cherry_pick.CherryPick(ctx, cherryStr, cherryPickOptions) if err != nil { return "", 0, 0, 0, err diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clean.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clean.go index 27c007c37d..4e723f83ea 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clean.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clean.go @@ -57,7 +57,8 @@ func doDoltClean(ctx *sql.Context, args []string) (int, error) { return 1, fmt.Errorf("Could not load database %s", dbName) } - roots, err = actions.CleanUntracked(ctx, roots, apr.Args, apr.ContainsAll(cli.DryRunFlag), false) + respectIgnoreRules := !apr.Contains(cli.ExcludeIgnoreRulesFlag) + roots, err = actions.CleanUntracked(ctx, roots, apr.Args, apr.ContainsAll(cli.DryRunFlag), false, respectIgnoreRules) if err != nil { return 1, fmt.Errorf("failed to clean; %w", err) } diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go index 43f90c12a7..2782527657 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_clone.go @@ -113,6 +113,13 @@ func getDirectoryAndUrlString(apr *argparser.ArgParseResults) (string, string, e } else if dir == "/" { return "", "", errhand.BuildDError("Could not infer repo name. Please explicitly define a directory for this url").Build() } + // Match `dolt clone` behavior: strip a trailing `.git` from inferred names. + if strings.HasSuffix(dir, ".git") { + dir = strings.TrimSuffix(dir, ".git") + if dir == "" { + return "", "", errhand.BuildDError("Could not infer repo name. Please explicitly define a directory for this url").Build() + } + } } return dir, urlStr, nil diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_commit.go b/go/libraries/doltcore/sqle/dprocedures/dolt_commit.go index b81f8a64b1..7c89edca6a 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_commit.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_commit.go @@ -163,14 +163,15 @@ func doDoltCommit(ctx *sql.Context, args []string) (string, bool, error) { } csp := actions.CommitStagedProps{ - Message: msg, - Date: t, - AllowEmpty: apr.Contains(cli.AllowEmptyFlag), - SkipEmpty: apr.Contains(cli.SkipEmptyFlag), - Amend: amend, - Force: apr.Contains(cli.ForceFlag), - Name: name, - Email: email, + Message: msg, + Date: t, + AllowEmpty: apr.Contains(cli.AllowEmptyFlag), + SkipEmpty: apr.Contains(cli.SkipEmptyFlag), + Amend: amend, + Force: apr.Contains(cli.ForceFlag), + Name: name, + Email: email, + SkipVerification: apr.Contains(cli.SkipVerificationFlag), } shouldSign, err := dsess.GetBooleanSystemVar(ctx, "gpgsign") diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_merge.go b/go/libraries/doltcore/sqle/dprocedures/dolt_merge.go index 6a0471163c..1f68de8e79 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_merge.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_merge.go @@ -180,7 +180,7 @@ func doDoltMerge(ctx *sql.Context, args []string) (string, int, int, string, err msg = userMsg } - ws, commit, conflicts, fastForward, message, err := performMerge(ctx, sess, ws, dbName, mergeSpec, apr.Contains(cli.NoCommitFlag), msg) + ws, commit, conflicts, fastForward, message, err := performMerge(ctx, sess, ws, dbName, mergeSpec, apr.Contains(cli.NoCommitFlag), msg, apr.Contains(cli.SkipVerificationFlag)) if err != nil { return commit, conflicts, fastForward, "", err } @@ -205,6 +205,7 @@ func performMerge( spec *merge.MergeSpec, noCommit bool, msg string, + skipVerification bool, ) (*doltdb.WorkingSet, string, int, int, string, error) { // todo: allow merges even when an existing merge is uncommitted if ws.MergeActive() { @@ -234,7 +235,7 @@ func performMerge( if canFF { if spec.FFMode == merge.NoFastForward { var commit *doltdb.Commit - ws, commit, err = executeNoFFMerge(ctx, sess, spec, msg, dbName, ws, noCommit) + ws, commit, err = executeNoFFMerge(ctx, sess, spec, msg, dbName, ws, noCommit, skipVerification) if err == doltdb.ErrUnresolvedConflictsOrViolations { // if there are unresolved conflicts, write the resulting working set back to the session and return an // error message @@ -306,7 +307,10 @@ func performMerge( author := fmt.Sprintf("%s <%s>", spec.Name, spec.Email) args := []string{"-m", msg, "--author", author} if spec.Force { - args = append(args, "--force") + args = append(args, "--"+cli.ForceFlag) + } + if skipVerification { + args = append(args, "--"+cli.SkipVerificationFlag) } commit, _, err = doDoltCommit(ctx, args) if err != nil { @@ -405,6 +409,7 @@ func executeNoFFMerge( dbName string, ws *doltdb.WorkingSet, noCommit bool, + skipVerification bool, ) (*doltdb.WorkingSet, *doltdb.Commit, error) { mergeRoot, err := spec.MergeC.GetRootValue(ctx) if err != nil { @@ -444,11 +449,12 @@ func executeNoFFMerge( } pendingCommit, err := dSess.NewPendingCommit(ctx, dbName, roots, actions.CommitStagedProps{ - Message: msg, - Date: spec.Date, - Force: spec.Force, - Name: spec.Name, - Email: spec.Email, + Message: msg, + Date: spec.Date, + Force: spec.Force, + Name: spec.Name, + Email: spec.Email, + SkipVerification: skipVerification, }) if err != nil { return nil, nil, err diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_pull.go b/go/libraries/doltcore/sqle/dprocedures/dolt_pull.go index 46f3a940b1..1cba0b33aa 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_pull.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_pull.go @@ -237,7 +237,7 @@ func doDoltPull(ctx *sql.Context, args []string) (int, int, string, error) { return noConflictsOrViolations, threeWayMerge, "", ErrUncommittedChanges.New() } - ws, _, conflicts, fastForward, message, err = performMerge(ctx, sess, ws, dbName, mergeSpec, apr.Contains(cli.NoCommitFlag), msg) + ws, _, conflicts, fastForward, message, err = performMerge(ctx, sess, ws, dbName, mergeSpec, apr.Contains(cli.NoCommitFlag), msg, apr.Contains(cli.SkipVerificationFlag)) if err != nil && !errors.Is(doltdb.ErrUpToDate, err) { return conflicts, fastForward, "", err } diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_rebase.go b/go/libraries/doltcore/sqle/dprocedures/dolt_rebase.go index 7f4374df45..a770f6485f 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_rebase.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_rebase.go @@ -216,7 +216,9 @@ func doDoltRebase(ctx *sql.Context, args []string) (int, string, error) { } else if apr.NArg() > 1 { return 1, "", fmt.Errorf("too many args") } - err = startRebase(ctx, apr.Arg(0), commitBecomesEmptyHandling, emptyCommitHandling) + + skipVerification := apr.Contains(cli.SkipVerificationFlag) + err = startRebase(ctx, apr.Arg(0), commitBecomesEmptyHandling, emptyCommitHandling, skipVerification) if err != nil { return 1, "", err } @@ -263,7 +265,7 @@ func processCommitBecomesEmptyParams(apr *argparser.ArgParseResults) (doltdb.Emp // startRebase starts a new interactive rebase operation. |upstreamPoint| specifies the commit where the new rebased // commits will be based off of, |commitBecomesEmptyHandling| specifies how to handle commits that are not empty, but // do not produce any changes when applied, and |emptyCommitHandling| specifies how to handle empty commits. -func startRebase(ctx *sql.Context, upstreamPoint string, commitBecomesEmptyHandling doltdb.EmptyCommitHandling, emptyCommitHandling doltdb.EmptyCommitHandling) error { +func startRebase(ctx *sql.Context, upstreamPoint string, commitBecomesEmptyHandling doltdb.EmptyCommitHandling, emptyCommitHandling doltdb.EmptyCommitHandling, skipVerification bool) error { if upstreamPoint == "" { return fmt.Errorf("no upstream branch specified") } @@ -351,7 +353,7 @@ func startRebase(ctx *sql.Context, upstreamPoint string, commitBecomesEmptyHandl } newWorkingSet, err := workingSet.StartRebase(ctx, upstreamCommit, rebaseBranch, branchRoots.Working, - commitBecomesEmptyHandling, emptyCommitHandling) + commitBecomesEmptyHandling, emptyCommitHandling, skipVerification) if err != nil { return err } @@ -716,7 +718,8 @@ func continueRebase(ctx *sql.Context) rebaseResult { result := processRebasePlanStep(ctx, &step, workingSet.RebaseState().CommitBecomesEmptyHandling(), - workingSet.RebaseState().EmptyCommitHandling()) + workingSet.RebaseState().EmptyCommitHandling(), + workingSet.RebaseState().SkipVerification()) if result.err != nil || result.status != 0 || result.halt { return result } @@ -803,7 +806,7 @@ func commitManuallyStagedChangesForStep(ctx *sql.Context, step rebase.RebasePlan } options, err := createCherryPickOptionsForRebaseStep(ctx, &step, workingSet.RebaseState().CommitBecomesEmptyHandling(), - workingSet.RebaseState().EmptyCommitHandling()) + workingSet.RebaseState().EmptyCommitHandling(), workingSet.RebaseState().SkipVerification()) doltDB, ok := doltSession.GetDoltDB(ctx, ctx.GetCurrentDatabase()) if !ok { @@ -861,6 +864,7 @@ func processRebasePlanStep( planStep *rebase.RebasePlanStep, commitBecomesEmptyHandling doltdb.EmptyCommitHandling, emptyCommitHandling doltdb.EmptyCommitHandling, + skipVerification bool, ) rebaseResult { // Make sure we have a transaction opened for the session // NOTE: After our first call to cherry-pick, the tx is committed, so a new tx needs to be started @@ -878,7 +882,7 @@ func processRebasePlanStep( return newRebaseSuccess("") } - options, err := createCherryPickOptionsForRebaseStep(ctx, planStep, commitBecomesEmptyHandling, emptyCommitHandling) + options, err := createCherryPickOptionsForRebaseStep(ctx, planStep, commitBecomesEmptyHandling, emptyCommitHandling, skipVerification) if err != nil { return newRebaseError(err) } @@ -886,12 +890,19 @@ func processRebasePlanStep( return handleRebaseCherryPick(ctx, planStep, *options) } -func createCherryPickOptionsForRebaseStep(ctx *sql.Context, planStep *rebase.RebasePlanStep, commitBecomesEmptyHandling doltdb.EmptyCommitHandling, emptyCommitHandling doltdb.EmptyCommitHandling) (*cherry_pick.CherryPickOptions, error) { +func createCherryPickOptionsForRebaseStep( + ctx *sql.Context, + planStep *rebase.RebasePlanStep, + commitBecomesEmptyHandling doltdb.EmptyCommitHandling, + emptyCommitHandling doltdb.EmptyCommitHandling, + skipVerification bool, +) (*cherry_pick.CherryPickOptions, error) { // Override the default empty commit handling options for cherry-pick, since // rebase has slightly different defaults options := cherry_pick.NewCherryPickOptions() options.CommitBecomesEmptyHandling = commitBecomesEmptyHandling options.EmptyCommitHandling = emptyCommitHandling + options.SkipVerification = skipVerification switch planStep.Action { case rebase.RebaseActionDrop, rebase.RebaseActionPick, rebase.RebaseActionEdit: diff --git a/go/libraries/doltcore/sqle/dprocedures/dolt_stash.go b/go/libraries/doltcore/sqle/dprocedures/dolt_stash.go index 74c5c76d35..01ca90d44f 100644 --- a/go/libraries/doltcore/sqle/dprocedures/dolt_stash.go +++ b/go/libraries/doltcore/sqle/dprocedures/dolt_stash.go @@ -392,14 +392,6 @@ func parseStashIndex(apr *argparser.ArgParseResults) (int, error) { return idx, nil } -func bulkDbEaFactory(dbData env.DbData[*sql.Context]) editor.DbEaFactory { - tmpDir, err := dbData.Rsw.TempTableFilesDir() - if err != nil { - return nil - } - return editor.NewBulkImportTEAFactory(dbData.Ddb.ValueReadWriter(), tmpDir) -} - func updateWorkingRoot(ctx *sql.Context, dbData env.DbData[*sql.Context], newRoot doltdb.RootValue) error { var h hash.Hash var wsRef ref.WorkingSetRef @@ -510,16 +502,12 @@ func handleMerge(ctx *sql.Context, dbName string, dbData env.DbData[*sql.Context return nil, nil, nil, err } - tmpDir, err := dbData.Rsw.TempTableFilesDir() - if err != nil { - return nil, nil, nil, err - } - tableResolver, err := dsess.GetTableResolver(ctx, dbName) if err != nil { return nil, nil, nil, err } - opts := editor.Options{Deaf: bulkDbEaFactory(dbData), Tempdir: tmpDir} + + opts := editor.Options{} result, err := merge.MergeRoots(ctx, tableResolver, curWorkingRoot, stashRoot, parentRoot, stashRoot, parentCommit, opts, merge.MergeOpts{IsCherryPick: false}) if err != nil { return nil, nil, nil, err diff --git a/go/libraries/doltcore/sqle/dsess/session.go b/go/libraries/doltcore/sqle/dsess/session.go index a4c781f1df..d25a736dcb 100644 --- a/go/libraries/doltcore/sqle/dsess/session.go +++ b/go/libraries/doltcore/sqle/dsess/session.go @@ -24,7 +24,6 @@ import ( "time" "github.com/dolthub/go-mysql-server/sql" - sqltypes "github.com/dolthub/go-mysql-server/sql/types" "github.com/shopspring/decimal" "github.com/dolthub/dolt/go/cmd/dolt/cli" @@ -454,32 +453,6 @@ func (d *DoltSession) clear() { } } -func (d *DoltSession) newWorkingSetForHead(ctx *sql.Context, wsRef ref.WorkingSetRef, dbName string) (*doltdb.WorkingSet, error) { - dbData, _ := d.GetDbData(nil, dbName) - - headSpec, _ := doltdb.NewCommitSpec("HEAD") - headRef, err := wsRef.ToHeadRef() - if err != nil { - return nil, err - } - - optCmt, err := dbData.Ddb.Resolve(ctx, headSpec, headRef) - if err != nil { - return nil, err - } - headCommit, ok := optCmt.ToCommit() - if !ok { - return nil, doltdb.ErrGhostCommitEncountered - } - - headRoot, err := headCommit.GetRootValue(ctx) - if err != nil { - return nil, err - } - - return doltdb.EmptyWorkingSet(wsRef).WithWorkingRoot(headRoot).WithStagedRoot(headRoot), nil -} - // CommitTransaction commits the in-progress transaction. Depending on session settings, this may write only a new // working set, or may additionally create a new dolt commit for the current HEAD. If more than one branch head has // changes, the transaction is rejected. @@ -1328,40 +1301,6 @@ func (d *DoltSession) setHeadRefSessionVar(ctx *sql.Context, db, value string) e func (d *DoltSession) setForeignKeyChecksSessionVar(ctx *sql.Context, key string, value interface{}) error { d.mu.Lock() defer d.mu.Unlock() - - convertedVal, _, err := sqltypes.Int64.Convert(ctx, value) - if err != nil { - return err - } - intVal := int64(0) - if convertedVal != nil { - intVal = convertedVal.(int64) - } - - if intVal == 0 { - for _, dbState := range d.dbStates { - for _, branchState := range dbState.heads { - if ws := branchState.WriteSession(); ws != nil { - opts := ws.GetOptions() - opts.ForeignKeyChecksDisabled = true - ws.SetOptions(opts) - } - } - } - } else if intVal == 1 { - for _, dbState := range d.dbStates { - for _, branchState := range dbState.heads { - if ws := branchState.WriteSession(); ws != nil { - opts := ws.GetOptions() - opts.ForeignKeyChecksDisabled = false - ws.SetOptions(opts) - } - } - } - } else { - return sql.ErrInvalidSystemVariableValue.New("foreign_key_checks", intVal) - } - return d.Session.SetSessionVariable(ctx, key, value) } diff --git a/go/libraries/doltcore/sqle/dtablefunctions/dolt_diff.go b/go/libraries/doltcore/sqle/dtablefunctions/dolt_diff.go index 2537d9fe7b..8c01eb2b7b 100644 --- a/go/libraries/doltcore/sqle/dtablefunctions/dolt_diff.go +++ b/go/libraries/doltcore/sqle/dtablefunctions/dolt_diff.go @@ -363,15 +363,6 @@ func interfaceToString(r interface{}) (string, error) { return str, nil } -func resolveRoot(ctx *sql.Context, sess *dsess.DoltSession, dbName, hashStr string) (*refDetails, error) { - root, commitTime, _, err := sess.ResolveRootForRef(ctx, dbName, hashStr) - if err != nil { - return nil, err - } - - return &refDetails{root: root, hashStr: hashStr, commitTime: commitTime}, nil -} - func resolveCommit(ctx *sql.Context, ddb *doltdb.DoltDB, headRef ref.DoltRef, cSpecStr string) (*doltdb.Commit, error) { cs, err := doltdb.NewCommitSpec(cSpecStr) if err != nil { diff --git a/go/libraries/doltcore/sqle/dtablefunctions/dolt_test_run.go b/go/libraries/doltcore/sqle/dtablefunctions/dolt_test_run.go index 59a1ff26ab..e76874f7c9 100644 --- a/go/libraries/doltcore/sqle/dtablefunctions/dolt_test_run.go +++ b/go/libraries/doltcore/sqle/dtablefunctions/dolt_test_run.go @@ -17,7 +17,9 @@ package dtablefunctions import ( "fmt" "io" + "strconv" "strings" + "time" gms "github.com/dolthub/go-mysql-server" "github.com/dolthub/go-mysql-server/sql" @@ -26,10 +28,13 @@ import ( "github.com/dolthub/vitess/go/vt/sqlparser" "github.com/gocraft/dbr/v2" "github.com/gocraft/dbr/v2/dialect" + "github.com/shopspring/decimal" + "golang.org/x/exp/constraints" - "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/overrides" + "github.com/dolthub/dolt/go/store/val" ) const testsRunDefaultRowCount = 10 @@ -39,12 +44,13 @@ var _ sql.CatalogTableFunction = (*TestsRunTableFunction)(nil) var _ sql.ExecSourceRel = (*TestsRunTableFunction)(nil) var _ sql.AuthorizationCheckerNode = (*TestsRunTableFunction)(nil) -type testResult struct { - testName string - groupName string - query string - status string - message string +// TestResult represents the result of running a single test +type TestResult struct { + TestName string + GroupName string + Query string + Status string + Message string } type TestsRunTableFunction struct { @@ -199,7 +205,7 @@ func (trtf *TestsRunTableFunction) RowIter(_ *sql.Context, _ sql.Row) (sql.RowIt return nil, err } - resultRow := sql.NewRow(result.testName, result.groupName, result.query, result.status, result.message) + resultRow := sql.NewRow(result.TestName, result.GroupName, result.Query, result.Status, result.Message) resultRows = append(resultRows, resultRow) } } @@ -220,7 +226,7 @@ func (trtf *TestsRunTableFunction) RowCount(_ *sql.Context) (uint64, bool, error return testsRunDefaultRowCount, false, nil } -func (trtf *TestsRunTableFunction) queryAndAssert(row sql.Row) (result testResult, err error) { +func (trtf *TestsRunTableFunction) queryAndAssert(row sql.Row) (result TestResult, err error) { testName, groupName, query, assertion, comparison, value, err := parseDoltTestsRow(trtf.ctx, row) if err != nil { return @@ -237,9 +243,9 @@ func (trtf *TestsRunTableFunction) queryAndAssert(row sql.Row) (result testResul if err != nil { message = fmt.Sprintf("Query error: %s", err.Error()) } else { - testPassed, message, err = actions.AssertData(trtf.ctx, *assertion, *comparison, value, queryResult) + testPassed, message, err = AssertData(trtf.ctx, *assertion, *comparison, value, queryResult) if err != nil { - return testResult{}, err + return TestResult{}, err } } } @@ -253,11 +259,49 @@ func (trtf *TestsRunTableFunction) queryAndAssert(row sql.Row) (result testResul if groupName != nil { groupString = *groupName } - result = testResult{*testName, groupString, *query, status, message} + result = TestResult{*testName, groupString, *query, status, message} + return result, nil +} + +func (trtf *TestsRunTableFunction) queryAndAssertWithFunc(row sql.Row, assertDataFunc AssertDataFunc) (result TestResult, err error) { + testName, groupName, query, assertion, comparison, value, err := parseDoltTestsRow(trtf.ctx, row) + if err != nil { + return + } + + message, err := validateQuery(trtf.ctx, trtf.catalog, *query) + if err != nil && message == "" { + message = fmt.Sprintf("query error: %s", err.Error()) + } + + var testPassed bool + if message == "" { + _, queryResult, _, err := trtf.engine.Query(trtf.ctx, *query) + if err != nil { + message = fmt.Sprintf("Query error: %s", err.Error()) + } else { + testPassed, message, err = assertDataFunc(trtf.ctx, *assertion, *comparison, value, queryResult) + if err != nil { + return TestResult{}, err + } + } + } + + status := "PASS" + if !testPassed { + status = "FAIL" + } + + var groupString string + if groupName != nil { + groupString = *groupName + } + result = TestResult{*testName, groupString, *query, status, message} return result, nil } func (trtf *TestsRunTableFunction) getDoltTestsData(arg string) ([]sql.Row, error) { + // Original behavior when root is nil - use SQL queries against current session var queries []string if arg == "*" { @@ -320,28 +364,31 @@ func IsWriteQuery(query string, ctx *sql.Context, catalog sql.Catalog) (bool, er } func parseDoltTestsRow(ctx *sql.Context, row sql.Row) (testName, groupName, query, assertion, comparison, value *string, err error) { - if testName, err = actions.GetStringColAsString(ctx, row[0]); err != nil { + if testName, err = getStringColAsString(ctx, row[0]); err != nil { return } - if groupName, err = actions.GetStringColAsString(ctx, row[1]); err != nil { + if groupName, err = getStringColAsString(ctx, row[1]); err != nil { return } - if query, err = actions.GetStringColAsString(ctx, row[2]); err != nil { + if query, err = getStringColAsString(ctx, row[2]); err != nil { return } - if assertion, err = actions.GetStringColAsString(ctx, row[3]); err != nil { + if assertion, err = getStringColAsString(ctx, row[3]); err != nil { return } - if comparison, err = actions.GetStringColAsString(ctx, row[4]); err != nil { + if comparison, err = getStringColAsString(ctx, row[4]); err != nil { return } - if value, err = actions.GetStringColAsString(ctx, row[5]); err != nil { + if value, err = getStringColAsString(ctx, row[5]); err != nil { return } return testName, groupName, query, assertion, comparison, value, nil } +// AssertDataFunc defines the function signature for asserting test data +type AssertDataFunc func(sqlCtx *sql.Context, assertion string, comparison string, value *string, queryResult sql.RowIter) (testPassed bool, message string, err error) + func validateQuery(ctx *sql.Context, catalog sql.Catalog, query string) (string, error) { // We first check if the query contains multiple sql statements if statements, err := sqlparser.SplitStatementToPieces(query); err != nil { @@ -361,3 +408,455 @@ func validateQuery(ctx *sql.Context, catalog sql.Catalog, query string) (string, } return "", nil } + +// Simple inline assertion constants to avoid circular imports +const ( + AssertionExpectedRows = "expected_rows" + AssertionExpectedColumns = "expected_columns" + AssertionExpectedSingleValue = "expected_single_value" +) + +// getStringColAsString safely converts a sql value to string +func getStringColAsString(sqlCtx *sql.Context, tableValue interface{}) (*string, error) { + if tableValue == nil { + return nil, nil + } + if ts, ok := tableValue.(*val.TextStorage); ok { + str, err := ts.Unwrap(sqlCtx) + if err != nil { + return nil, err + } + return &str, nil + } else if str, ok := tableValue.(string); ok { + return &str, nil + } else { + return nil, fmt.Errorf("unexpected type %T, was expecting string", tableValue) + } +} + +// readTableDataFromDoltTable reads test data directly from a dolt table +func (trtf *TestsRunTableFunction) readTableDataFromDoltTable(table *doltdb.Table, arg string) ([]sql.Row, error) { + // This is a complex implementation that requires reading table data directly from dolt storage + // For now, return an error that clearly indicates this needs to be implemented + // The table scan would involve: + // 1. Getting the table schema + // 2. Creating a table iterator + // 3. Reading and filtering rows based on the arg (test_name or test_group) + // 4. Converting dolt storage format to SQL rows + // + // This is a significant implementation that requires understanding dolt's storage internals + return nil, fmt.Errorf("direct table reading from dolt storage not yet implemented for table scan of dolt_tests - this requires implementing table iteration and row conversion from dolt's internal storage format") +} + +// AssertData parses an assertion, comparison, and value, then returns the status of the test. +// Valid comparison are: "==", "!=", "<", ">", "<=", and ">=". +// testPassed indicates whether the test was successful or not. +// message is a string used to indicate test failures, and will not halt the overall process. +// message will be empty if the test passed. +// err indicates runtime failures and will stop dolt_test_run from proceeding. +func AssertData(sqlCtx *sql.Context, assertion string, comparison string, value *string, queryResult sql.RowIter) (testPassed bool, message string, err error) { + switch assertion { + case AssertionExpectedRows: + message, err = expectRows(sqlCtx, comparison, value, queryResult) + case AssertionExpectedColumns: + message, err = expectColumns(sqlCtx, comparison, value, queryResult) + case AssertionExpectedSingleValue: + message, err = expectSingleValue(sqlCtx, comparison, value, queryResult) + default: + return false, fmt.Sprintf("%s is not a valid assertion type", assertion), nil + } + + if err != nil { + return false, "", err + } else if message != "" { + return false, message, nil + } + return true, "", nil +} + +func expectSingleValue(sqlCtx *sql.Context, comparison string, value *string, queryResult sql.RowIter) (message string, err error) { + row, err := queryResult.Next(sqlCtx) + if err == io.EOF { + return fmt.Sprintf("expected_single_value expects exactly one cell. Received 0 rows"), nil + } else if err != nil { + return "", err + } + + if len(row) != 1 { + return fmt.Sprintf("expected_single_value expects exactly one cell. Received multiple columns"), nil + } + _, err = queryResult.Next(sqlCtx) + if err == nil { //If multiple rows were given, we should error out + return fmt.Sprintf("expected_single_value expects exactly one cell. Received multiple rows"), nil + } else if err != io.EOF { // "True" error, so we should quit out + return "", err + } + + if value == nil { // If we're expecting a null value, we don't need to type switch + return compareNullValue(comparison, row[0], AssertionExpectedSingleValue), nil + } + + // Check if the expected value is a boolean string, and if so, coerce the actual value to boolean, with the exception + // of "0" and "1", which are valid integers and are covered below. + if *value != "0" && *value != "1" { + if expectedBool, err := strconv.ParseBool(*value); err == nil { + actualBool, boolErr := getInterfaceAsBool(row[0]) + if boolErr != nil { + return fmt.Sprintf("Could not convert value to boolean: %v", boolErr), nil + } + return compareBooleans(comparison, expectedBool, actualBool, AssertionExpectedSingleValue), nil + } + } + + switch actualValue := row[0].(type) { + case int8: + expectedInt, err := strconv.ParseInt(*value, 10, 64) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, int8(expectedInt), actualValue, AssertionExpectedSingleValue), nil + case int16: + expectedInt, err := strconv.ParseInt(*value, 10, 64) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, int16(expectedInt), actualValue, AssertionExpectedSingleValue), nil + case int32: + expectedInt, err := strconv.ParseInt(*value, 10, 64) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, int32(expectedInt), actualValue, AssertionExpectedSingleValue), nil + case int64: + expectedInt, err := strconv.ParseInt(*value, 10, 64) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, expectedInt, actualValue, AssertionExpectedSingleValue), nil + case int: + expectedInt, err := strconv.ParseInt(*value, 10, 64) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, int(expectedInt), actualValue, AssertionExpectedSingleValue), nil + case uint8: + expectedUint, err := strconv.ParseUint(*value, 10, 32) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, uint8(expectedUint), actualValue, AssertionExpectedSingleValue), nil + case uint16: + expectedUint, err := strconv.ParseUint(*value, 10, 32) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, uint16(expectedUint), actualValue, AssertionExpectedSingleValue), nil + case uint32: + expectedUint, err := strconv.ParseUint(*value, 10, 32) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, uint32(expectedUint), actualValue, AssertionExpectedSingleValue), nil + case uint64: + expectedUint, err := strconv.ParseUint(*value, 10, 64) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, expectedUint, actualValue, AssertionExpectedSingleValue), nil + case uint: + expectedUint, err := strconv.ParseUint(*value, 10, 64) + if err != nil { + return fmt.Sprintf("Could not compare non integer value '%s', with %d", *value, actualValue), nil + } + return compareTestAssertion(comparison, uint(expectedUint), actualValue, AssertionExpectedSingleValue), nil + case float64: + expectedFloat, err := strconv.ParseFloat(*value, 64) + if err != nil { + return fmt.Sprintf("Could not compare non float value '%s', with %f", *value, actualValue), nil + } + return compareTestAssertion(comparison, expectedFloat, actualValue, AssertionExpectedSingleValue), nil + case float32: + expectedFloat, err := strconv.ParseFloat(*value, 32) + if err != nil { + return fmt.Sprintf("Could not compare non float value '%s', with %f", *value, actualValue), nil + } + return compareTestAssertion(comparison, float32(expectedFloat), actualValue, AssertionExpectedSingleValue), nil + case decimal.Decimal: + expectedDecimal, err := decimal.NewFromString(*value) + if err != nil { + return fmt.Sprintf("Could not compare non decimal value '%s', with %s", *value, actualValue), nil + } + return compareDecimals(comparison, expectedDecimal, actualValue, AssertionExpectedSingleValue), nil + case time.Time: + expectedTime, format, err := parseTestsDate(*value) + if err != nil { + return fmt.Sprintf("%s does not appear to be a valid date", *value), nil + } + return compareDates(comparison, expectedTime, actualValue, format, AssertionExpectedSingleValue), nil + case *val.TextStorage, string: + actualString, err := GetStringColAsString(sqlCtx, actualValue) + if err != nil { + return "", err + } + return compareTestAssertion(comparison, *value, *actualString, AssertionExpectedSingleValue), nil + default: + return fmt.Sprintf("Type %T is not supported. Open an issue at https://github.com/dolthub/dolt/issues to see it added", actualValue), nil + } +} + +func expectRows(sqlCtx *sql.Context, comparison string, value *string, queryResult sql.RowIter) (message string, err error) { + if value == nil { + return "null is not a valid assertion for expected_rows", nil + } + expectedRows, err := strconv.Atoi(*value) + if err != nil { + return fmt.Sprintf("cannot run assertion on non integer value: %s", *value), nil + } + + var numRows int + for { + _, err := queryResult.Next(sqlCtx) + if err == io.EOF { + break + } else if err != nil { + return "", err + } + numRows++ + } + return compareTestAssertion(comparison, expectedRows, numRows, AssertionExpectedRows), nil +} + +func expectColumns(sqlCtx *sql.Context, comparison string, value *string, queryResult sql.RowIter) (message string, err error) { + if value == nil { + return "null is not a valid assertion for expected_rows", nil + } + expectedColumns, err := strconv.Atoi(*value) + if err != nil { + return fmt.Sprintf("cannot run assertion on non integer value: %s", *value), nil + } + + var numColumns int + row, err := queryResult.Next(sqlCtx) + if err != nil && err != io.EOF { + return "", err + } + numColumns = len(row) + return compareTestAssertion(comparison, expectedColumns, numColumns, AssertionExpectedColumns), nil +} + +// compareTestAssertion is a generic function used for comparing string, ints, floats. +// It takes in a comparison string from one of: "==", "!=", "<", ">", "<=", ">=" +// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise +func compareTestAssertion[T constraints.Ordered](comparison string, expectedValue, actualValue T, assertionType string) string { + switch comparison { + case "==": + if actualValue != expectedValue { + return fmt.Sprintf("Assertion failed: %s equal to %v, got %v", assertionType, expectedValue, actualValue) + } + case "!=": + if actualValue == expectedValue { + return fmt.Sprintf("Assertion failed: %s not equal to %v, got %v", assertionType, expectedValue, actualValue) + } + case "<": + if actualValue >= expectedValue { + return fmt.Sprintf("Assertion failed: %s less than %v, got %v", assertionType, expectedValue, actualValue) + } + case "<=": + if actualValue > expectedValue { + return fmt.Sprintf("Assertion failed: %s less than or equal to %v, got %v", assertionType, expectedValue, actualValue) + } + case ">": + if actualValue <= expectedValue { + return fmt.Sprintf("Assertion failed: %s greater than %v, got %v", assertionType, expectedValue, actualValue) + } + case ">=": + if actualValue < expectedValue { + return fmt.Sprintf("Assertion failed: %s greater than or equal to %v, got %v", assertionType, expectedValue, actualValue) + } + default: + return fmt.Sprintf("%s is not a valid comparison type", comparison) + } + return "" +} + +// parseTestsDate is an internal function that parses the queried string according to allowed time formats for dolt_tests. +// It returns the parsed time, the format that succeeded, and an error if applicable. +func parseTestsDate(value string) (parsedTime time.Time, format string, err error) { + // List of valid formats + formats := []string{ + time.DateOnly, + time.DateTime, + time.TimeOnly, + time.RFC3339, + time.RFC1123Z, + } + + for _, format := range formats { + if parsedTime, parseErr := time.Parse(format, value); parseErr == nil { + return parsedTime, format, nil + } else { + err = parseErr + } + } + return time.Time{}, "", err +} + +// compareDates is a function used for comparing time values. +// It takes in a comparison string from one of: "==", "!=", "<", ">", "<=", ">=" +// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise +func compareDates(comparison string, expectedValue, realValue time.Time, format string, assertionType string) string { + expectedStr := expectedValue.Format(format) + realStr := realValue.Format(format) + switch comparison { + case "==": + if !expectedValue.Equal(realValue) { + return fmt.Sprintf("Assertion failed: %s equal to %s, got %s", assertionType, expectedStr, realStr) + } + case "!=": + if expectedValue.Equal(realValue) { + return fmt.Sprintf("Assertion failed: %s not equal to %s, got %s", assertionType, expectedStr, realStr) + } + case "<": + if realValue.Equal(expectedValue) || realValue.After(expectedValue) { + return fmt.Sprintf("Assertion failed: %s less than %s, got %s", assertionType, expectedStr, realStr) + } + case "<=": + if realValue.After(expectedValue) { + return fmt.Sprintf("Assertion failed: %s less than or equal to %s, got %s", assertionType, expectedStr, realStr) + } + case ">": + if realValue.Before(expectedValue) || realValue.Equal(expectedValue) { + return fmt.Sprintf("Assertion failed: %s greater than %s, got %s", assertionType, expectedStr, realStr) + } + case ">=": + if realValue.Before(expectedValue) { + return fmt.Sprintf("Assertion failed: %s greater than or equal to %s, got %s", assertionType, expectedStr, realStr) + } + default: + return fmt.Sprintf("%s is not a valid comparison type", comparison) + } + return "" +} + +// compareDecimals is a function used for comparing decimals. +// It takes in a comparison string from one of: "==", "!=", "<", ">", "<=", ">=" +// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise +func compareDecimals(comparison string, expectedValue, realValue decimal.Decimal, assertionType string) string { + switch comparison { + case "==": + if !expectedValue.Equal(realValue) { + return fmt.Sprintf("Assertion failed: %s equal to %v, got %v", assertionType, expectedValue, realValue) + } + case "!=": + if expectedValue.Equal(realValue) { + return fmt.Sprintf("Assertion failed: %s not equal to %v, got %v", assertionType, expectedValue, realValue) + } + case "<": + if realValue.GreaterThanOrEqual(expectedValue) { + return fmt.Sprintf("Assertion failed: %s less than %v, got %v", assertionType, expectedValue, realValue) + } + case "<=": + if realValue.GreaterThan(expectedValue) { + return fmt.Sprintf("Assertion failed: %s less than or equal to %v, got %v", assertionType, expectedValue, realValue) + } + case ">": + if realValue.LessThanOrEqual(expectedValue) { + return fmt.Sprintf("Assertion failed: %s greater than %v, got %v", assertionType, expectedValue, realValue) + } + case ">=": + if realValue.LessThan(expectedValue) { + return fmt.Sprintf("Assertion failed: %s greater than or equal to %v, got %v", assertionType, expectedValue, realValue) + } + default: + return fmt.Sprintf("%s is not a valid comparison type", comparison) + } + return "" +} + +// getTinyIntColAsBool returns the value interface{} as a bool +// This is necessary because the query engine may return a tinyint column as a bool, int, or other types. +// Based on GetTinyIntColAsBool from commands/utils.go, which we can't depend on here due to package cycles. +func getInterfaceAsBool(col interface{}) (bool, error) { + switch v := col.(type) { + case bool: + return v, nil + case int: + return v == 1, nil + case int8: + return v == 1, nil + case int16: + return v == 1, nil + case int32: + return v == 1, nil + case int64: + return v == 1, nil + case uint: + return v == 1, nil + case uint8: + return v == 1, nil + case uint16: + return v == 1, nil + case uint32: + return v == 1, nil + case uint64: + return v == 1, nil + case string: + return v == "1", nil + default: + return false, fmt.Errorf("unexpected type %T, was expecting bool, int, or string", v) + } +} + +// compareBooleans is a function used for comparing boolean values. +// It takes in a comparison string from one of: "==", "!=" +// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise +func compareBooleans(comparison string, expectedValue, realValue bool, assertionType string) string { + switch comparison { + case "==": + if expectedValue != realValue { + return fmt.Sprintf("Assertion failed: %s equal to %t, got %t", assertionType, expectedValue, realValue) + } + case "!=": + if expectedValue == realValue { + return fmt.Sprintf("Assertion failed: %s not equal to %t, got %t", assertionType, expectedValue, realValue) + } + default: + return fmt.Sprintf("%s is not a valid comparison for boolean values. Only '==' and '!=' are supported", comparison) + } + return "" +} + +// compareNullValue is a function used for comparing a null value. +// It takes in a comparison string from one of: "==", "!=" +// It returns a string. The string is empty if the assertion passed, or has a message explaining the failure otherwise +func compareNullValue(comparison string, actualValue interface{}, assertionType string) string { + switch comparison { + case "==": + if actualValue != nil { + return fmt.Sprintf("Assertion failed: %s equal to NULL, got %v", assertionType, actualValue) + } + case "!=": + if actualValue == nil { + return fmt.Sprintf("Assertion failed: %s not equal to NULL, got NULL", assertionType) + } + default: + return fmt.Sprintf("%s is not a valid comparison for NULL values", comparison) + } + return "" +} + +// GetStringColAsString is a function that returns a text column as a string. +// This is necessary as the dolt_tests system table returns *val.TextStorage types under certain situations, +// so we use a special parser to get the correct string values +func GetStringColAsString(sqlCtx *sql.Context, tableValue interface{}) (*string, error) { + if ts, ok := tableValue.(*val.TextStorage); ok { + str, err := ts.Unwrap(sqlCtx) + return &str, err + } else if str, ok := tableValue.(string); ok { + return &str, nil + } else if tableValue == nil { + return nil, nil + } else { + return nil, fmt.Errorf("unexpected type %T, was expecting string", tableValue) + } +} diff --git a/go/libraries/doltcore/sqle/dtables/binlog_table.go b/go/libraries/doltcore/sqle/dtables/binlog_table.go deleted file mode 100644 index 5dbd8d1c03..0000000000 --- a/go/libraries/doltcore/sqle/dtables/binlog_table.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dtables - -import ( - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/types" - "github.com/dolthub/vitess/go/sqltypes" - - "github.com/dolthub/dolt/go/libraries/doltcore/branch_control" - - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" -) - -const ( - AccessBinlogTableName = AccessTableName + "_binlog" - NamespaceBinlogTableName = NamespaceTableName + "_binlog" -) - -// accessBinlogSchema is the schema for the "dolt_branch_control_binlog" table. -var accessBinlogSchema = sql.Schema{ - &sql.Column{ - Name: "index", - Type: types.Int64, - Source: AccessBinlogTableName, - PrimaryKey: true, - }, - &sql.Column{ - Name: "operation", - Type: types.MustCreateEnumType([]string{"insert", "delete"}, sql.Collation_utf8mb4_0900_bin), - Source: AccessBinlogTableName, - PrimaryKey: false, - }, - &sql.Column{ - Name: "branch", - Type: types.MustCreateString(sqltypes.VarChar, 16383, sql.Collation_utf8mb4_0900_ai_ci), - Source: AccessBinlogTableName, - PrimaryKey: false, - }, - &sql.Column{ - Name: "user", - Type: types.MustCreateString(sqltypes.VarChar, 16383, sql.Collation_utf8mb4_0900_bin), - Source: AccessBinlogTableName, - PrimaryKey: false, - }, - &sql.Column{ - Name: "host", - Type: types.MustCreateString(sqltypes.VarChar, 16383, sql.Collation_utf8mb4_0900_ai_ci), - Source: AccessBinlogTableName, - PrimaryKey: false, - }, - &sql.Column{ - Name: "permissions", - Type: types.MustCreateSetType(PermissionsStrings, sql.Collation_utf8mb4_0900_ai_ci), - Source: AccessBinlogTableName, - PrimaryKey: false, - }, -} - -// namespaceBinlogSchema is the schema for the "dolt_branch_namespace_control_binlog" table. -var namespaceBinlogSchema = sql.Schema{ - &sql.Column{ - Name: "index", - Type: types.Int64, - Source: NamespaceBinlogTableName, - PrimaryKey: true, - }, - &sql.Column{ - Name: "operation", - Type: types.MustCreateEnumType([]string{"insert", "delete"}, sql.Collation_utf8mb4_0900_bin), - Source: NamespaceBinlogTableName, - PrimaryKey: false, - }, - &sql.Column{ - Name: "branch", - Type: types.MustCreateString(sqltypes.VarChar, 16383, sql.Collation_utf8mb4_0900_ai_ci), - Source: NamespaceBinlogTableName, - PrimaryKey: false, - }, - &sql.Column{ - Name: "user", - Type: types.MustCreateString(sqltypes.VarChar, 16383, sql.Collation_utf8mb4_0900_bin), - Source: NamespaceBinlogTableName, - PrimaryKey: false, - }, - &sql.Column{ - Name: "host", - Type: types.MustCreateString(sqltypes.VarChar, 16383, sql.Collation_utf8mb4_0900_ai_ci), - Source: NamespaceBinlogTableName, - PrimaryKey: false, - }, -} - -// BinlogTable provides a queryable view over the Binlog. -type BinlogTable struct { - Log *branch_control.Binlog - IsAccess bool -} - -var _ sql.Table = BinlogTable{} - -// Name implements the interface sql.Table. -func (b BinlogTable) Name() string { - if b.IsAccess { - return AccessBinlogTableName - } else { - return NamespaceBinlogTableName - } -} - -// String implements the interface sql.Table. -func (b BinlogTable) String() string { - if b.IsAccess { - return AccessBinlogTableName - } else { - return NamespaceBinlogTableName - } -} - -// Schema implements the interface sql.Table. -func (b BinlogTable) Schema() sql.Schema { - if b.IsAccess { - return accessBinlogSchema - } else { - return namespaceBinlogSchema - } -} - -// Collation implements the interface sql.Table. -func (b BinlogTable) Collation() sql.CollationID { - return sql.Collation_Default -} - -// Partitions implements the interface sql.Table. -func (b BinlogTable) Partitions(context *sql.Context) (sql.PartitionIter, error) { - return index.SinglePartitionIterFromNomsMap(nil), nil -} - -// PartitionRows implements the interface sql.Table. -func (b BinlogTable) PartitionRows(context *sql.Context, partition sql.Partition) (sql.RowIter, error) { - b.Log.RWMutex.RLock() - defer b.Log.RWMutex.RUnlock() - - binlogRows := b.Log.Rows() - rows := make([]sql.Row, len(binlogRows)) - for i := 0; i < len(binlogRows); i++ { - logRow := binlogRows[i] - operation := uint16(1) - if !logRow.IsInsert { - operation = 2 - } - - if b.IsAccess { - rows[i] = sql.Row{ - int64(i), - operation, - logRow.Branch, - logRow.User, - logRow.Host, - logRow.Permissions, - } - } else { - rows[i] = sql.Row{ - int64(i), - operation, - logRow.Branch, - logRow.User, - logRow.Host, - } - } - } - return sql.RowsToRowIter(rows...), nil -} diff --git a/go/libraries/doltcore/sqle/dtables/diff_table.go b/go/libraries/doltcore/sqle/dtables/diff_table.go index 7730561051..cbbb4eb983 100644 --- a/go/libraries/doltcore/sqle/dtables/diff_table.go +++ b/go/libraries/doltcore/sqle/dtables/diff_table.go @@ -24,9 +24,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/diff" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/rowconv" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/expreval" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" @@ -591,38 +589,6 @@ func (dt *DiffTable) PreciseMatch() bool { return false } -// tableData returns the map of primary key to values for the specified table (or an empty map if the tbl is null) -// and the schema of the table (or EmptySchema if tbl is null). -func tableData(ctx *sql.Context, tbl *doltdb.Table, ddb *doltdb.DoltDB) (durable.Index, schema.Schema, error) { - var data durable.Index - var err error - - if tbl == nil { - data, err = durable.NewEmptyPrimaryIndex(ctx, ddb.ValueReadWriter(), ddb.NodeStore(), schema.EmptySchema) - if err != nil { - return nil, nil, err - } - } else { - data, err = tbl.GetRowData(ctx) - if err != nil { - return nil, nil, err - } - } - - var sch schema.Schema - if tbl == nil { - sch = schema.EmptySchema - } else { - sch, err = tbl.GetSchema(ctx) - - if err != nil { - return nil, nil, err - } - } - - return data, sch, nil -} - type TblInfoAtCommit struct { date *types.Timestamp tbl *doltdb.Table @@ -891,20 +857,6 @@ func (dps *DiffPartitions) Close(*sql.Context) error { return nil } -// rowConvForSchema creates a RowConverter for transforming rows with the given schema a target schema. -func (dp DiffPartition) rowConvForSchema(ctx context.Context, vrw types.ValueReadWriter, targetSch, srcSch schema.Schema) (*rowconv.RowConverter, error) { - if schema.SchemasAreEqual(srcSch, schema.EmptySchema) { - return rowconv.IdentityConverter, nil - } - - fm, err := rowconv.TagMappingByTagAndName(srcSch, targetSch) - if err != nil { - return nil, err - } - - return rowconv.NewRowConverter(ctx, vrw, fm) -} - // GetDiffTableSchemaAndJoiner returns the schema for the diff table given a // target schema for a row |sch|. In the old storage format, it also returns the // associated joiner. diff --git a/go/libraries/doltcore/sqle/dtables/expression.go b/go/libraries/doltcore/sqle/dtables/expression.go deleted file mode 100755 index 263c213387..0000000000 --- a/go/libraries/doltcore/sqle/dtables/expression.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2022 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dtables - -import ( - "strings" - - "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/go-mysql-server/sql/expression" - - "github.com/dolthub/dolt/go/libraries/utils/set" -) - -type Predicate func(sql.Expression) bool - -// ColumnPredicate returns a predicate function for expressions on the column names given -func ColumnPredicate(colNameSet *set.StrSet) Predicate { - return func(filter sql.Expression) bool { - isCommitFilter := true - sql.Inspect(filter, func(e sql.Expression) (cont bool) { - if e == nil { - return true - } - - switch val := e.(type) { - case *expression.GetField: - if !colNameSet.Contains(strings.ToLower(val.Name())) { - isCommitFilter = false - return false - } - } - - return true - }) - - return isCommitFilter - } -} - -// FilterFilters returns the subset of the expressions given that match the given predicate -func FilterFilters(filters []sql.Expression, predicate func(filter sql.Expression) bool) []sql.Expression { - matching := make([]sql.Expression, 0, len(filters)) - for _, f := range filters { - if predicate(f) { - matching = append(matching, f) - } - } - return matching -} diff --git a/go/libraries/doltcore/sqle/dtables/stashes_table.go b/go/libraries/doltcore/sqle/dtables/stashes_table.go index 938230e1c4..2125d259fd 100644 --- a/go/libraries/doltcore/sqle/dtables/stashes_table.go +++ b/go/libraries/doltcore/sqle/dtables/stashes_table.go @@ -156,49 +156,3 @@ func (itr *StashItr) Next(*sql.Context) (sql.Row, error) { func (itr *StashItr) Close(*sql.Context) error { return nil } - -var _ sql.RowReplacer = stashWriter{nil} -var _ sql.RowUpdater = stashWriter{nil} -var _ sql.RowInserter = stashWriter{nil} -var _ sql.RowDeleter = stashWriter{nil} - -type stashWriter struct { - rt *StashesTable -} - -// Insert inserts the row given, returning an error if it cannot. Insert will be called once for each row to process -// for the insert operation, which may involve many rows. After all rows in an operation have been processed, Close -// is called. -func (bWr stashWriter) Insert(_ *sql.Context, _ sql.Row) error { - return fmt.Errorf("the dolt_stashes table is read-only; use the dolt_stash stored procedure to edit stashes") -} - -// Update the given row. Provides both the old and new rows. -func (bWr stashWriter) Update(_ *sql.Context, _ sql.Row, _ sql.Row) error { - return fmt.Errorf("the dolt_stash table is read-only; use the dolt_stash stored procedure to edit stashes") -} - -// Delete deletes the given row. Returns ErrDeleteRowNotFound if the row was not found. Delete will be called once for -// each row to process for the delete operation, which may involve many rows. After all rows have been processed, -// Close is called. -func (bWr stashWriter) Delete(_ *sql.Context, _ sql.Row) error { - return fmt.Errorf("the dolt_stash table is read-only; use the dolt_stash stored procedure to edit stashes") -} - -// StatementBegin implements the interface sql.TableEditor. Currently a no-op. -func (bWr stashWriter) StatementBegin(*sql.Context) {} - -// DiscardChanges implements the interface sql.TableEditor. Currently a no-op. -func (bWr stashWriter) DiscardChanges(_ *sql.Context, _ error) error { - return nil -} - -// StatementComplete implements the interface sql.TableEditor. Currently a no-op. -func (bWr stashWriter) StatementComplete(*sql.Context) error { - return nil -} - -// Close finalizes the delete operation, persisting the result. -func (bWr stashWriter) Close(*sql.Context) error { - return nil -} diff --git a/go/libraries/doltcore/sqle/dtables/unscoped_diff_table.go b/go/libraries/doltcore/sqle/dtables/unscoped_diff_table.go index 4192a2e8e0..c41441797b 100644 --- a/go/libraries/doltcore/sqle/dtables/unscoped_diff_table.go +++ b/go/libraries/doltcore/sqle/dtables/unscoped_diff_table.go @@ -23,7 +23,6 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/go-mysql-server/sql/expression" "github.com/dolthub/go-mysql-server/sql/plan" - "github.com/dolthub/go-mysql-server/sql/transform" "github.com/dolthub/go-mysql-server/sql/types" "github.com/dolthub/dolt/go/libraries/doltcore/diff" @@ -462,64 +461,6 @@ func (itr *doltDiffCommitHistoryRowItr) Close(*sql.Context) error { return nil } -// isTableDataEmpty return true if the table does not contain any data -func isTableDataEmpty(ctx *sql.Context, table *doltdb.Table) (bool, error) { - rowData, err := table.GetRowData(ctx) - if err != nil { - return false, err - } - - return rowData.Empty() -} - -// commitFilterForDiffTableFilterExprs returns CommitFilter used for CommitItr. -func commitFilterForDiffTableFilterExprs(filters []sql.Expression) (doltdb.CommitFilter[*sql.Context], error) { - filters = transformFilters(filters...) - - return func(ctx *sql.Context, h hash.Hash, optCmt *doltdb.OptionalCommit) (filterOut bool, err error) { - cm, ok := optCmt.ToCommit() - if !ok { - return false, doltdb.ErrGhostCommitEncountered - } - - meta, err := cm.GetCommitMeta(ctx) - if err != nil { - return false, err - } - for _, filter := range filters { - res, err := filter.Eval(ctx, sql.Row{h.String(), meta.Name, meta.Time()}) - if err != nil { - return false, err - } - b, ok := res.(bool) - if ok && !b { - return true, nil - } - } - - return false, err - }, nil -} - -// transformFilters return filter expressions with index specified for rows used in CommitFilter. -func transformFilters(filters ...sql.Expression) []sql.Expression { - for i := range filters { - filters[i], _, _ = transform.Expr(filters[i], func(e sql.Expression) (sql.Expression, transform.TreeIdentity, error) { - gf, ok := e.(*expression.GetField) - if !ok { - return e, transform.SameTree, nil - } - switch gf.Name() { - case commitHashCol: - return gf.WithIndex(0), transform.NewTree, nil - default: - return gf, transform.SameTree, nil - } - }) - } - return filters -} - func getCommitsFromCommitHashEquality(ctx *sql.Context, ddb *doltdb.DoltDB, filters []sql.Expression) ([]*doltdb.Commit, bool) { var commits []*doltdb.Commit var isCommitHashEquality bool diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go index f52955b01f..f926cc2264 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go @@ -1210,6 +1210,11 @@ func TestDoltDdlScripts(t *testing.T) { RunDoltDdlScripts(t, harness) } +func TestDoltCommitVerificationScripts(t *testing.T) { + harness := newDoltEnginetestHarness(t) + RunDoltCommitVerificationScripts(t, harness) +} + func TestBrokenDdlScripts(t *testing.T) { for _, script := range BrokenDDLScripts { t.Skip(script.Name) diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go index 9d6e27b461..b257a67119 100755 --- a/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go @@ -512,7 +512,8 @@ func RunStoredProceduresTest(t *testing.T, h DoltEnginetestHarness) { } func RunDoltStoredProceduresTest(t *testing.T, h DoltEnginetestHarness) { - for _, script := range DoltProcedureTests { + scripts := append(DoltProcedureTests, DoltCleanProcedureScripts...) + for _, script := range scripts { func() { h := h.NewHarness(t) h.UseLocalFileSystem() @@ -523,7 +524,8 @@ func RunDoltStoredProceduresTest(t *testing.T, h DoltEnginetestHarness) { } func RunDoltStoredProceduresPreparedTest(t *testing.T, h DoltEnginetestHarness) { - for _, script := range DoltProcedureTests { + scripts := append(DoltProcedureTests, DoltCleanProcedureScripts...) + for _, script := range scripts { func() { h := h.NewHarness(t) h.UseLocalFileSystem() @@ -2145,3 +2147,12 @@ func RunTransactionTestsWithEngineSetup(t *testing.T, setupEngine func(*gms.Engi }) } } + +func RunDoltCommitVerificationScripts(t *testing.T, harness DoltEnginetestHarness) { + for _, script := range DoltCommitVerificationScripts { + harness := harness.NewHarness(t) + + enginetest.TestScript(t, harness, script) + harness.Close() + } +} diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go index 127ac5ec51..97d2bc748b 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_harness.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_harness.go @@ -190,7 +190,6 @@ func (d *DoltHarness) resetScripts() []setup.SetupScript { for i := range dbs { db := dbs[i] resetCmds = append(resetCmds, setup.SetupScript{fmt.Sprintf("use %s", db)}) - // Any auto increment tables must be dropped and recreated to get a fresh state for the global auto increment // sequence trackers _, aiTables := enginetest.MustQuery(ctx, d.engine, @@ -218,6 +217,7 @@ func (d *DoltHarness) resetScripts() []setup.SetupScript { resetCmds = append(resetCmds, setup.SetupScript{fmt.Sprintf("drop database if exists %s", db)}) } } + resetCmds = append(resetCmds, setup.SetupScript{"use mydb"}) return resetCmds } @@ -229,7 +229,7 @@ func commitScripts(dbs []string) []setup.SetupScript { db := dbs[i] commitCmds = append(commitCmds, fmt.Sprintf("use %s", db)) commitCmds = append(commitCmds, "call dolt_add('.')") - commitCmds = append(commitCmds, fmt.Sprintf("call dolt_commit('--allow-empty', '-am', 'checkpoint enginetest database %s', '--date', '1970-01-01T12:00:00')", db)) + commitCmds = append(commitCmds, fmt.Sprintf("call dolt_commit('--allow-empty', '-am', 'checkpoint enginetest database %s', '--date', '1970-01-01T12:00:00', '--skip-verification')", db)) } commitCmds = append(commitCmds, "use mydb") return []setup.SetupScript{commitCmds} diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_clean.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_clean.go new file mode 100644 index 0000000000..d3455770d4 --- /dev/null +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_clean.go @@ -0,0 +1,80 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enginetest + +import ( + "github.com/dolthub/go-mysql-server/enginetest/queries" + "github.com/dolthub/go-mysql-server/sql" +) + +// DoltCleanProcedureScripts are script tests for the dolt_clean procedure. +var DoltCleanProcedureScripts = []queries.ScriptTest{ + { + Name: "dolt_clean does not drop tables matching dolt_ignore", + SetUpScript: []string{ + "CREATE TABLE ignored_foo (id int primary key);", + "INSERT INTO ignored_foo VALUES (1);", + "INSERT INTO dolt_ignore VALUES ('ignored_*', true);", + "CALL dolt_add('dolt_ignore');", + "CALL dolt_commit('-m', 'add dolt_ignore');", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SELECT * FROM ignored_foo;", + Expected: []sql.Row{{1}}, + }, + { + Query: "CALL dolt_clean();", + Expected: []sql.Row{{0}}, + }, + { + Query: "SELECT * FROM ignored_foo;", + Expected: []sql.Row{{1}}, + }, + { + Query: "SHOW TABLES;", + Expected: []sql.Row{{"ignored_foo"}}, + }, + }, + }, + { + Name: "dolt_clean -x drops tables matching dolt_ignore", + SetUpScript: []string{ + "CREATE TABLE ignored_bar (id int primary key);", + "INSERT INTO ignored_bar VALUES (1);", + "INSERT INTO dolt_ignore VALUES ('ignored_*', true);", + "CALL dolt_add('dolt_ignore');", + "CALL dolt_commit('-m', 'add dolt_ignore');", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SELECT * FROM ignored_bar;", + Expected: []sql.Row{{1}}, + }, + { + Query: "CALL dolt_clean('-x');", + Expected: []sql.Row{{0}}, + }, + { + Query: "SELECT * FROM ignored_bar;", + ExpectedErrStr: "table not found: ignored_bar", + }, + { + Query: "SHOW TABLES;", + Expected: []sql.Row{}, + }, + }, + }, +} diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_commit_verification.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_commit_verification.go new file mode 100644 index 0000000000..0a2251cc95 --- /dev/null +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_commit_verification.go @@ -0,0 +1,538 @@ +// Copyright 2025 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enginetest + +import ( + "regexp" + + "github.com/dolthub/go-mysql-server/enginetest" + "github.com/dolthub/go-mysql-server/enginetest/queries" + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/go-mysql-server/sql/types" + + "github.com/dolthub/dolt/go/store/hash" +) + +// commitHashValidator validates commit hash format (32 character hex) +type commitHashValidator struct{} + +var _ enginetest.CustomValueValidator = &commitHashValidator{} + +func (chv *commitHashValidator) Validate(val interface{}) (bool, error) { + h, ok := val.(string) + if !ok { + return false, nil + } + + _, ok = hash.MaybeParse(h) + return ok, nil +} + +// successfulRebaseMessageValidator validates successful rebase message format +type successfulRebaseMessageValidator struct{} + +var _ enginetest.CustomValueValidator = &successfulRebaseMessageValidator{} +var successfulRebaseRegex = regexp.MustCompile(`^Successfully rebased.*`) + +func (srmv *successfulRebaseMessageValidator) Validate(val interface{}) (bool, error) { + message, ok := val.(string) + if !ok { + return false, nil + } + return successfulRebaseRegex.MatchString(message), nil +} + +var commitHash = &commitHashValidator{} +var successfulRebaseMessage = &successfulRebaseMessageValidator{} + +var DoltCommitVerificationScripts = []queries.ScriptTest{ + { + Name: "test verification system variables exist and have correct defaults", + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SHOW GLOBAL VARIABLES LIKE 'dolt_commit_verification_groups'", + Expected: []sql.Row{ + {"dolt_commit_verification_groups", ""}, + }, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "test verification system variables can be set", + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SET GLOBAL dolt_commit_verification_groups = '*'", + Expected: []sql.Row{{types.OkResult{}}}, + }, + { + Query: "SHOW GLOBAL VARIABLES LIKE 'dolt_commit_verification_groups'", + Expected: []sql.Row{ + {"dolt_commit_verification_groups", "*"}, + }, + }, + { + Query: "SET GLOBAL dolt_commit_verification_groups = 'unit,integration'", + Expected: []sql.Row{{types.OkResult{}}}, + }, + { + Query: "SHOW GLOBAL VARIABLES LIKE 'dolt_commit_verification_groups'", + Expected: []sql.Row{ + {"dolt_commit_verification_groups", "unit,integration"}, + }, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "commit verification enabled - all tests pass", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com'), (2, 'Bob', 'bob@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_users_count', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '2'), " + + "('test_alice_exists', 'unit', 'SELECT COUNT(*) FROM users WHERE name = \"Alice\"', 'expected_single_value', '==', '1')", + "CALL dolt_add('.')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_commit('-m', 'Commit with passing tests')", + ExpectedColumns: sql.Schema{ + {Name: "hash", Type: types.LongText, Nullable: false}, + }, + Expected: []sql.Row{{commitHash}}, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "commit verification enabled - tests fail, commit aborted", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com'), (2, 'Bob', 'bob@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_users_count', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '2'), " + + "('test_will_fail', 'integration', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '999')", + "CALL dolt_add('.')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_commit('-m', 'Commit that should fail verification')", + ExpectedErrStr: "commit verification failed: test_will_fail (Assertion failed: expected_single_value equal to 999, got 2)", + }, + { + Query: "CALL dolt_commit('--skip-verification','-m', 'skip verification')", + Expected: []sql.Row{{commitHash}}, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "commit with test verification - specific test groups", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = 'unit'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com'), (2, 'Bob', 'bob@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_users_count', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '2'), " + + "('test_will_fail', 'integration', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '999')", + "CALL dolt_add('.')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_commit('-m', 'Commit with unit tests only')", + Expected: []sql.Row{{commitHash}}, + }, + { + Query: "SET GLOBAL dolt_commit_verification_groups = 'integration'", + SkipResultsCheck: true, + }, + { + Query: "CALL dolt_commit('--allow-empty', '--amend', '-m', 'fail please')", + ExpectedErrStr: "commit verification failed: test_will_fail (Assertion failed: expected_single_value equal to 999, got 2)", + }, + { + Query: "CALL dolt_commit('--allow-empty', '--amend', '--skip-verification', '-m', 'skip the tests')", + Expected: []sql.Row{{commitHash}}, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "cherry-pick with test verification enabled - tests pass", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_user_count_update', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '1')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'add test')", + "CALL dolt_checkout('-b', 'feature')", + "INSERT INTO users VALUES (2, 'Bob', 'bob@example.com')", + "UPDATE dolt_tests SET assertion_value = '2' WHERE test_name = 'test_user_count_update'", + "CALL dolt_add('.')", + "call dolt_commit_hash_out(@commit_1_hash,'--skip-verification', '-m', 'Add Bob and update test')", + "INSERT INTO users VALUES (3, 'Charlie', 'chuck@exampl.com')", + "CALL dolt_add('.')", + "call dolt_commit_hash_out(@commit_2_hash,'--skip-verification', '-m', 'Add Charlie')", + "CALL dolt_checkout('main')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_cherry_pick(@commit_1_hash)", + Expected: []sql.Row{{commitHash, int64(0), int64(0), int64(0)}}, + }, + { + Query: "CALL dolt_cherry_pick(@commit_2_hash)", + ExpectedErrStr: "commit verification failed: test_user_count_update (Assertion failed: expected_single_value equal to 2, got 3)", + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "cherry-pick with test verification enabled - tests fail, aborted", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_users_count', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '1')", + "CALL dolt_add('.')", + "CALL dolt_commit('-m', 'Initial commit')", + "CALL dolt_checkout('-b', 'feature')", + "INSERT INTO users VALUES (2, 'Bob', 'bob@example.com')", + "CALL dolt_add('.')", + "call dolt_commit_hash_out(@commit_hash,'--skip-verification', '-m', 'Add Bob but dont update test')", + "CALL dolt_checkout('main')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_cherry_pick(@commit_hash)", + ExpectedErrStr: "commit verification failed: test_users_count (Assertion failed: expected_single_value equal to 1, got 2)", + }, + { + Query: "CALL dolt_cherry_pick('--skip-verification', @commit_hash)", + Expected: []sql.Row{{commitHash, int64(0), int64(0), int64(0)}}, + }, + { + Query: "select * from dolt_test_run('*')", + Expected: []sql.Row{ + {"test_users_count", "unit", "SELECT COUNT(*) FROM users", "FAIL", "Assertion failed: expected_single_value equal to 1, got 2"}, + }, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "rebase with test verification enabled - tests pass", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_users_count', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '1')", + "CALL dolt_add('.')", + "CALL dolt_commit('-m', 'Initial commit')", + "DELETE FROM users where id = 1", + "INSERT INTO users VALUES (1, 'Zed', 'zed@example.com')", + "CALL dolt_commit('-am', 'drop Alice, add Zed')", // tests still pass here. + "CALL dolt_checkout('-b', 'feature', 'HEAD~1')", + "INSERT INTO users VALUES (2, 'Bob', 'bob@example.com')", + "UPDATE dolt_tests SET assertion_value = '2' WHERE test_name = 'test_users_count'", + "CALL dolt_add('.')", + "CALL dolt_commit('-m', 'Add Bob and update test')", + "INSERT INTO users VALUES (3, 'Charlie', 'charlie@example.com')", + "UPDATE dolt_tests SET assertion_value = '3' WHERE test_name = 'test_users_count'", + "CALL dolt_add('.')", + "CALL dolt_commit('-m', 'Add Charlie, update test')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_rebase('main')", + Expected: []sql.Row{{int64(0), successfulRebaseMessage}}, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + + { + Name: "rebase with test verification enabled - tests fail, aborted", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_users_count', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '1')", + "CALL dolt_add('.')", + "CALL dolt_commit('-m', 'Initial commit')", + "CALL dolt_checkout('-b', 'feature')", + "INSERT INTO users VALUES (2, 'Bob', 'bob@example.com')", + "UPDATE dolt_tests SET assertion_value = '2' WHERE test_name = 'test_users_count'", + "CALL dolt_add('.')", + "CALL dolt_commit('-m', 'Add Bob but dont update test')", + "CALL dolt_checkout('main')", + "INSERT INTO users VALUES (3, 'Charlie', 'charlie@example.com')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Add Charlie')", // this will trip the existing test. + "CALL dolt_checkout('feature')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_rebase('main')", + ExpectedErrStr: "commit verification failed: test_users_count (Assertion failed: expected_single_value equal to 2, got 3)", + }, + { + Query: "CALL dolt_rebase('--abort')", + Expected: []sql.Row{{0, "Interactive rebase aborted"}}, + }, + { + Query: "CALL dolt_rebase('--skip-verification', 'main')", + Expected: []sql.Row{{int64(0), successfulRebaseMessage}}, + }, + { + Query: "select * from dolt_test_run('*')", + Expected: []sql.Row{ + {"test_users_count", "unit", "SELECT COUNT(*) FROM users", "FAIL", "Assertion failed: expected_single_value equal to 2, got 3"}, + }, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "interactive rebase with --skip-verification flag should persist across continue operations", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_users_count', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '1')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Initial commit')", + "CALL dolt_checkout('-b', 'feature')", + "INSERT INTO users VALUES (2, 'Bob', 'bob@example.com')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Add Bob but dont update test')", // This will cause test to fail + "INSERT INTO users VALUES (3, 'Charlie', 'charlie@example.com')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Add Charlie')", + "CALL dolt_checkout('main')", + "INSERT INTO users VALUES (4, 'David', 'david@example.com')", // Add a commit to main to create divergence + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Add David on main')", + "CALL dolt_checkout('feature')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_rebase('--interactive', '--skip-verification', 'main')", + Expected: []sql.Row{{0, "interactive rebase started on branch dolt_rebase_feature; adjust the rebase plan in the dolt_rebase table, then continue rebasing by calling dolt_rebase('--continue')"}}, + }, + { + Query: "CALL dolt_rebase('--continue')", // This should NOT require --skip-verification flag but should still skip tests + Expected: []sql.Row{{int64(0), successfulRebaseMessage}}, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "test verification with no dolt_tests errors", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com')", + "CALL dolt_add('.')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_commit('-m', 'Commit without dolt_tests table')", + ExpectedErrStr: "failed to run dolt_test_run for group *: could not find tests for argument: *", + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "test verification with mixed test groups - only specified groups run", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = 'unit'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com'), (2, 'Bob', 'bob@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_users_unit', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '2'), " + + "('test_users_integration', 'integration', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '999')", + "CALL dolt_add('.')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_commit('-m', 'Commit with unit tests only - should pass')", + Expected: []sql.Row{{commitHash}}, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "test verification error message includes test details", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com'), (2, 'Bob', 'bob@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_specific_failure', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '999')", + "CALL dolt_add('.')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_commit('-m', 'Commit with specific test failure')", + ExpectedErrStr: "commit verification failed: test_specific_failure (Assertion failed: expected_single_value equal to 999, got 2)", + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "merge with test verification enabled - tests pass", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_alice_exists', 'unit', 'SELECT COUNT(*) FROM users WHERE name = \"Alice\"', 'expected_single_value', '==', '1')", + "CALL dolt_add('.')", + "CALL dolt_commit('-m', 'Initial commit')", + "CALL dolt_checkout('-b', 'feature')", + "INSERT INTO users VALUES (2, 'Bob', 'bob@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_bob_exists', 'unit', 'SELECT COUNT(*) FROM users WHERE name = \"Bob\"', 'expected_single_value', '==', '1')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Add Bob')", + "CALL dolt_checkout('main')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_merge('feature')", + Expected: []sql.Row{{commitHash, int64(1), int64(0), "merge successful"}}, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "merge with test verification enabled - tests fail, merge aborted", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_will_fail', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '999')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Initial commit with failing test')", + "CALL dolt_checkout('-b', 'feature')", + "INSERT INTO users VALUES (2, 'Bob', 'bob@example.com')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Add Bob')", + "CALL dolt_checkout('main')", + "INSERT INTO users VALUES (3, 'Charlie', 'charlie@example.com')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Add Charlie to force non-FF merge')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_merge('feature')", + ExpectedErrStr: "commit verification failed: test_will_fail (Assertion failed: expected_single_value equal to 999, got 3)", + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, + { + Name: "merge with --skip-verification flag bypasses verification", + SetUpScript: []string{ + "SET GLOBAL dolt_commit_verification_groups = '*'", + "CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100) NOT NULL, email VARCHAR(100))", + "INSERT INTO users VALUES (1, 'Alice', 'alice@example.com')", + "INSERT INTO dolt_tests (test_name, test_group, test_query, assertion_type, assertion_comparator, assertion_value) VALUES " + + "('test_will_fail', 'unit', 'SELECT COUNT(*) FROM users', 'expected_single_value', '==', '999')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Initial commit with failing test')", + "CALL dolt_checkout('-b', 'feature')", + "INSERT INTO users VALUES (2, 'Bob', 'bob@example.com')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Add Bob')", + "CALL dolt_checkout('main')", + "INSERT INTO users VALUES (3, 'Charlie', 'charlie@example.com')", + "CALL dolt_add('.')", + "CALL dolt_commit('--skip-verification', '-m', 'Add Charlie to force non-FF merge')", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "CALL dolt_merge('--skip-verification', 'feature')", + Expected: []sql.Row{{commitHash, int64(0), int64(0), "merge successful"}}, + }, + { + Query: "select * from dolt_test_run('*')", + Expected: []sql.Row{ + {"test_will_fail", "unit", "SELECT COUNT(*) FROM users", "FAIL", "Assertion failed: expected_single_value equal to 999, got 3"}, + }, + }, + { // Test harness bleeds GLOBAL variable changes across tests, so reset after each test. + Query: "SET GLOBAL dolt_commit_verification_groups = ''", + SkipResultsCheck: true, + }, + }, + }, +} diff --git a/go/libraries/doltcore/sqle/enginetest/dolt_queries_nonlocal.go b/go/libraries/doltcore/sqle/enginetest/dolt_queries_nonlocal.go index 7c0222691a..eae0562253 100644 --- a/go/libraries/doltcore/sqle/enginetest/dolt_queries_nonlocal.go +++ b/go/libraries/doltcore/sqle/enginetest/dolt_queries_nonlocal.go @@ -209,4 +209,46 @@ var NonlocalScripts = []queries.ScriptTest{ }, }, }, + { + // https://github.com/dolthub/dolt/issues/10462 + Name: "nonlocal table is not affected by dolt_clean()", + SetUpScript: []string{ + "CREATE TABLE global_test (id int auto_increment primary key, name varchar(100));", + "INSERT INTO global_test (id, name) VALUES (1, 'one');", + "CREATE TABLE foo (id int auto_increment primary key);", + "INSERT INTO dolt_nonlocal_tables (table_name, target_ref, options) VALUES ('global_*', 'main', 'immediate');", + "CALL dolt_add('dolt_nonlocal_tables');", + "CALL dolt_commit('-m', 'set up dolt_nonlocal_tables');", + }, + Assertions: []queries.ScriptTestAssertion{ + { + Query: "SELECT * FROM global_test;", + Expected: []sql.Row{{1, "one"}}, + }, + { + Query: "CALL dolt_clean();", + Expected: []sql.Row{{0}}, + }, + { + Query: "SELECT * FROM global_test;", + Expected: []sql.Row{{1, "one"}}, + }, + { + Query: "SHOW TABLES;", + Expected: []sql.Row{{"global_test"}}, + }, + { + Query: "CALL dolt_clean('-x')", + Expected: []sql.Row{{0}}, + }, + { + Query: "SELECT * FROM global_test;", + Expected: []sql.Row{{1, "one"}}, + }, + { + Query: "SHOW TABLES;", + Expected: []sql.Row{{"global_test"}}, + }, + }, + }, } diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index 1c2548696d..e4f8ef2eb1 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -16,7 +16,6 @@ package index import ( "context" - "errors" "fmt" "strings" "sync/atomic" @@ -30,7 +29,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/pool" @@ -60,12 +58,7 @@ type DoltIndex interface { Format() *types.NomsBinFormat IsPrimaryKey() bool - valueReadWriter() types.ValueReadWriter - - getDurableState(*sql.Context, DoltTableable) (*durableIndexState, error) coversColumns(s *durableIndexState, columns []uint64) bool - sqlRowConverter(*durableIndexState, []uint64) *KVToSqlRowConverter - lookupTags(s *durableIndexState) map[uint64]int } func NewBranchNameIndex(i *doltIndex) *BranchNameIndex { @@ -523,52 +516,6 @@ func (s *durableIndexState) coversAllColumns(i *doltIndex) bool { return covers } -func (s *durableIndexState) lookupTags(i *doltIndex) map[uint64]int { - cached := s.cachedLookupTags.Load() - if cached == nil { - tags := i.Schema().GetPKCols().Tags - sz := len(tags) - if sz == 0 { - sz = 1 - } - tocache := make(map[uint64]int, sz) - for i, tag := range tags { - tocache[tag] = i - } - if len(tocache) == 0 { - tocache[schema.KeylessRowIdTag] = 0 - } - s.cachedLookupTags.Store(tocache) - cached = tocache - } - return cached.(map[uint64]int) -} - -func projectionsEqual(x, y []uint64) bool { - if len(x) != len(y) { - return false - } - var i, j int - for i < len(x) && j < len(y) { - if x[i] != y[j] { - return false - } - i++ - j++ - } - return true -} -func (s *durableIndexState) sqlRowConverter(i *doltIndex, proj []uint64) *KVToSqlRowConverter { - cachedProjections := s.cachedProjections.Load() - cachedConverter := s.cachedSqlRowConverter.Load() - if cachedConverter == nil || !projectionsEqual(proj, cachedProjections.([]uint64)) { - cachedConverter = NewKVToSqlRowConverterForCols(i.Format(), i.Schema(), proj) - s.cachedSqlRowConverter.Store(cachedConverter) - s.cachedProjections.Store(proj) - } - return cachedConverter.(*KVToSqlRowConverter) -} - type cachedDurableIndexes struct { val atomic.Value } @@ -755,7 +702,7 @@ func (di *doltIndex) getDurableState(ctx *sql.Context, ti DoltTableable) (*durab } func (di *doltIndex) prollyRanges(ctx *sql.Context, ns tree.NodeStore, ranges ...sql.MySQLRange) ([]prolly.Range, error) { - //todo(max): it is important that *doltIndexLookup maintains a reference + // todo(max): it is important that *doltIndexLookup maintains a reference // to empty sqlRanges, otherwise the analyzer will dismiss the index and // chose a less optimal lookup index. This is a GMS concern, so GMS should // really not rely on the integrator to maintain this tenuous relationship. @@ -773,121 +720,6 @@ func (di *doltIndex) prollyRanges(ctx *sql.Context, ns tree.NodeStore, ranges .. return pranges, nil } -func (di *doltIndex) nomsRanges(ctx *sql.Context, iranges ...sql.MySQLRange) ([]*noms.ReadRange, error) { - // This might remain nil if the given nomsRanges each contain an EmptyRange for one of the columns. This will just - // cause the lookup to return no rows, which is the desired behavior. - var readRanges []*noms.ReadRange - - ranges := make([]sql.MySQLRange, len(iranges)) - - for i := range iranges { - ranges[i] = DropTrailingAllColumnExprs(iranges[i]) - } - - ranges, err := SplitNullsFromRanges(ranges) - if err != nil { - return nil, err - } - -RangeLoop: - for _, rang := range ranges { - if len(rang) > len(di.columns) { - return nil, nil - } - - var lowerKeys []interface{} - for _, rangeColumnExpr := range rang { - if rangeColumnExpr.HasLowerBound() { - lowerKeys = append(lowerKeys, sql.GetMySQLRangeCutKey(rangeColumnExpr.LowerBound)) - } else { - break - } - } - lowerboundTuple, err := di.keysToTuple(ctx, lowerKeys) - if err != nil { - return nil, err - } - - rangeCheck := make(nomsRangeCheck, len(rang)) - for i, rangeColumnExpr := range rang { - // An empty column expression will mean that no values for this column can be matched, so we can discard the - // entire range. - if ok, err := rangeColumnExpr.IsEmpty(); err != nil { - return nil, err - } else if ok { - continue RangeLoop - } - - cb := columnBounds{} - // We promote each type as the value has already been validated against the type - promotedType := di.columns[i].TypeInfo.Promote() - if rangeColumnExpr.HasLowerBound() { - key := sql.GetMySQLRangeCutKey(rangeColumnExpr.LowerBound) - val, err := promotedType.ConvertValueToNomsValue(ctx, di.vrw, key) - if err != nil { - return nil, err - } - if rangeColumnExpr.LowerBound.TypeAsLowerBound() == sql.Closed { - // For each lowerbound case, we set the upperbound to infinity, as the upperbound can increment to - // get to the desired overall case while retaining whatever was set for the lowerbound. - cb.boundsCase = boundsCase_greaterEquals_infinity - } else { - cb.boundsCase = boundsCase_greater_infinity - } - cb.lowerbound = val - } else { - cb.boundsCase = boundsCase_infinity_infinity - } - if rangeColumnExpr.HasUpperBound() { - key := sql.GetMySQLRangeCutKey(rangeColumnExpr.UpperBound) - val, err := promotedType.ConvertValueToNomsValue(ctx, di.vrw, key) - if err != nil { - return nil, err - } - if rangeColumnExpr.UpperBound.TypeAsUpperBound() == sql.Closed { - // Bounds cases are enum aliases on bytes, and they're arranged such that we can increment the case - // that was previously set when evaluating the lowerbound to get the proper overall case. - cb.boundsCase += 1 - } else { - cb.boundsCase += 2 - } - cb.upperbound = val - } - if rangeColumnExpr.Type() == sql.RangeType_EqualNull { - cb.boundsCase = boundsCase_isNull - } - rangeCheck[i] = cb - } - - // If the suffix checks will always succeed (both bounds are infinity) then they can be removed to reduce the - // number of checks that are called per-row. Always leave one check to skip NULLs. - for i := len(rangeCheck) - 1; i > 0 && len(rangeCheck) > 1; i-- { - if rangeCheck[i].boundsCase == boundsCase_infinity_infinity { - rangeCheck = rangeCheck[:i] - } else { - break - } - } - - readRanges = append(readRanges, &noms.ReadRange{ - Start: lowerboundTuple, - Inclusive: true, // The checks handle whether a value is included or not - Reverse: false, - Check: rangeCheck, - }) - } - - return readRanges, nil -} - -func (di *doltIndex) sqlRowConverter(s *durableIndexState, columns []uint64) *KVToSqlRowConverter { - return s.sqlRowConverter(di, columns) -} - -func (di *doltIndex) lookupTags(s *durableIndexState) map[uint64]int { - return s.lookupTags(di) -} - func (di *doltIndex) coversColumns(s *durableIndexState, cols []uint64) bool { if cols == nil { return s.coversAllColumns(di) @@ -1125,29 +957,6 @@ func (di *doltIndex) FullTextKeyColumns(ctx *sql.Context) (fulltext.KeyColumns, }, nil } -// keysToTuple returns a tuple that indicates the starting point for an index. The empty tuple will cause the index to -// start at the very beginning. -func (di *doltIndex) keysToTuple(ctx *sql.Context, keys []interface{}) (types.Tuple, error) { - nbf := di.vrw.Format() - if len(keys) > len(di.columns) { - return types.EmptyTuple(nbf), errors.New("too many keys for the column count") - } - - vals := make([]types.Value, len(keys)*2) - for i := range keys { - col := di.columns[i] - // As an example, if our TypeInfo is Int8, we should not fail to create a tuple if we are returning all keys - // that have a value of less than 9001, thus we promote the TypeInfo to the widest type. - val, err := col.TypeInfo.Promote().ConvertValueToNomsValue(ctx, di.vrw, keys[i]) - if err != nil { - return types.EmptyTuple(nbf), err - } - vals[2*i] = types.Uint(col.Tag) - vals[2*i+1] = val - } - return types.NewTuple(nbf, vals...) -} - var sharePool = pool.NewBuffPool() func maybeGetKeyBuilder(idx durable.Index) *val.TupleBuilder { @@ -1415,21 +1224,6 @@ func getRangeCutValue(ctx context.Context, cut sql.MySQLRangeCut, typ sql.Type) return ret, err } -// DropTrailingAllColumnExprs returns the Range with any |AllColumnExprs| at the end of it removed. -// -// Sometimes when we construct read ranges against laid out index structures, -// we want to ignore these trailing clauses. -func DropTrailingAllColumnExprs(r sql.MySQLRange) sql.MySQLRange { - i := len(r) - for i > 0 { - if r[i-1].Type() != sql.RangeType_All { - break - } - i-- - } - return r[:i] -} - // SplitNullsFromRange given a sql.Range, splits it up into multiple ranges, where each column expr // that could be NULL and non-NULL is replaced with two column expressions, one // matching only NULL, and one matching the non-NULL component. @@ -1478,19 +1272,6 @@ func SplitNullsFromRange(r sql.MySQLRange) ([]sql.MySQLRange, error) { return res, nil } -// SplitNullsFromRanges splits nulls from ranges. -func SplitNullsFromRanges(rs []sql.MySQLRange) ([]sql.MySQLRange, error) { - var ret []sql.MySQLRange - for _, r := range rs { - nr, err := SplitNullsFromRange(r) - if err != nil { - return nil, err - } - ret = append(ret, nr...) - } - return ret, nil -} - // LookupToPointSelectStr converts a set of point lookups on string // fields, returning a nil list and false if any expression failed // to convert. diff --git a/go/libraries/doltcore/sqle/index/dolt_map_iter.go b/go/libraries/doltcore/sqle/index/dolt_map_iter.go deleted file mode 100644 index e5ea730451..0000000000 --- a/go/libraries/doltcore/sqle/index/dolt_map_iter.go +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package index - -import ( - "context" - "errors" - "io" - - "github.com/dolthub/go-mysql-server/sql" - - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/types" -) - -// KVToSqlRowConverter takes noms types.Value key value pairs and converts them directly to a sql.Row. It -// can be configured to only process a portion of the columns and map columns to desired output columns. -type KVToSqlRowConverter struct { - nbf *types.NomsBinFormat - tagToSqlColIdx map[uint64]int - cols []schema.Column - - // rowSize is the number of columns in the output row. This may be bigger than the number of columns being converted, - // but not less. When rowSize is bigger than the number of columns being processed that means that some of the columns - // in the output row will be filled with nils - rowSize int - valsFromKey int - valsFromVal int - maxValTag uint64 -} - -func NewKVToSqlRowConverter(nbf *types.NomsBinFormat, tagToSqlColIdx map[uint64]int, cols []schema.Column, rowSize int) *KVToSqlRowConverter { - valsFromKey, valsFromVal, maxValTag := getValLocations(tagToSqlColIdx, cols) - - return &KVToSqlRowConverter{ - nbf: nbf, - cols: cols, - tagToSqlColIdx: tagToSqlColIdx, - rowSize: rowSize, - valsFromKey: valsFromKey, - valsFromVal: valsFromVal, - maxValTag: maxValTag, - } -} - -// get counts of where the values we want converted come from so we can skip entire tuples at times. -func getValLocations(tagToSqlColIdx map[uint64]int, cols []schema.Column) (int, int, uint64) { - var fromKey int - var fromVal int - var maxValTag uint64 - for _, col := range cols { - if _, ok := tagToSqlColIdx[col.Tag]; ok { - if col.IsPartOfPK { - fromKey++ - } else { - fromVal++ - maxValTag = max(maxValTag, col.Tag) - } - } - } - - return fromKey, fromVal, maxValTag -} - -// NewKVToSqlRowConverterForCols returns a KVToSqlConverter instance based on the list of columns passed in -func NewKVToSqlRowConverterForCols(nbf *types.NomsBinFormat, sch schema.Schema, columns []uint64) *KVToSqlRowConverter { - allCols := sch.GetAllCols().GetColumns() - tagToSqlColIdx := make(map[uint64]int) - var outCols []schema.Column - if len(columns) > 0 { - outCols = make([]schema.Column, len(columns)) - for i, tag := range columns { - schIdx := sch.GetAllCols().TagToIdx[tag] - outCols[i] = allCols[schIdx] - tagToSqlColIdx[tag] = i - } - } else { - outCols = allCols - for i, col := range allCols { - tagToSqlColIdx[col.Tag] = i - } - } - - return NewKVToSqlRowConverter(nbf, tagToSqlColIdx, outCols, len(outCols)) -} - -// ConvertKVToSqlRow returns a sql.Row generated from the key and value provided. -func (conv *KVToSqlRowConverter) ConvertKVToSqlRow(k, v types.Value) (sql.Row, error) { - keyTup, ok := k.(types.Tuple) - - if !ok { - return nil, errors.New("invalid key is not a tuple") - } - - var valTup types.Tuple - if !types.IsNull(v) { - valTup, ok = v.(types.Tuple) - - if !ok { - return nil, errors.New("invalid value is not a tuple") - } - } else { - valTup = types.EmptyTuple(conv.nbf) - } - - return conv.ConvertKVTuplesToSqlRow(keyTup, valTup) -} - -// ConvertKVTuplesToSqlRow returns a sql.Row generated from the key and value provided. -func (conv *KVToSqlRowConverter) ConvertKVTuplesToSqlRow(k, v types.Tuple) (sql.Row, error) { - tupItr := types.TupleItrPool.Get().(*types.TupleIterator) - defer types.TupleItrPool.Put(tupItr) - - cols := make([]interface{}, conv.rowSize) - if conv.valsFromKey > 0 { - // keys are not in sorted order so cannot use max tag to early exit - err := conv.processTuple(cols, conv.valsFromKey, 0xFFFFFFFFFFFFFFFF, k, tupItr) - - if err != nil { - return nil, err - } - } - - if conv.valsFromVal > 0 { - err := conv.processTuple(cols, conv.valsFromVal, conv.maxValTag, v, tupItr) - - if err != nil { - return nil, err - } - } - - return cols, nil -} - -func (conv *KVToSqlRowConverter) processTuple(cols []interface{}, valsToFill int, maxTag uint64, tup types.Tuple, tupItr *types.TupleIterator) error { - err := tupItr.InitForTuple(tup) - - if err != nil { - return err - } - - nbf := tup.Format() - primReader, numPrimitives := tupItr.CodecReader() - - filled := 0 - for pos := uint64(0); pos+1 < numPrimitives; pos += 2 { - if filled >= valsToFill { - break - } - - tagKind := primReader.ReadKind() - - if tagKind != types.UintKind { - return errors.New("Encountered unexpected kind while attempting to read tag") - } - - tag64 := primReader.ReadUint() - - if tag64 > maxTag && tag64 != schema.KeylessRowCardinalityTag && tag64 != schema.KeylessRowIdTag { - break - } - - if sqlColIdx, ok := conv.tagToSqlColIdx[tag64]; !ok { - err = primReader.SkipValue(nbf) - - if err != nil { - return err - } - } else { - cols[sqlColIdx], err = conv.cols[sqlColIdx].TypeInfo.ReadFrom(nbf, primReader) - - if err != nil { - return err - } - - filled++ - } - } - - return nil -} - -// KVGetFunc defines a function that returns a Key Value pair -type KVGetFunc func(ctx context.Context) (types.Tuple, types.Tuple, error) - -func GetGetFuncForMapIter(nbf *types.NomsBinFormat, mapItr types.MapIterator) func(ctx context.Context) (types.Tuple, types.Tuple, error) { - return func(ctx context.Context) (types.Tuple, types.Tuple, error) { - k, v, err := mapItr.Next(ctx) - - if err != nil { - return types.Tuple{}, types.Tuple{}, err - } else if k == nil { - return types.Tuple{}, types.Tuple{}, io.EOF - } - - valTup, ok := v.(types.Tuple) - if !ok { - valTup = types.EmptyTuple(nbf) - } - - return k.(types.Tuple), valTup, nil - } -} - -// DoltMapIter uses a types.MapIterator to iterate over a types.Map and returns sql.Row instances that it reads and -// converts -type DoltMapIter struct { - kvGet KVGetFunc - closeKVGetter func() error - conv *KVToSqlRowConverter -} - -// NewDoltMapIter returns a new DoltMapIter -func NewDoltMapIter(keyValGet KVGetFunc, closeKVGetter func() error, conv *KVToSqlRowConverter) *DoltMapIter { - return &DoltMapIter{ - kvGet: keyValGet, - closeKVGetter: closeKVGetter, - conv: conv, - } -} - -// Next returns the next sql.Row until all rows are returned at which point (nil, io.EOF) is returned. -func (dmi *DoltMapIter) Next(ctx *sql.Context) (sql.Row, error) { - k, v, err := dmi.kvGet(ctx) - if err != nil { - return nil, err - } - - return dmi.conv.ConvertKVTuplesToSqlRow(k, v) -} - -func (dmi *DoltMapIter) Close(*sql.Context) error { - if dmi.closeKVGetter != nil { - return dmi.closeKVGetter() - } - - return nil -} diff --git a/go/libraries/doltcore/sqle/index/index_reader.go b/go/libraries/doltcore/sqle/index/index_reader.go index 130e6485ed..3cee7e9348 100644 --- a/go/libraries/doltcore/sqle/index/index_reader.go +++ b/go/libraries/doltcore/sqle/index/index_reader.go @@ -26,7 +26,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" "github.com/dolthub/dolt/go/store/prolly" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/types" @@ -174,7 +173,6 @@ func (itr *rangePartitionIter) nextProllyPartition() (sql.Partition, error) { } type rangePartition struct { - nomsRange *noms.ReadRange key []byte prollyRange prolly.Range isReverse bool @@ -743,45 +741,6 @@ func (ib *keylessIndexImplBuilder) NewRangeMapIter(ctx context.Context, r prolly return &keylessLookupIter{pri: clustered, secIter: indexIter, pkMap: indexMap, pkBld: keyBld, prefixDesc: keyDesc}, nil } -type keylessMapIter struct { - indexIter prolly.MapIter - - // clusteredMap transforms secondary index keys - // into clustered index keys - clusteredMap val.OrdinalMapping - clusteredBld *val.TupleBuilder - clustered prolly.Map -} - -var _ prolly.MapIter = (*keylessMapIter)(nil) - -// Next implements prolly.MapIter -func (i *keylessMapIter) Next(ctx context.Context) (val.Tuple, val.Tuple, error) { - idxKey, _, err := i.indexIter.Next(ctx) - if err != nil { - return nil, nil, err - } - - for to := range i.clusteredMap { - from := i.clusteredMap.MapOrdinal(to) - i.clusteredBld.PutRaw(to, idxKey.GetField(from)) - } - pk, err := i.clusteredBld.Build(sharePool) - if err != nil { - return nil, nil, err - } - - var value val.Tuple - err = i.clustered.Get(ctx, pk, func(k, v val.Tuple) error { - value = v - return nil - }) - if err != nil { - return nil, nil, err - } - return pk, value, nil -} - // NewPartitionRowIter implements IndexScanBuilder func (ib *keylessIndexImplBuilder) NewPartitionRowIter(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) { var prollyRange prolly.Range @@ -818,183 +777,3 @@ func (ib *keylessIndexImplBuilder) NewSecondaryIter(strict bool, cnt int, nullSa prefixDesc: secondary.KeyDesc().PrefixDesc(cnt), }, nil } - -// boundsCase determines the case upon which the bounds are tested. -type boundsCase byte - -// For each boundsCase, the first element is the lowerbound and the second element is the upperbound -const ( - boundsCase_infinity_infinity boundsCase = iota - boundsCase_infinity_lessEquals - boundsCase_infinity_less - boundsCase_greaterEquals_infinity - boundsCase_greaterEquals_lessEquals - boundsCase_greaterEquals_less - boundsCase_greater_infinity - boundsCase_greater_lessEquals - boundsCase_greater_less - boundsCase_isNull -) - -// columnBounds are used to compare a given value in the noms row iterator. -type columnBounds struct { - lowerbound types.Value - upperbound types.Value - boundsCase -} - -// nomsRangeCheck is used to compare a tuple against a set of comparisons in the noms row iterator. -type nomsRangeCheck []columnBounds - -var _ noms.InRangeCheck = nomsRangeCheck{} - -// Between returns whether the given types.Value is between the bounds. In addition, this returns if the value is outside -// the bounds and above the upperbound. -func (cb columnBounds) Between(ctx context.Context, vr types.ValueReader, val types.Value) (ok bool, over bool, err error) { - // Only boundCase_isNull matches NULL values, - // otherwise we terminate the range scan. - // This is checked early to bypass unpredictable - // null type comparisons. - if val.Kind() == types.NullKind { - isNullCase := cb.boundsCase == boundsCase_isNull - return isNullCase, !isNullCase, nil - } - - switch cb.boundsCase { - case boundsCase_infinity_infinity: - return true, false, nil - case boundsCase_infinity_lessEquals: - ok, err := cb.upperbound.Less(ctx, vr.Format(), val) - if err != nil || ok { - return false, true, err - } - case boundsCase_infinity_less: - ok, err := val.Less(ctx, vr.Format(), cb.upperbound) - if err != nil || !ok { - return false, true, err - } - case boundsCase_greaterEquals_infinity: - ok, err := val.Less(ctx, vr.Format(), cb.lowerbound) - if err != nil || ok { - return false, false, err - } - case boundsCase_greaterEquals_lessEquals: - ok, err := val.Less(ctx, vr.Format(), cb.lowerbound) - if err != nil || ok { - return false, false, err - } - ok, err = cb.upperbound.Less(ctx, vr.Format(), val) - if err != nil || ok { - return false, true, err - } - case boundsCase_greaterEquals_less: - ok, err := val.Less(ctx, vr.Format(), cb.lowerbound) - if err != nil || ok { - return false, false, err - } - ok, err = val.Less(ctx, vr.Format(), cb.upperbound) - if err != nil || !ok { - return false, true, err - } - case boundsCase_greater_infinity: - ok, err := cb.lowerbound.Less(ctx, vr.Format(), val) - if err != nil || !ok { - return false, false, err - } - case boundsCase_greater_lessEquals: - ok, err := cb.lowerbound.Less(ctx, vr.Format(), val) - if err != nil || !ok { - return false, false, err - } - ok, err = cb.upperbound.Less(ctx, vr.Format(), val) - if err != nil || ok { - return false, true, err - } - case boundsCase_greater_less: - ok, err := cb.lowerbound.Less(ctx, vr.Format(), val) - if err != nil || !ok { - return false, false, err - } - ok, err = val.Less(ctx, vr.Format(), cb.upperbound) - if err != nil || !ok { - return false, true, err - } - case boundsCase_isNull: - // an isNull scan skips non-nulls, but does not terminate - return false, false, nil - default: - return false, false, fmt.Errorf("unknown bounds") - } - return true, false, nil -} - -// Equals returns whether the calling columnBounds is equivalent to the given columnBounds. -func (cb columnBounds) Equals(otherBounds columnBounds) bool { - if cb.boundsCase != otherBounds.boundsCase { - return false - } - if cb.lowerbound == nil || otherBounds.lowerbound == nil { - if cb.lowerbound != nil || otherBounds.lowerbound != nil { - return false - } - } else if !cb.lowerbound.Equals(otherBounds.lowerbound) { - return false - } - if cb.upperbound == nil || otherBounds.upperbound == nil { - if cb.upperbound != nil || otherBounds.upperbound != nil { - return false - } - } else if !cb.upperbound.Equals(otherBounds.upperbound) { - return false - } - return true -} - -// Check implements the interface noms.InRangeCheck. -func (nrc nomsRangeCheck) Check(ctx context.Context, vr types.ValueReader, tuple types.Tuple) (valid bool, skip bool, err error) { - itr := types.TupleItrPool.Get().(*types.TupleIterator) - defer types.TupleItrPool.Put(itr) - err = itr.InitForTuple(tuple) - if err != nil { - return false, false, err - } - - for i := 0; i < len(nrc) && itr.HasMore(); i++ { - if err := itr.Skip(); err != nil { - return false, false, err - } - _, val, err := itr.Next() - if err != nil { - return false, false, err - } - if val == nil { - break - } - - ok, over, err := nrc[i].Between(ctx, vr, val) - if err != nil { - return false, false, err - } - if !ok { - return i != 0 || !over, true, nil - } - } - return true, false, nil -} - -// Equals returns whether the calling nomsRangeCheck is equivalent to the given nomsRangeCheck. -func (nrc nomsRangeCheck) Equals(otherNrc nomsRangeCheck) bool { - if len(nrc) != len(otherNrc) { - return false - } - for i := range nrc { - if !nrc[i].Equals(otherNrc[i]) { - return false - } - } - return true -} - -type nomsKeyIter interface { - ReadKey(ctx context.Context) (types.Tuple, error) -} diff --git a/go/libraries/doltcore/sqle/index/testutils.go b/go/libraries/doltcore/sqle/index/testutils.go index 80a17ec63f..1bbb3e3d39 100644 --- a/go/libraries/doltcore/sqle/index/testutils.go +++ b/go/libraries/doltcore/sqle/index/testutils.go @@ -17,195 +17,9 @@ package index import ( "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" "github.com/dolthub/dolt/go/store/prolly" - "github.com/dolthub/dolt/go/store/types" ) -func ClosedRange(tpl1, tpl2 types.Tuple) *noms.ReadRange { - return CustomRange(tpl1, tpl2, sql.Closed, sql.Closed) -} - -func OpenRange(tpl1, tpl2 types.Tuple) *noms.ReadRange { - return CustomRange(tpl1, tpl2, sql.Open, sql.Open) -} - -func CustomRange(tpl1, tpl2 types.Tuple, bt1, bt2 sql.MySQLRangeBoundType) *noms.ReadRange { - var nrc nomsRangeCheck - _ = tpl1.IterFields(func(tupleIndex uint64, tupleVal types.Value) (stop bool, err error) { - if tupleIndex%2 == 0 { - return false, nil - } - if bt1 == sql.Closed { - nrc = append(nrc, columnBounds{ - boundsCase: boundsCase_greaterEquals_infinity, - lowerbound: tupleVal, - }) - } else { - nrc = append(nrc, columnBounds{ - boundsCase: boundsCase_greater_infinity, - lowerbound: tupleVal, - }) - } - return false, nil - }) - _ = tpl2.IterFields(func(tupleIndex uint64, tupleVal types.Value) (stop bool, err error) { - if tupleIndex%2 == 0 { - return false, nil - } - idx := (tupleIndex - 1) / 2 - if bt2 == sql.Closed { - // Bounds cases are enum aliases on bytes, and they're arranged such that we can increment the case - // that was previously set when evaluating the lowerbound to get the proper overall case. - nrc[idx].boundsCase += 1 - nrc[idx].upperbound = tupleVal - } else { - nrc[idx].boundsCase += 2 - nrc[idx].upperbound = tupleVal - } - return false, nil - }) - return &noms.ReadRange{ - Start: tpl1, - Inclusive: true, - Reverse: false, - Check: nrc, - } -} - -func GreaterThanRange(tpl types.Tuple) *noms.ReadRange { - var nrc nomsRangeCheck - _ = tpl.IterFields(func(tupleIndex uint64, tupleVal types.Value) (stop bool, err error) { - if tupleIndex%2 == 0 { - return false, nil - } - nrc = append(nrc, columnBounds{ - boundsCase: boundsCase_greater_infinity, - lowerbound: tupleVal, - }) - return false, nil - }) - return &noms.ReadRange{ - Start: tpl, - Inclusive: true, - Reverse: false, - Check: nrc, - } -} - -func LessThanRange(tpl types.Tuple) *noms.ReadRange { - var nrc nomsRangeCheck - _ = tpl.IterFields(func(tupleIndex uint64, tupleVal types.Value) (stop bool, err error) { - if tupleIndex%2 == 0 { - return false, nil - } - nrc = append(nrc, columnBounds{ - boundsCase: boundsCase_infinity_less, - upperbound: tupleVal, - }) - return false, nil - }) - return &noms.ReadRange{ - Start: types.EmptyTuple(types.Format_Default), - Inclusive: true, - Reverse: false, - Check: nrc, - } -} - -func GreaterOrEqualRange(tpl types.Tuple) *noms.ReadRange { - var nrc nomsRangeCheck - _ = tpl.IterFields(func(tupleIndex uint64, tupleVal types.Value) (stop bool, err error) { - if tupleIndex%2 == 0 { - return false, nil - } - nrc = append(nrc, columnBounds{ - boundsCase: boundsCase_greaterEquals_infinity, - lowerbound: tupleVal, - }) - return false, nil - }) - return &noms.ReadRange{ - Start: tpl, - Inclusive: true, - Reverse: false, - Check: nrc, - } -} - -func LessOrEqualRange(tpl types.Tuple) *noms.ReadRange { - var nrc nomsRangeCheck - _ = tpl.IterFields(func(tupleIndex uint64, tupleVal types.Value) (stop bool, err error) { - if tupleIndex%2 == 0 { - return false, nil - } - nrc = append(nrc, columnBounds{ - boundsCase: boundsCase_infinity_lessEquals, - upperbound: tupleVal, - }) - return false, nil - }) - return &noms.ReadRange{ - Start: types.EmptyTuple(types.Format_Default), - Inclusive: true, - Reverse: false, - Check: nrc, - } -} - -func NullRange() *noms.ReadRange { - return &noms.ReadRange{ - Start: types.EmptyTuple(types.Format_Default), - Inclusive: true, - Reverse: false, - Check: nomsRangeCheck{ - { - boundsCase: boundsCase_isNull, - }, - }, - } -} - -func NotNullRange() *noms.ReadRange { - return &noms.ReadRange{ - Start: types.EmptyTuple(types.Format_Default), - Inclusive: true, - Reverse: false, - Check: nomsRangeCheck{ - { - boundsCase: boundsCase_infinity_infinity, - }, - }, - } -} - -func AllRange() *noms.ReadRange { - return &noms.ReadRange{ - Start: types.EmptyTuple(types.Format_Default), - Inclusive: true, - Reverse: false, - Check: nomsRangeCheck{}, - } -} - -func ReadRangesEqual(nr1, nr2 *noms.ReadRange) bool { - if nr1 == nil || nr2 == nil { - if nr1 == nil && nr2 == nil { - return true - } - return false - } - if nr1.Inclusive != nr2.Inclusive || nr1.Reverse != nr2.Reverse || !nr1.Start.Equals(nr2.Start) || - !nr1.Check.(nomsRangeCheck).Equals(nr2.Check.(nomsRangeCheck)) { - return false - } - return true -} - -func NomsRangesFromIndexLookup(ctx *sql.Context, lookup sql.IndexLookup) ([]*noms.ReadRange, error) { - return lookup.Index.(*doltIndex).nomsRanges(ctx, lookup.Ranges.(sql.MySQLRangeCollection)...) -} - func ProllyRangesFromIndexLookup(ctx *sql.Context, lookup sql.IndexLookup) ([]prolly.Range, error) { idx := lookup.Index.(*doltIndex) return idx.prollyRanges(ctx, idx.ns, lookup.Ranges.(sql.MySQLRangeCollection)...) diff --git a/go/libraries/doltcore/sqle/kvexec/count_agg_test.go b/go/libraries/doltcore/sqle/kvexec/count_agg_test.go index 11c919c2e5..087c99d7c4 100644 --- a/go/libraries/doltcore/sqle/kvexec/count_agg_test.go +++ b/go/libraries/doltcore/sqle/kvexec/count_agg_test.go @@ -96,13 +96,7 @@ func TestCountAgg(t *testing.T) { dEnv := dtestutils.CreateTestEnv() defer dEnv.DoltDB(ctx).Close() - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := sqle.NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), opts) + db, err := sqle.NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), editor.Options{}) require.NoError(t, err) engine, sqlCtx, err := sqle.NewTestEngine(dEnv, context.Background(), db) diff --git a/go/libraries/doltcore/sqle/kvexec/lookup_join_test.go b/go/libraries/doltcore/sqle/kvexec/lookup_join_test.go index 87fb866d94..2ffc1831c7 100644 --- a/go/libraries/doltcore/sqle/kvexec/lookup_join_test.go +++ b/go/libraries/doltcore/sqle/kvexec/lookup_join_test.go @@ -164,13 +164,7 @@ func TestLookupJoin(t *testing.T) { dEnv := dtestutils.CreateTestEnv() defer dEnv.DoltDB(ctx).Close() - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := sqle.NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), opts) + db, err := sqle.NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), editor.Options{}) require.NoError(t, err) engine, sqlCtx, err := sqle.NewTestEngine(dEnv, context.Background(), db) diff --git a/go/libraries/doltcore/sqle/logictest/dolt/doltharness.go b/go/libraries/doltcore/sqle/logictest/dolt/doltharness.go index 9cfe3d84c0..5fba577815 100644 --- a/go/libraries/doltcore/sqle/logictest/dolt/doltharness.go +++ b/go/libraries/doltcore/sqle/logictest/dolt/doltharness.go @@ -301,16 +301,7 @@ func schemaToSchemaString(sch sql.Schema) (string, error) { } func sqlNewEngine(ctx context.Context, dEnv *env.DoltEnv) (*sqle.Engine, dsess.DoltDatabaseProvider, error) { - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, nil, err - } - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return nil, nil, err - } - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := dsql.NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), opts) + db, err := dsql.NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), editor.Options{}) if err != nil { return nil, nil, err } diff --git a/go/libraries/doltcore/sqle/procedures_table_test.go b/go/libraries/doltcore/sqle/procedures_table_test.go index d2bc556a73..829169bfbc 100644 --- a/go/libraries/doltcore/sqle/procedures_table_test.go +++ b/go/libraries/doltcore/sqle/procedures_table_test.go @@ -36,15 +36,9 @@ import ( func TestProceduresMigration(t *testing.T) { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - timestamp := time.Now().Truncate(time.Minute).UTC() - sqlCtx, db := newDatabaseWithProcedures(ctx, t, dEnv, opts, timestamp) + sqlCtx, db := newDatabaseWithProcedures(ctx, t, dEnv, editor.Options{}, timestamp) t.Run("test migration logic", func(t *testing.T) { // Call the logic to migrate it to the latest schema diff --git a/go/libraries/doltcore/sqle/replication.go b/go/libraries/doltcore/sqle/replication.go index 7dccacc342..101d92bba8 100644 --- a/go/libraries/doltcore/sqle/replication.go +++ b/go/libraries/doltcore/sqle/replication.go @@ -55,16 +55,7 @@ func GetCommitHooks(ctx context.Context, dEnv *env.DoltEnv, logger io.Writer) ([ // skip errors related to database construction only and return a partially functional dsqle.ReadReplicaDatabase // that will log warnings when attempting to perform replica commands. func newReplicaDatabase(ctx context.Context, name string, remoteName string, dEnv *env.DoltEnv) (ReadReplicaDatabase, error) { - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return ReadReplicaDatabase{}, err - } - - opts := editor.Options{ - Deaf: deaf, - } - - db, err := NewDatabase(ctx, name, dEnv.DbData(ctx), opts) + db, err := NewDatabase(ctx, name, dEnv.DbData(ctx), editor.Options{}) if err != nil { return ReadReplicaDatabase{}, err } diff --git a/go/libraries/doltcore/sqle/rows.go b/go/libraries/doltcore/sqle/rows.go index 477832ab01..76027c0224 100644 --- a/go/libraries/doltcore/sqle/rows.go +++ b/go/libraries/doltcore/sqle/rows.go @@ -26,36 +26,6 @@ import ( "github.com/dolthub/dolt/go/store/types" ) -var _ sql.RowIter = (*keylessRowIter)(nil) - -type keylessRowIter struct { - keyedIter *index.DoltMapIter - lastRead sql.Row - cardIdx int - nonCardCols int - lastCard uint64 -} - -func (k *keylessRowIter) Next(ctx *sql.Context) (sql.Row, error) { - if k.lastCard == 0 { - r, err := k.keyedIter.Next(ctx) - - if err != nil { - return nil, err - } - - k.lastCard = r[k.cardIdx].(uint64) - k.lastRead = r[:k.nonCardCols] - } - - k.lastCard-- - return k.lastRead, nil -} - -func (k keylessRowIter) Close(ctx *sql.Context) error { - return k.keyedIter.Close(ctx) -} - // Returns a new row iterator for the table given func newRowIterator(ctx context.Context, tbl *doltdb.Table, projCols []uint64, partition doltTablePartition) (sql.RowIter, error) { sch, err := tbl.GetSchema(ctx) diff --git a/go/libraries/doltcore/sqle/schema_table.go b/go/libraries/doltcore/sqle/schema_table.go index 4c628c623f..20cdf5480c 100644 --- a/go/libraries/doltcore/sqle/schema_table.go +++ b/go/libraries/doltcore/sqle/schema_table.go @@ -197,10 +197,6 @@ func SchemaTableSchema() schema.Schema { return schema.MustSchemaFromCols(schemasTableCols) } -func NewEmptySchemaTable() sql.Table { - return &SchemaTable{} -} - func NewSchemaTable(backingTable sql.Table) *SchemaTable { if backingTable == nil { return &SchemaTable{} diff --git a/go/libraries/doltcore/sqle/schema_table_test.go b/go/libraries/doltcore/sqle/schema_table_test.go index 74487685e7..62accc13c5 100644 --- a/go/libraries/doltcore/sqle/schema_table_test.go +++ b/go/libraries/doltcore/sqle/schema_table_test.go @@ -48,12 +48,8 @@ func unwrapRows(t *testing.T, rows []sql.Row) (unwrappedRows []sql.Row) { func TestAncientSchemaTableMigration(t *testing.T) { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), opts) + + db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), editor.Options{}) require.NoError(t, err) _, sqlCtx, err := NewTestEngine(dEnv, context.Background(), db) @@ -113,12 +109,7 @@ func TestAncientSchemaTableMigration(t *testing.T) { func TestV1SchemasTable(t *testing.T) { ctx := context.Background() dEnv := dtestutils.CreateTestEnv() - tmpDir, err := dEnv.TempTableFilesDir() - require.NoError(t, err) - deaf, err := dEnv.DbEaFactory(ctx) - require.NoError(t, err) - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), opts) + db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), editor.Options{}) require.NoError(t, err) _, sqlCtx, err := NewTestEngine(dEnv, context.Background(), db) diff --git a/go/libraries/doltcore/sqle/schema_util_test.go b/go/libraries/doltcore/sqle/schema_util_test.go index 0735135837..140b66ec7d 100644 --- a/go/libraries/doltcore/sqle/schema_util_test.go +++ b/go/libraries/doltcore/sqle/schema_util_test.go @@ -16,7 +16,6 @@ package sqle import ( "fmt" - "strconv" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" @@ -63,100 +62,6 @@ func NewResultSetRow(colVals ...types.Value) row.Row { return r } -// NewRow creates a new row with the values given, using ascending tag numbers starting at 0. -// Uses the first value as the primary key. -func NewRow(colVals ...types.Value) row.Row { - return NewRowWithPks(colVals[0:1], colVals[1:]...) -} - -// NewRowWithPks creates a new row with the values given, using ascending tag numbers starting at 0. -func NewRowWithPks(pkColVals []types.Value, nonPkVals ...types.Value) row.Row { - var cols []schema.Column - taggedVals := make(row.TaggedValues) - var tag int64 - - for _, val := range pkColVals { - var constraints []schema.ColConstraint - constraints = append(constraints, schema.NotNullConstraint{}) - cols = append(cols, schema.NewColumn(strconv.FormatInt(tag, 10), uint64(tag), val.Kind(), true, constraints...)) - taggedVals[uint64(tag)] = val - tag++ - } - - for _, val := range nonPkVals { - cols = append(cols, schema.NewColumn(strconv.FormatInt(tag, 10), uint64(tag), val.Kind(), false)) - taggedVals[uint64(tag)] = val - tag++ - } - - colColl := schema.NewColCollection(cols...) - sch := schema.MustSchemaFromCols(colColl) - - r, err := row.New(types.Format_Default, sch, taggedVals) - - if err != nil { - panic(err) - } - - return r -} - -// NewRowWithSchema creates a new row with the using the provided schema. -func NewRowWithSchema(sch schema.Schema, vals ...types.Value) row.Row { - tv := make(row.TaggedValues) - var i int - sch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) { - tv[tag] = vals[i] - i++ - return false, nil - }) - - r, err := row.New(types.Format_Default, sch, tv) - if err != nil { - panic(err) - } - - return r -} - -// NewSchema creates a new schema with the pairs of column names and types given. -// Uses the first column as the primary key. -func NewSchema(colNamesAndTypes ...interface{}) schema.Schema { - return NewSchemaForTable("", colNamesAndTypes...) -} - -// NewSchemaForTable creates a new schema for the table with the name given with the pairs of column names and types -// given. Uses the first column as the primary key. -func NewSchemaForTable(tableName string, colNamesAndTypes ...interface{}) schema.Schema { - if len(colNamesAndTypes)%2 != 0 { - panic("Non-even number of inputs passed to NewSchema") - } - - // existingTags *set.Uint64Set, tableName string, existingColKinds []types.NomsKind, newColName string, newColKind types.NomsKind - nomsKinds := make([]types.NomsKind, 0) - tags := make(schema.TagMapping) - - cols := make([]schema.Column, len(colNamesAndTypes)/2) - for i := 0; i < len(colNamesAndTypes); i += 2 { - name := colNamesAndTypes[i].(string) - nomsKind := colNamesAndTypes[i+1].(types.NomsKind) - - tag := schema.AutoGenerateTag(tags, tableName, nomsKinds, name, nomsKind) - tags.Add(tag, tableName) - nomsKinds = append(nomsKinds, nomsKind) - - isPk := i/2 == 0 - var constraints []schema.ColConstraint - if isPk { - constraints = append(constraints, schema.NotNullConstraint{}) - } - cols[i/2] = schema.NewColumn(name, tag, nomsKind, isPk, constraints...) - } - - colColl := schema.NewColCollection(cols...) - return schema.MustSchemaFromCols(colColl) -} - // Returns the logical concatenation of the schemas and rows given, rewriting all tag numbers to begin at zero. The row // returned will have a new schema identical to the result of compressSchema. func ConcatRows(schemasAndRows ...interface{}) row.Row { diff --git a/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go b/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go index eb99a9c09b..61085c38b2 100644 --- a/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go +++ b/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go @@ -199,37 +199,6 @@ func RowAsUpdateStmt(ctx *sql.Context, r row.Row, tableName string, tableSch sch return b.String(), nil } -// RowAsTupleString converts a row into it's tuple string representation for SQL insert statements. -func RowAsTupleString(ctx *sql.Context, r row.Row, tableSch schema.Schema) (string, error) { - var b strings.Builder - - b.WriteString("(") - seenOne := false - // TAGS: Use of tags here is safe since it's constrained to a single table - _, err := r.IterSchema(tableSch, func(tag uint64, val types.Value) (stop bool, err error) { - if seenOne { - b.WriteRune(',') - } - col, _ := tableSch.GetAllCols().GetByTag(tag) - sqlString, err := ValueAsSqlString(ctx, col.TypeInfo, val) - if err != nil { - return true, err - } - - b.WriteString(sqlString) - seenOne = true - return false, err - }) - - if err != nil { - return "", err - } - - b.WriteString(")") - - return b.String(), nil -} - // InsertStatementPrefix returns the first part of an SQL insert query for a given table func InsertStatementPrefix(ctx *sql.Context, tableName string, tableSch schema.Schema) (string, error) { var b strings.Builder diff --git a/go/libraries/doltcore/sqle/sqlselect_test.go b/go/libraries/doltcore/sqle/sqlselect_test.go index e352d97ad4..6b9484b86e 100644 --- a/go/libraries/doltcore/sqle/sqlselect_test.go +++ b/go/libraries/doltcore/sqle/sqlselect_test.go @@ -1386,8 +1386,6 @@ func testSelectQuery(t *testing.T, test SelectTest) { assertSchemasEqual(t, sqlSchema, sch) } -const TableWithHistoryName = "test_table" - var InitialHistSch = dtestutils.MustSchema(idColTag0TypeUUID, firstColTag1TypeStr, lastColTag2TypeStr) var AddAddrAt3HistSch = dtestutils.MustSchema(idColTag0TypeUUID, firstColTag1TypeStr, lastColTag2TypeStr, addrColTag3TypeStr) var AddAgeAt4HistSch = dtestutils.MustSchema(idColTag0TypeUUID, firstColTag1TypeStr, lastColTag2TypeStr, ageColTag4TypeInt) @@ -1421,100 +1419,6 @@ type HistoryNode struct { Children []HistoryNode } -// mustRowData converts a slice of row.TaggedValues into a noms types.Map containing that data. -func mustRowData(t *testing.T, ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema, colVals []row.TaggedValues) *types.Map { - m, err := types.NewMap(ctx, vrw) - require.NoError(t, err) - - me := m.Edit() - for _, taggedVals := range colVals { - r, err := row.New(types.Format_Default, sch, taggedVals) - require.NoError(t, err) - - me = me.Set(r.NomsMapKey(sch), r.NomsMapValue(sch)) - } - - m, err = me.Map(ctx) - require.NoError(t, err) - - return &m -} - -func CreateHistory(ctx context.Context, dEnv *env.DoltEnv, t *testing.T) []HistoryNode { - vrw := dEnv.DoltDB(ctx).ValueReadWriter() - - return []HistoryNode{ - { - Branch: "seed", - CommitMsg: "Seeding with initial user data", - Updates: map[string]TableUpdate{ - TableWithHistoryName: { - NewSch: InitialHistSch, - NewRowData: mustRowData(t, ctx, vrw, InitialHistSch, []row.TaggedValues{ - {0: types.Int(0), 1: types.String("Aaron"), 2: types.String("Son")}, - {0: types.Int(1), 1: types.String("Brian"), 2: types.String("Hendriks")}, - {0: types.Int(2), 1: types.String("Tim"), 2: types.String("Sehn")}, - }), - }, - }, - Children: []HistoryNode{ - { - Branch: "add-age", - CommitMsg: "Adding int age to users with tag 3", - Updates: map[string]TableUpdate{ - TableWithHistoryName: { - NewSch: AddAgeAt4HistSch, - NewRowData: mustRowData(t, ctx, vrw, AddAgeAt4HistSch, []row.TaggedValues{ - {0: types.Int(0), 1: types.String("Aaron"), 2: types.String("Son"), 4: types.Int(35)}, - {0: types.Int(1), 1: types.String("Brian"), 2: types.String("Hendriks"), 4: types.Int(38)}, - {0: types.Int(2), 1: types.String("Tim"), 2: types.String("Sehn"), 4: types.Int(37)}, - {0: types.Int(3), 1: types.String("Zach"), 2: types.String("Musgrave"), 4: types.Int(37)}, - }), - }, - }, - Children: nil, - }, - { - Branch: env.DefaultInitBranch, - CommitMsg: "Adding string address to users with tag 3", - Updates: map[string]TableUpdate{ - TableWithHistoryName: { - NewSch: AddAddrAt3HistSch, - NewRowData: mustRowData(t, ctx, vrw, AddAddrAt3HistSch, []row.TaggedValues{ - {0: types.Int(0), 1: types.String("Aaron"), 2: types.String("Son"), 3: types.String("123 Fake St")}, - {0: types.Int(1), 1: types.String("Brian"), 2: types.String("Hendriks"), 3: types.String("456 Bull Ln")}, - {0: types.Int(2), 1: types.String("Tim"), 2: types.String("Sehn"), 3: types.String("789 Not Real Ct")}, - {0: types.Int(3), 1: types.String("Zach"), 2: types.String("Musgrave")}, - {0: types.Int(4), 1: types.String("Matt"), 2: types.String("Jesuele")}, - }), - }, - }, - Children: []HistoryNode{ - { - Branch: env.DefaultInitBranch, - CommitMsg: "Re-add age as a uint with tag 4", - Updates: map[string]TableUpdate{ - TableWithHistoryName: { - NewSch: ReaddAgeAt5HistSch, - NewRowData: mustRowData(t, ctx, vrw, ReaddAgeAt5HistSch, []row.TaggedValues{ - {0: types.Int(0), 1: types.String("Aaron"), 2: types.String("Son"), 3: types.String("123 Fake St"), 5: types.Uint(35)}, - {0: types.Int(1), 1: types.String("Brian"), 2: types.String("Hendriks"), 3: types.String("456 Bull Ln"), 5: types.Uint(38)}, - {0: types.Int(2), 1: types.String("Tim"), 2: types.String("Sehn"), 3: types.String("789 Not Real Ct"), 5: types.Uint(37)}, - {0: types.Int(3), 1: types.String("Zach"), 2: types.String("Musgrave"), 3: types.String("-1 Imaginary Wy"), 5: types.Uint(37)}, - {0: types.Int(4), 1: types.String("Matt"), 2: types.String("Jesuele")}, - {0: types.Int(5), 1: types.String("Daylon"), 2: types.String("Wilkins")}, - }), - }, - }, - Children: nil, - }, - }, - }, - }, - }, - } -} - var idColTag0TypeUUID = schema.NewColumn("id", 0, types.IntKind, true) var firstColTag1TypeStr = schema.NewColumn("first_name", 1, types.StringKind, false) var lastColTag2TypeStr = schema.NewColumn("last_name", 2, types.StringKind, false) @@ -1534,19 +1438,6 @@ var DiffSchema = dtestutils.MustSchema( schema.NewColumn("diff_type", 14, types.StringKind, false), ) -// TODO: this shouldn't be here -func createWorkingRootUpdate() map[string]TableUpdate { - return map[string]TableUpdate{ - TableWithHistoryName: { - RowUpdates: []row.Row{ - mustRow(row.New(types.Format_Default, ReaddAgeAt5HistSch, row.TaggedValues{ - 0: types.Int(6), 1: types.String("Katie"), 2: types.String("McCulloch"), - })), - }, - }, - } -} - func validateTest(t *testing.T, test SelectTest) { if (test.ExpectedRows == nil) != (test.ExpectedSchema == nil && test.ExpectedSqlSchema == nil) { require.Fail(t, "Incorrect test setup: schema and rows must both be provided if one is") diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index 44bee564c8..5d5dc039ab 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -15,8 +15,6 @@ package sqlutil import ( - "context" - "errors" "fmt" "strings" @@ -56,173 +54,6 @@ func DoltRowToSqlRow(doltRow row.Row, sch schema.Schema) (sql.Row, error) { return sql.NewRow(colVals...), nil } -// SqlRowToDoltRow constructs a Dolt row.Row from a go-mysql-server sql.Row. -func SqlRowToDoltRow(ctx context.Context, vrw types.ValueReadWriter, r sql.Row, doltSchema schema.Schema) (row.Row, error) { - if schema.IsKeyless(doltSchema) { - return keylessDoltRowFromSqlRow(ctx, vrw, r, doltSchema) - } - return pkDoltRowFromSqlRow(ctx, vrw, r, doltSchema) -} - -// DoltKeyValueAndMappingFromSqlRow converts a sql.Row to key and value tuples and keeps a mapping from tag to value that -// can be used to speed up index key generation for foreign key checks. -func DoltKeyValueAndMappingFromSqlRow(ctx context.Context, vrw types.ValueReadWriter, r sql.Row, doltSchema schema.Schema) (types.Tuple, types.Tuple, map[uint64]types.Value, error) { - numCols := doltSchema.GetAllCols().Size() - vals := make([]types.Value, numCols*2) - tagToVal := make(map[uint64]types.Value, numCols) - - nonPKCols := doltSchema.GetNonPKCols() - numNonPKVals := nonPKCols.Size() * 2 - nonPKVals := vals[:numNonPKVals] - pkVals := vals[numNonPKVals:] - - for i, c := range doltSchema.GetAllCols().GetColumns() { - val := r[i] - if val == nil { - continue - } - - nomsVal, err := c.TypeInfo.ConvertValueToNomsValue(ctx, vrw, val) - if err != nil { - return types.Tuple{}, types.Tuple{}, nil, err - } - - tagToVal[c.Tag] = nomsVal - } - - nonPKIdx := 0 - for _, tag := range nonPKCols.SortedTags { - // nonPkCols sorted by ascending tag order - if val, ok := tagToVal[tag]; ok { - nonPKVals[nonPKIdx] = types.Uint(tag) - nonPKVals[nonPKIdx+1] = val - nonPKIdx += 2 - } - } - - pkIdx := 0 - for _, tag := range doltSchema.GetPKCols().Tags { - // pkCols are in the primary key defined order - if val, ok := tagToVal[tag]; ok { - pkVals[pkIdx] = types.Uint(tag) - pkVals[pkIdx+1] = val - pkIdx += 2 - } - } - - nonPKVals = nonPKVals[:nonPKIdx] - - nbf := vrw.Format() - keyTuple, err := types.NewTuple(nbf, pkVals...) - - if err != nil { - return types.Tuple{}, types.Tuple{}, nil, err - } - - valTuple, err := types.NewTuple(nbf, nonPKVals...) - - if err != nil { - return types.Tuple{}, types.Tuple{}, nil, err - } - - return keyTuple, valTuple, tagToVal, nil -} - -// DoltKeyAndMappingFromSqlRow converts a sql.Row to key tuple and keeps a mapping from tag to value that -// can be used to speed up index key generation for foreign key checks. -func DoltKeyAndMappingFromSqlRow(ctx context.Context, vrw types.ValueReadWriter, r sql.Row, doltSchema schema.Schema) (types.Tuple, map[uint64]types.Value, error) { - if r == nil { - return types.EmptyTuple(vrw.Format()), nil, sql.ErrUnexpectedNilRow.New() - } - - allCols := doltSchema.GetAllCols() - pkCols := doltSchema.GetPKCols() - - numCols := allCols.Size() - numPKCols := pkCols.Size() - pkVals := make([]types.Value, numPKCols*2) - tagToVal := make(map[uint64]types.Value, numCols) - - if len(r) < numCols { - numCols = len(r) - } - - for i := 0; i < numCols; i++ { - schCol := allCols.GetByIndex(i) - val := r[i] - if val == nil { - continue - } - - tag := schCol.Tag - nomsVal, err := schCol.TypeInfo.ConvertValueToNomsValue(ctx, vrw, val) - - if err != nil { - return types.Tuple{}, nil, err - } - - tagToVal[tag] = nomsVal - } - - pkOrds := doltSchema.GetPkOrdinals() - for i, pkCol := range pkCols.GetColumns() { - ord := pkOrds[i] - val := r[ord] - if val == nil { - return types.Tuple{}, nil, errors.New("not all pk columns have a value") - } - pkVals[i*2] = types.Uint(pkCol.Tag) - pkVals[i*2+1] = tagToVal[pkCol.Tag] - } - - nbf := vrw.Format() - keyTuple, err := types.NewTuple(nbf, pkVals...) - - if err != nil { - return types.Tuple{}, nil, err - } - - return keyTuple, tagToVal, nil -} - -func pkDoltRowFromSqlRow(ctx context.Context, vrw types.ValueReadWriter, r sql.Row, doltSchema schema.Schema) (row.Row, error) { - taggedVals := make(row.TaggedValues) - allCols := doltSchema.GetAllCols() - for i, val := range r { - tag := allCols.Tags[i] - schCol := allCols.TagToCol[tag] - if val != nil { - var err error - taggedVals[tag], err = schCol.TypeInfo.ConvertValueToNomsValue(ctx, vrw, val) - if err != nil { - return nil, err - } - } - } - return row.New(vrw.Format(), doltSchema, taggedVals) -} - -func keylessDoltRowFromSqlRow(ctx context.Context, vrw types.ValueReadWriter, sqlRow sql.Row, sch schema.Schema) (row.Row, error) { - j := 0 - vals := make([]types.Value, sch.GetAllCols().Size()*2) - - for idx, val := range sqlRow { - if val != nil { - col := sch.GetAllCols().GetByIndex(idx) - nv, err := col.TypeInfo.ConvertValueToNomsValue(ctx, vrw, val) - if err != nil { - return nil, err - } - - vals[j] = types.Uint(col.Tag) - vals[j+1] = nv - j += 2 - } - } - - return row.KeylessRow(vrw.Format(), vals[:j]...) -} - // BinaryAsHexDisplayValue is a wrapper for binary values that should be displayed as hex strings. type BinaryAsHexDisplayValue string diff --git a/go/libraries/doltcore/sqle/sqlutil/static_errors.go b/go/libraries/doltcore/sqle/sqlutil/static_errors.go index ca724a908b..bc1dd2dfaf 100644 --- a/go/libraries/doltcore/sqle/sqlutil/static_errors.go +++ b/go/libraries/doltcore/sqle/sqlutil/static_errors.go @@ -35,27 +35,6 @@ func (t *StaticErrorTable) LookupPartitions(_ *sql.Context, _ sql.IndexLookup) ( return nil, t.err } -func NewStaticErrorTable(orig sql.Table, err error) sql.Table { - return &StaticErrorTable{orig, err} -} - -type StaticErrorRowIter struct { - err error -} - -func NewStaticErrorRowIter(err error) sql.RowIter { - return &StaticErrorRowIter{err} -} - -func (i *StaticErrorRowIter) Next(*sql.Context) (sql.Row, error) { - return nil, i.err -} - -func (i *StaticErrorRowIter) Close(*sql.Context) error { - // Or i.err? - return nil -} - type StaticErrorEditor struct { err error } diff --git a/go/libraries/doltcore/sqle/statspro/controller.go b/go/libraries/doltcore/sqle/statspro/controller.go index bec363e485..1f4798d1af 100644 --- a/go/libraries/doltcore/sqle/statspro/controller.go +++ b/go/libraries/doltcore/sqle/statspro/controller.go @@ -611,21 +611,7 @@ func (sc *StatsController) initStorage(ctx context.Context, fs filesys.Filesys) return nil, err } - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return nil, err - } - - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, err - } - opts := editor.Options{ - Deaf: deaf, - Tempdir: tmpDir, - } - - statsDb, err := sqle.NewDatabase(ctx, "stats", dEnv.DbData(ctx), opts) + statsDb, err := sqle.NewDatabase(ctx, "stats", dEnv.DbData(ctx), editor.Options{}) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/sqle/system_variables.go b/go/libraries/doltcore/sqle/system_variables.go index 81d256f80a..7162d03084 100644 --- a/go/libraries/doltcore/sqle/system_variables.go +++ b/go/libraries/doltcore/sqle/system_variables.go @@ -22,6 +22,7 @@ import ( "github.com/dolthub/go-mysql-server/sql/types" _ "github.com/dolthub/go-mysql-server/sql/variables" + "github.com/dolthub/dolt/go/libraries/doltcore/env/actions" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess" ) @@ -292,6 +293,13 @@ var DoltSystemVariables = []sql.SystemVariable{ Type: types.NewSystemBoolType(dsess.AllowCICreation), Default: int8(0), }, + &sql.MysqlSystemVariable{ + Name: actions.DoltCommitVerificationGroups, + Dynamic: true, + Scope: sql.GetMysqlScope(sql.SystemVariableScope_Global), + Type: types.NewSystemStringType(actions.DoltCommitVerificationGroups), + Default: "", + }, } func AddDoltSystemVariables() { diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index fd6322d472..2bab3ce8fe 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -1798,9 +1798,7 @@ func (t *AlterableDoltTable) RewriteInserter( return nil, fmt.Errorf("cannot rebuild index on a headless branch") } - opts := dbState.WriteSession().GetOptions() - opts.ForeignKeyChecksDisabled = true - writeSession := writer.NewWriteSession(dt.Format(), newWs, ait, opts) + writeSession := writer.NewWriteSession(dt.Format(), newWs, ait, dbState.WriteSession().GetOptions()) ed, err := writeSession.GetTableWriter(ctx, t.TableName(), t.db.RevisionQualifiedName(), sess.SetWorkingRoot, false) if err != nil { @@ -1847,9 +1845,7 @@ func fullTextRewriteEditor( return nil, fmt.Errorf("cannot rebuild index on read only database %s", t.Name()) } - opts := dbState.WriteSession().GetOptions() - opts.ForeignKeyChecksDisabled = true - writeSession := writer.NewWriteSession(dt.Format(), newWs, ait, opts) + writeSession := writer.NewWriteSession(dt.Format(), newWs, ait, dbState.WriteSession().GetOptions()) parentEditor, err := writeSession.GetTableWriter(ctx, t.TableName(), t.db.RevisionQualifiedName(), sess.SetWorkingRoot, false) if err != nil { diff --git a/go/libraries/doltcore/sqle/temp_table.go b/go/libraries/doltcore/sqle/temp_table.go index f0a37c0287..5e5ae8aa4e 100644 --- a/go/libraries/doltcore/sqle/temp_table.go +++ b/go/libraries/doltcore/sqle/temp_table.go @@ -233,10 +233,6 @@ func (t *TempTable) Collation() sql.CollationID { return sql.CollationID(t.sch.GetCollation()) } -func (t *TempTable) sqlSchema() sql.PrimaryKeySchema { - return t.pkSch -} - func (t *TempTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { rows, err := t.table.GetRowData(ctx) if err != nil { diff --git a/go/libraries/doltcore/sqle/testdata.go b/go/libraries/doltcore/sqle/testdata.go index 993b5c4538..cc0865990b 100644 --- a/go/libraries/doltcore/sqle/testdata.go +++ b/go/libraries/doltcore/sqle/testdata.go @@ -15,15 +15,12 @@ package sqle import ( - "context" "fmt" "reflect" "time" - "github.com/dolthub/go-mysql-server/sql" "github.com/google/uuid" - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" @@ -171,22 +168,6 @@ func DatetimeStrToTimestamp(datetime string) time.Time { return time } -func newAppsRow2(charId, epId int, comment string) row.Row { - vals := row.TaggedValues{ - AppCharacterTag: types.Int(charId), - AppEpTag: types.Int(epId), - AppCommentsTag: types.String(comment), - } - - r, err := row.New(types.Format_Default, AppearancesTestSchema, vals) - - if err != nil { - panic(err) - } - - return r -} - // Most rows don't have these optional fields set, as they aren't needed for basic testing func NewPeopleRowWithOptionalFields(id int, first, last string, isMarried bool, age int, rating float64, uid uuid.UUID, numEpisodes uint64) row.Row { isMarriedVal := types.Int(0) @@ -228,23 +209,6 @@ var Ep1 = newEpsRow2(1, "Simpsons Roasting On an Open Fire", "1989-12-18 03:00:0 var Ep2 = newEpsRow2(2, "Bart the Genius", "1990-01-15 03:00:00", 9.0) var Ep3 = newEpsRow2(3, "Homer's Odyssey", "1990-01-22 03:00:00", 7.0) var Ep4 = newEpsRow2(4, "There's No Disgrace Like Home", "1990-01-29 03:00:00", 8.5) -var AllEpsRows = Rs(Ep1, Ep2, Ep3, Ep4) - -// These are made up, not the actual show data -var app1 = newAppsRow2(HomerId, 1, "Homer is great in this one") -var app2 = newAppsRow2(MargeId, 1, "Marge is here too") -var app3 = newAppsRow2(HomerId, 2, "Homer is great in this one too") -var app4 = newAppsRow2(BartId, 2, "This episode is named after Bart") -var app5 = newAppsRow2(LisaId, 2, "Lisa is here too") -var app6 = newAppsRow2(MoeId, 2, "I think there's a prank call scene") -var app7 = newAppsRow2(HomerId, 3, "Homer is in every episode") -var app8 = newAppsRow2(MargeId, 3, "Marge shows up a lot too") -var app9 = newAppsRow2(LisaId, 3, "Lisa is the best Simpson") -var app10 = newAppsRow2(BarneyId, 3, "I'm making this all up") - -// nobody in episode 4, that one was terrible -// Unlike the other tables, you can't count on the order of these rows matching the insertion order. -var AllAppsRows = Rs(app1, app2, app3, app4, app5, app6, app7, app8, app9, app10) // Convenience func to avoid the boilerplate of typing []row.Row{} all the time func Rs(rows ...row.Row) []row.Row { @@ -307,23 +271,3 @@ func MutateRow(sch schema.Schema, r row.Row, tagsAndVals ...interface{}) row.Row return mutated } - -func GetAllRows(root doltdb.RootValue, tableName string) ([]sql.Row, error) { - ctx := context.Background() - table, _, err := root.GetTable(ctx, doltdb.TableName{Name: tableName}) - if err != nil { - return nil, err - } - - rowIdx, err := table.GetRowData(ctx) - if err != nil { - return nil, err - } - - sch, err := table.GetSchema(ctx) - if err != nil { - return nil, err - } - - return SqlRowsFromDurableIndex(rowIdx, sch) -} diff --git a/go/libraries/doltcore/sqle/testutil.go b/go/libraries/doltcore/sqle/testutil.go index db90aaa51e..10fd4e08dd 100644 --- a/go/libraries/doltcore/sqle/testutil.go +++ b/go/libraries/doltcore/sqle/testutil.go @@ -46,17 +46,7 @@ import ( // ExecuteSql executes all the SQL non-select statements given in the string against the root value given and returns // the updated root, or an error. Statements in the input string are split by `;\n` func ExecuteSql(ctx context.Context, dEnv *env.DoltEnv, root doltdb.RootValue, statements string) (doltdb.RootValue, error) { - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, err - } - - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return nil, err - } - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), opts) + db, err := NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), editor.Options{}) if err != nil { return nil, err } @@ -166,17 +156,7 @@ func ExecuteSelect(ctx context.Context, dEnv *env.DoltEnv, root doltdb.RootValue Rsr: dEnv.RepoStateReader(), } - tmpDir, err := dEnv.TempTableFilesDir() - if err != nil { - return nil, err - } - - deaf, err := dEnv.DbEaFactory(ctx) - if err != nil { - return nil, err - } - opts := editor.Options{Deaf: deaf, Tempdir: tmpDir} - db, err := NewDatabase(context.Background(), "dolt", dbData, opts) + db, err := NewDatabase(context.Background(), "dolt", dbData, editor.Options{}) if err != nil { return nil, err } @@ -313,20 +293,6 @@ func SubsetSchema(sch schema.Schema, colNames ...string) schema.Schema { return schema.UnkeyedSchemaFromCols(colColl) } -// DoltSchemaFromAlterableTable is a utility for integration tests -func DoltSchemaFromAlterableTable(t *AlterableDoltTable) schema.Schema { - return t.sch -} - -// DoltTableFromAlterableTable is a utility for integration tests -func DoltTableFromAlterableTable(ctx *sql.Context, t *AlterableDoltTable) *doltdb.Table { - dt, err := t.DoltTable.DoltTable(ctx) - if err != nil { - panic(err) - } - return dt -} - func drainIter(ctx *sql.Context, iter sql.RowIter) error { for { _, err := iter.Next(ctx) diff --git a/go/libraries/doltcore/table/editor/bulk_import_tea.go b/go/libraries/doltcore/table/editor/bulk_import_tea.go deleted file mode 100644 index 74e2c7f683..0000000000 --- a/go/libraries/doltcore/table/editor/bulk_import_tea.go +++ /dev/null @@ -1,441 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package editor - -import ( - "context" - "errors" - "io" - - "github.com/dolthub/dolt/go/libraries/doltcore/remotestorage" - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/table" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" - "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/types" - "github.com/dolthub/dolt/go/store/types/edits" -) - -var _ TableEditAccumulator = (*BulkImportTEA)(nil) - -// BulkImportTEA is a TableEditAccumulator implementation used to improve the perf of bulk edits. It does not implement -// commit and rollback -type BulkImportTEA struct { - teaf DbEaFactory - capMon remotestorage.CapacityMonitor - emptyTuple types.Tuple - - vr types.ValueReader - ea types.EditAccumulator - rowData types.Map - - // opCount contains the number of edits that would be applied in materializing the edits - opCount int64 - adds map[hash.Hash]bool - deletes map[hash.Hash]bool -} - -// Delete adds a row to be deleted when these edits are eventually applied. Updates are modeled as a delete and an insert -func (tea *BulkImportTEA) Delete(keyHash hash.Hash, key types.Tuple) error { - // key is stored in the tea.ea, hash is stored in tea.deletes. Capacity is just an estimate and gets off if a - // // key is added and/or deleted more than once. - size := key.Size() + hash.ByteLen - if tea.capMon.CapacityExceeded(size) { - return errors.New("capacity exceeded") - } - - tea.opCount++ - tea.ea.AddEdit(key, nil) - - tea.deletes[keyHash] = true - delete(tea.adds, keyHash) - return nil -} - -// Insert adds a row to be inserted when these edits are eventually applied. Updates are modeled as a delete and an insert. -func (tea *BulkImportTEA) Insert(keyHash hash.Hash, key types.Tuple, val types.Tuple) error { - // key and val are stored in the tea.ea, hash is stored in tea.adds. Capacity is just an estimate and gets off if a - // key is added and/or deleted more than once. - size := key.Size() + val.Size() + hash.ByteLen - if tea.capMon.CapacityExceeded(size) { - return errors.New("capacity exceeded") - } - - tea.opCount++ - tea.ea.AddEdit(key, val) - - tea.adds[keyHash] = true - delete(tea.deletes, keyHash) - return nil -} - -// Get returns a *doltKVP if the current TableEditAccumulator contains the given key, or it exists in the row data. -// This assumes that the given hash is for the given key. -func (tea *BulkImportTEA) Get(ctx context.Context, keyHash hash.Hash, key types.Tuple) (*doltKVP, bool, error) { - if tea.deletes[keyHash] { - return nil, false, nil - } - - if tea.adds[keyHash] { - return &doltKVP{k: key, v: tea.emptyTuple}, true, nil - } - - v, ok, err := tea.rowData.MaybeGetTuple(ctx, key) - - if err != nil { - return nil, false, err - } - - if !ok { - return nil, false, nil - } - - return &doltKVP{k: key, v: v}, true, nil -} - -func (tea *BulkImportTEA) HasPartial(ctx context.Context, idxSch schema.Schema, partialKeyHash hash.Hash, partialKey types.Tuple) ([]hashedTuple, error) { - var err error - var matches []hashedTuple - var mapIter table.ReadCloser = noms.NewNomsRangeReader(tea.vr, idxSch, tea.rowData, []*noms.ReadRange{ - {Start: partialKey, Inclusive: true, Reverse: false, Check: noms.InRangeCheckPartial(partialKey)}}) - defer mapIter.Close(ctx) - var r row.Row - for r, err = mapIter.ReadRow(ctx); err == nil; r, err = mapIter.ReadRow(ctx) { - tplKeyVal, err := r.NomsMapKey(idxSch).Value(ctx) - if err != nil { - return nil, err - } - key := tplKeyVal.(types.Tuple) - tplValVal, err := r.NomsMapValue(idxSch).Value(ctx) - if err != nil { - return nil, err - } - val := tplValVal.(types.Tuple) - keyHash, err := key.Hash(key.Format()) - if err != nil { - return nil, err - } - matches = append(matches, hashedTuple{key, val, keyHash}) - } - - if err != io.EOF { - return nil, err - } - - for i := len(matches) - 1; i >= 0; i-- { - if _, ok := tea.deletes[matches[i].hash]; ok { - matches[i] = matches[len(matches)-1] - matches = matches[:len(matches)-1] - } - } - return matches, nil -} - -// Commit is the default behavior and does nothing -func (tea *BulkImportTEA) Commit(ctx context.Context, nbf *types.NomsBinFormat) error { - return nil -} - -// Rollback operation is a no-op on BulkImportTEA -func (tea *BulkImportTEA) Rollback(ctx context.Context) error { - return nil -} - -// MaterializeEdits applies the in memory edits to the row data and returns types.Map -func (tea *BulkImportTEA) MaterializeEdits(ctx context.Context, nbf *types.NomsBinFormat) (m types.Map, err error) { - ea := tea.ea - defer ea.Close(ctx) - - itr, err := ea.FinishedEditing(ctx) - if err != nil { - return types.EmptyMap, err - } - - currMap := tea.rowData - for !itr.ReachedEOF() { - currMap, _, err = types.ApplyNEdits(ctx, itr, currMap, 256*1024) - if err != nil { - return types.EmptyMap, err - } - } - - *tea = *(tea.teaf.NewTableEA(ctx, currMap).(*BulkImportTEA)) - return currMap, nil -} - -var _ IndexEditAccumulator = (*BulkImportIEA)(nil) - -// BulkImportIEA is a IndexEditAccumulator implementation used to improve the perf of bulk edits. It does not implement -// commit and rollback -type BulkImportIEA struct { - teaf DbEaFactory - capMon remotestorage.CapacityMonitor - emptyTuple types.Tuple - - vr types.ValueReader - ea types.EditAccumulator - rowData types.Map - - // opCount contains the number of edits that would be applied in materializing the edits - opCount int64 - adds map[hash.Hash]struct{} - deletes map[hash.Hash]struct{} - partialAdds map[hash.Hash]hashedTuple -} - -// Delete adds a row to be deleted when these edits are eventually applied. -func (iea *BulkImportIEA) Delete(ctx context.Context, keyHash, partialKeyHash hash.Hash, key, value types.Tuple) error { - // key is stored in iea.ea, keyHash is stored in iea.deletes. Capacity is just an estimate and gets off if a key is added and/or deleted more than once. - if iea.capMon.CapacityExceeded(key.Size()) { - return errors.New("capacity exceeded") - } - - iea.opCount++ - iea.ea.AddEdit(key, nil) - - iea.deletes[keyHash] = struct{}{} - delete(iea.adds, keyHash) - delete(iea.partialAdds, keyHash) - - return nil -} - -// Insert adds a row to be inserted when these edits are eventually applied. -func (iea *BulkImportIEA) Insert(ctx context.Context, keyHash, partialKeyHash hash.Hash, key, val types.Tuple) error { - // key and val are stored in the iea.ea, keyHash is stored in iea.adds, and iea.partialAdds. partialKeyHash is stored in iea.partialAdds[keyHash]. - // Capacity is just an estimate and gets off if a key is added and/or deleted more than once. - size := key.Size() + val.Size() + (3 * hash.ByteLen) - if iea.capMon.CapacityExceeded(size) { - return errors.New("capacity exceeded") - } - - iea.opCount++ - iea.ea.AddEdit(key, val) - - iea.adds[keyHash] = struct{}{} - delete(iea.deletes, keyHash) - - if _, ok := iea.partialAdds[partialKeyHash]; !ok { - iea.partialAdds[partialKeyHash] = hashedTuple{key, iea.emptyTuple, keyHash} - } - - return nil -} - -// Has returns true if the current TableEditAccumulator contains the given key, or it exists in the row data. -func (iea *BulkImportIEA) Has(ctx context.Context, keyHash hash.Hash, key types.Tuple) (bool, error) { - if _, ok := iea.deletes[keyHash]; ok { - return false, nil - } - - if _, ok := iea.adds[keyHash]; ok { - return true, nil - } - - ok, err := iea.rowData.Has(ctx, key) - - if err != nil { - return false, err - } else if !ok { - return false, nil - } - - return true, nil -} - -// HasPartial returns true if the current TableEditAccumulator contains the given partialKey -func (iea *BulkImportIEA) HasPartial(ctx context.Context, idxSch schema.Schema, partialKeyHash hash.Hash, partialKey types.Tuple) ([]hashedTuple, error) { - if hasNulls, err := partialKey.Contains(types.NullValue); err != nil { - return nil, err - } else if hasNulls { // rows with NULL are considered distinct, and therefore we do not match on them - return nil, nil - } - - var err error - var matches []hashedTuple - var mapIter table.ReadCloser = noms.NewNomsRangeReader(iea.vr, idxSch, iea.rowData, []*noms.ReadRange{ - {Start: partialKey, Inclusive: true, Reverse: false, Check: noms.InRangeCheckPartial(partialKey)}}) - defer mapIter.Close(ctx) - var r row.Row - for r, err = mapIter.ReadRow(ctx); err == nil; r, err = mapIter.ReadRow(ctx) { - tplKeyVal, err := r.NomsMapKey(idxSch).Value(ctx) - if err != nil { - return nil, err - } - key := tplKeyVal.(types.Tuple) - tplValVal, err := r.NomsMapValue(idxSch).Value(ctx) - if err != nil { - return nil, err - } - val := tplValVal.(types.Tuple) - keyHash, err := key.Hash(key.Format()) - if err != nil { - return nil, err - } - matches = append(matches, hashedTuple{key, val, keyHash}) - } - - if err != io.EOF { - return nil, err - } - - for i := len(matches) - 1; i >= 0; i-- { - // If we've removed a key that's present here, remove it from the slice - if _, ok := iea.deletes[matches[i].hash]; ok { - matches[i] = matches[len(matches)-1] - matches = matches[:len(matches)-1] - } - } - match, ok := iea.partialAdds[partialKeyHash] - if ok { - matches = append(matches, match) - } - return matches, nil -} - -// Commit is the default behavior and does nothing -func (iea *BulkImportIEA) Commit(ctx context.Context, nbf *types.NomsBinFormat) error { - return nil -} - -// Rollback operation no-op on BulkImportIEA -func (iea *BulkImportIEA) Rollback(ctx context.Context) error { - return nil -} - -// MaterializeEdits commits and applies the in memory edits to the row data -func (iea *BulkImportIEA) MaterializeEdits(ctx context.Context, nbf *types.NomsBinFormat) (m types.Map, err error) { - ea := iea.ea - defer ea.Close(ctx) - - itr, err := ea.FinishedEditing(ctx) - if err != nil { - return types.EmptyMap, err - } - - currMap := iea.rowData - for !itr.ReachedEOF() { - currMap, _, err = types.ApplyNEdits(ctx, itr, currMap, 256*1024) - if err != nil { - return types.EmptyMap, err - } - } - - *iea = *(iea.teaf.NewIndexEA(ctx, currMap).(*BulkImportIEA)) - return currMap, nil -} - -var _ DbEaFactory = (*BulkImportTEAFactory)(nil) - -type BulkImportTEAFactory struct { - vrw types.ValueReadWriter - directory string -} - -func NewBulkImportTEAFactory(vrw types.ValueReadWriter, directory string) *BulkImportTEAFactory { - return &BulkImportTEAFactory{ - vrw: vrw, - directory: directory, - } -} - -func (b *BulkImportTEAFactory) NewTableEA(ctx context.Context, rowData types.Map) TableEditAccumulator { - const flushInterval = 256 * 1024 - - createMapEA := func() types.EditAccumulator { - return types.CreateEditAccForMapEdits(b.vrw) - } - - ea := edits.NewDiskBackedEditAcc(ctx, b.vrw, flushInterval, b.directory, createMapEA) - return &BulkImportTEA{ - teaf: b, - capMon: remotestorage.NewUncappedCapacityMonitor(), - rowData: rowData, - ea: ea, - adds: make(map[hash.Hash]bool), - deletes: make(map[hash.Hash]bool), - emptyTuple: types.EmptyTuple(b.vrw.Format()), - } -} - -func (b *BulkImportTEAFactory) NewIndexEA(ctx context.Context, rowData types.Map) IndexEditAccumulator { - const flushInterval = 256 * 1024 - - createMapEA := func() types.EditAccumulator { - return types.CreateEditAccForMapEdits(b.vrw) - } - - ea := edits.NewDiskBackedEditAcc(ctx, b.vrw, flushInterval, b.directory, createMapEA) - return &BulkImportIEA{ - teaf: b, - capMon: remotestorage.NewUncappedCapacityMonitor(), - rowData: rowData, - ea: ea, - adds: make(map[hash.Hash]struct{}), - deletes: make(map[hash.Hash]struct{}), - partialAdds: make(map[hash.Hash]hashedTuple), - emptyTuple: types.EmptyTuple(b.vrw.Format()), - } -} - -var _ DbEaFactory = (*InMemDEAF)(nil) - -type InMemDEAF struct { - vr types.ValueReader - capMon remotestorage.CapacityMonitor -} - -func NewInMemDeafWithMaxCapacity(vr types.ValueReader, maxCapacity int64) DbEaFactory { - var capMon remotestorage.CapacityMonitor - if maxCapacity > 0 { - capMon = remotestorage.NewFixedCapacityMonitor(maxCapacity) - } else { - capMon = remotestorage.NewUncappedCapacityMonitor() - } - - return &InMemDEAF{vr: vr, capMon: capMon} -} - -func NewInMemDeaf(vr types.ValueReader) DbEaFactory { - return NewInMemDeafWithMaxCapacity(vr, -1) -} - -func (i *InMemDEAF) NewTableEA(ctx context.Context, rowData types.Map) TableEditAccumulator { - ea := edits.NewAsyncSortedEditsWithDefaults(i.vr) - return &BulkImportTEA{ - teaf: i, - capMon: i.capMon, - rowData: rowData, - ea: ea, - adds: make(map[hash.Hash]bool), - deletes: make(map[hash.Hash]bool), - emptyTuple: types.EmptyTuple(i.vr.Format()), - } -} - -func (i *InMemDEAF) NewIndexEA(ctx context.Context, rowData types.Map) IndexEditAccumulator { - ea := edits.NewAsyncSortedEditsWithDefaults(i.vr) - return &BulkImportIEA{ - teaf: i, - capMon: i.capMon, - rowData: rowData, - ea: ea, - adds: make(map[hash.Hash]struct{}), - deletes: make(map[hash.Hash]struct{}), - partialAdds: make(map[hash.Hash]hashedTuple), - emptyTuple: types.EmptyTuple(i.vr.Format()), - } -} diff --git a/go/libraries/doltcore/table/editor/editor_options.go b/go/libraries/doltcore/table/editor/editor_options.go index d352826260..d332723a5a 100644 --- a/go/libraries/doltcore/table/editor/editor_options.go +++ b/go/libraries/doltcore/table/editor/editor_options.go @@ -15,64 +15,22 @@ package editor import ( - "context" - "fmt" - "strings" - "github.com/dolthub/dolt/go/store/types" ) +const ( + invalidEaId = 0xFFFFFFFF +) + type PKDuplicateCb func(newKeyString, indexName string, existingKey, existingVal types.Tuple, isPk bool) error // Options are properties that define different functionality for the tableEditSession. +// TODO next: all these fields are write-only, remove them type Options struct { - ForeignKeyChecksDisabled bool // If true, then ALL foreign key checks AND updates (through CASCADE, etc.) are skipped - Deaf DbEaFactory - Tempdir string - // TargetStaging is true if the table is being edited in the staging root, as opposed to the working root (rare). TargetStaging bool } -// WithDeaf returns a new Options with the given edit accumulator factory class -func (o Options) WithDeaf(deaf DbEaFactory) Options { - o.Deaf = deaf - return o -} - func TestEditorOptions(vrw types.ValueReadWriter) Options { - return Options{ - ForeignKeyChecksDisabled: false, - Deaf: NewInMemDeaf(vrw), - } -} - -// formatKey returns a comma-separated string representation of the key given. -func formatKey(ctx context.Context, key types.Value) (string, error) { - tuple, ok := key.(types.Tuple) - if !ok { - return "", fmt.Errorf("Expected types.Tuple but got %T", key) - } - - var vals []string - iter, err := tuple.Iterator() - if err != nil { - return "", err - } - - for iter.HasMore() { - i, val, err := iter.Next() - if err != nil { - return "", err - } - if i%2 == 1 { - str, err := types.EncodedValue(ctx, val) - if err != nil { - return "", err - } - vals = append(vals, str) - } - } - - return fmt.Sprintf("[%s]", strings.Join(vals, ",")), nil + return Options{} } diff --git a/go/libraries/doltcore/table/editor/index_edit_accumulator.go b/go/libraries/doltcore/table/editor/index_edit_accumulator.go deleted file mode 100644 index 453102c56b..0000000000 --- a/go/libraries/doltcore/table/editor/index_edit_accumulator.go +++ /dev/null @@ -1,444 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package editor - -import ( - "context" - "io" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/table" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" - "github.com/dolthub/dolt/go/libraries/utils/set" - "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/types" - "github.com/dolthub/dolt/go/store/types/edits" -) - -// var for testing -var indexFlushThreshold int64 = 256 * 1024 - -type IndexEditAccumulator interface { - // Delete adds a row to be deleted when these edits are eventually applied. - Delete(ctx context.Context, keyHash, partialKeyHash hash.Hash, key, value types.Tuple) error - - // Insert adds a row to be inserted when these edits are eventually applied. - Insert(ctx context.Context, keyHash, partialKeyHash hash.Hash, key, value types.Tuple) error - - // Has returns true if the current TableEditAccumulator contains the given key, or it exists in the row data. - Has(ctx context.Context, keyHash hash.Hash, key types.Tuple) (bool, error) - - // HasPartial returns true if the current TableEditAccumulator contains the given partialKey - HasPartial(ctx context.Context, idxSch schema.Schema, partialKeyHash hash.Hash, partialKey types.Tuple) ([]hashedTuple, error) - - // Commit applies the in memory edits to the list of committed in memory edits - Commit(ctx context.Context, nbf *types.NomsBinFormat) error - - // Rollback rolls back in memory edits until it reaches the state represented by the savedTea - Rollback(ctx context.Context) error - - // MaterializeEdits commits and applies the in memory edits to the row data - MaterializeEdits(ctx context.Context, nbf *types.NomsBinFormat) (types.Map, error) -} - -// hashedTuple is a tuple accompanied by a hash. The representing value of the hash is dependent on the function -// it is obtained from. -type hashedTuple struct { - key types.Tuple - value types.Tuple - hash hash.Hash -} - -// inMemIndexEdits represent row adds and deletes that have not been written to the underlying storage and only exist in memory -type inMemIndexEdits struct { - // addedPartialKeys is a map of partial keys to a map of full keys that match the partial key - partialAdds map[hash.Hash]map[hash.Hash]types.Tuple - // These hashes represent the hash of the partial key, with the tuple being the full key - deletes map[hash.Hash]*hashedTuple - // These hashes represent the hash of the partial key, with the tuple being the full key - adds map[hash.Hash]*hashedTuple - ops int64 -} - -func newInMemIndexEdits() *inMemIndexEdits { - return &inMemIndexEdits{ - partialAdds: make(map[hash.Hash]map[hash.Hash]types.Tuple), - deletes: make(map[hash.Hash]*hashedTuple), - adds: make(map[hash.Hash]*hashedTuple), - } -} - -// MergeIn merges changes from another inMemIndexEdits object into this instance -func (edits *inMemIndexEdits) MergeIn(other *inMemIndexEdits) { - for keyHash, ht := range other.deletes { - delete(edits.adds, keyHash) - edits.deletes[keyHash] = ht - } - - for keyHash, ht := range other.adds { - delete(edits.deletes, keyHash) - edits.adds[keyHash] = ht - } - - for partialKeyHash, keyHashToPartialKey := range other.partialAdds { - if dest, ok := edits.partialAdds[partialKeyHash]; !ok { - edits.partialAdds[partialKeyHash] = keyHashToPartialKey - } else { - for keyHash, partialKey := range keyHashToPartialKey { - dest[keyHash] = partialKey - } - } - } - - edits.ops += other.ops -} - -// Has returns whether a key hash has been added as an insert, or a delete in this inMemIndexEdits object -func (edits *inMemIndexEdits) Has(keyHash hash.Hash) (added, deleted bool) { - if _, ok := edits.adds[keyHash]; ok { - return true, false - } - if _, ok := edits.deletes[keyHash]; ok { - return false, true - } - return false, false -} - -// indexEditAccumulatorImpl is the index equivalent of the tableEditAccumulatorImpl. -// -// indexEditAccumulatorImpl accumulates edits that need to be applied to the index row data. It needs to be able to -// support rollback and commit without having to materialize the types.Map. To do this it tracks committed and uncommitted -// modifications in memory. When a commit occurs the list of uncommitted changes are added to the list of committed changes. -// When a rollback occurs uncommitted changes are dropped. -// -// In addition to the in memory edits, the changes are applied to committedEA when a commit occurs. It is possible -// for the uncommitted changes to become so large that they need to be flushed to disk. At this point we change modes to write all edits -// to a separate map edit accumulator as they occur until the next commit occurs. -type indexEditAccumulatorImpl struct { - vr types.ValueReader - - // state of the index last time edits were applied - rowData types.Map - - // in memory changes which will be applied to the rowData when the map is materialized - committed *inMemIndexEdits - uncommitted *inMemIndexEdits - - // accumulatorIdx defines the order in which types.EditAccumulators will be applied - accumulatorIdx uint64 - - // flusher manages flushing of the types.EditAccumulators to disk when needed - flusher *edits.DiskEditFlusher - - // committedEaIds tracks ids of edit accumulators which have changes that have been committed - committedEaIds *set.Uint64Set - // uncommittedEAIds tracks ids of edit accumulators which have not been committed yet. - uncommittedEaIds *set.Uint64Set - - // commitEA is the types.EditAccumulator containing the committed changes that are being accumulated currently - commitEA types.EditAccumulator - // commitEAId is the id used for ordering the commitEA with other types.EditAccumulators that will be applied when - // materializing all changes. - commitEAId uint64 - - // flushingUncommitted is a flag that tracks whether we are in a state where we write uncommitted map edits to uncommittedEA - flushingUncommitted bool - // lastFlush is the number of uncommitted ops that had occurred at the time of the last flush - lastFlush int64 - // uncommittedEA is a types.EditAccumulator that we write to as uncommitted edits come in when the number of uncommitted - // edits becomes large - uncommittedEA types.EditAccumulator - // uncommittedEAId is the id used for ordering the uncommittedEA with other types.EditAccumulators that will be applied - // when materializing all changes - uncommittedEAId uint64 -} - -var _ IndexEditAccumulator = (*indexEditAccumulatorImpl)(nil) - -func (iea *indexEditAccumulatorImpl) flushUncommitted() { - // if we are not already actively writing edits to the uncommittedEA then change the state and push all in mem edits - // to a types.EditAccumulator - if !iea.flushingUncommitted { - iea.flushingUncommitted = true - - if iea.commitEA != nil && iea.commitEA.EditsAdded() > 0 { - // if there are uncommitted flushed changes we need to flush the committed changes first - // so they can be applied before the uncommitted flushed changes and future changes can be applied after - iea.committedEaIds.Add(iea.commitEAId) - iea.flusher.Flush(iea.commitEA, iea.commitEAId) - - iea.commitEA = nil - iea.commitEAId = invalidEaId - } - - iea.uncommittedEA = edits.NewAsyncSortedEditsWithDefaults(iea.vr) - iea.uncommittedEAId = iea.accumulatorIdx - iea.accumulatorIdx++ - - for _, ht := range iea.uncommitted.adds { - iea.uncommittedEA.AddEdit(ht.key, ht.value) - } - - for _, ht := range iea.uncommitted.deletes { - iea.uncommittedEA.AddEdit(ht.key, nil) - } - } - - // flush uncommitted - iea.lastFlush = iea.uncommitted.ops - iea.uncommittedEaIds.Add(iea.uncommittedEAId) - iea.flusher.Flush(iea.uncommittedEA, iea.uncommittedEAId) - - // initialize a new types.EditAccumulator for additional uncommitted edits to be written to. - iea.uncommittedEA = edits.NewAsyncSortedEditsWithDefaults(iea.vr) - iea.uncommittedEAId = iea.accumulatorIdx - iea.accumulatorIdx++ -} - -// Insert adds a row to be inserted when these edits are eventually applied. -func (iea *indexEditAccumulatorImpl) Insert(ctx context.Context, keyHash, partialKeyHash hash.Hash, key, value types.Tuple) error { - if _, ok := iea.uncommitted.deletes[keyHash]; ok { - delete(iea.uncommitted.deletes, keyHash) - } else { - iea.uncommitted.adds[keyHash] = &hashedTuple{key, value, partialKeyHash} - if matchingMap, ok := iea.uncommitted.partialAdds[partialKeyHash]; ok { - matchingMap[keyHash] = key - } else { - iea.uncommitted.partialAdds[partialKeyHash] = map[hash.Hash]types.Tuple{keyHash: key} - } - } - - iea.uncommitted.ops++ - if iea.flushingUncommitted { - iea.uncommittedEA.AddEdit(key, value) - - if iea.uncommitted.ops-iea.lastFlush > indexFlushThreshold { - iea.flushUncommitted() - } - } else if iea.uncommitted.ops > indexFlushThreshold { - iea.flushUncommitted() - } - return nil -} - -// Delete adds a row to be deleted when these edits are eventually applied. -func (iea *indexEditAccumulatorImpl) Delete(ctx context.Context, keyHash, partialKeyHash hash.Hash, key, value types.Tuple) error { - if _, ok := iea.uncommitted.adds[keyHash]; ok { - delete(iea.uncommitted.adds, keyHash) - delete(iea.uncommitted.partialAdds[partialKeyHash], keyHash) - } else { - iea.uncommitted.deletes[keyHash] = &hashedTuple{key, value, partialKeyHash} - } - - iea.uncommitted.ops++ - if iea.flushingUncommitted { - iea.uncommittedEA.AddEdit(key, nil) - - if iea.uncommitted.ops-iea.lastFlush > indexFlushThreshold { - iea.flushUncommitted() - } - } else if iea.uncommitted.ops > indexFlushThreshold { - iea.flushUncommitted() - } - return nil -} - -// Has returns whether the current indexEditAccumulatorImpl contains the given key. This assumes that the given hash is for -// the given key. -func (iea *indexEditAccumulatorImpl) Has(ctx context.Context, keyHash hash.Hash, key types.Tuple) (bool, error) { - // in order of most recent changes to least recent falling back to whats in the materialized row data - orderedMods := []*inMemIndexEdits{iea.uncommitted, iea.committed} - for _, mods := range orderedMods { - added, deleted := mods.Has(keyHash) - - if added { - return true, nil - } else if deleted { - return false, nil - } - } - - _, ok, err := iea.rowData.MaybeGetTuple(ctx, key) - return ok, err -} - -// HasPartial returns whether the current indexEditAccumulatorImpl contains the given partial key. This assumes that the -// given hash is for the given key. The hashes returned represent the hash of the returned tuple. -func (iea *indexEditAccumulatorImpl) HasPartial(ctx context.Context, idxSch schema.Schema, partialKeyHash hash.Hash, partialKey types.Tuple) ([]hashedTuple, error) { - if hasNulls, err := partialKey.Contains(types.NullValue); err != nil { - return nil, err - } else if hasNulls { // rows with NULL are considered distinct, and therefore we do not match on them - return nil, nil - } - - var err error - var matches []hashedTuple - var mapIter table.ReadCloser = noms.NewNomsRangeReader(iea.vr, idxSch, iea.rowData, []*noms.ReadRange{ - {Start: partialKey, Inclusive: true, Reverse: false, Check: noms.InRangeCheckPartial(partialKey)}}) - defer mapIter.Close(ctx) - var r row.Row - for r, err = mapIter.ReadRow(ctx); err == nil; r, err = mapIter.ReadRow(ctx) { - tplKeyVal, err := r.NomsMapKey(idxSch).Value(ctx) - if err != nil { - return nil, err - } - key := tplKeyVal.(types.Tuple) - tplValVal, err := r.NomsMapValue(idxSch).Value(ctx) - if err != nil { - return nil, err - } - val := tplValVal.(types.Tuple) - keyHash, err := key.Hash(key.Format()) - if err != nil { - return nil, err - } - matches = append(matches, hashedTuple{key, val, keyHash}) - } - - if err != io.EOF { - return nil, err - } - - // reapply partial key edits in order - orderedMods := []*inMemIndexEdits{iea.committed, iea.uncommitted} - for _, mods := range orderedMods { - for i := len(matches) - 1; i >= 0; i-- { - // If we've removed a key that's present here, remove it from the slice - if _, ok := mods.deletes[matches[i].hash]; ok { - matches[i] = matches[len(matches)-1] - matches = matches[:len(matches)-1] - } - } - for addedHash, addedTpl := range mods.partialAdds[partialKeyHash] { - matches = append(matches, hashedTuple{addedTpl, types.EmptyTuple(addedTpl.Format()), addedHash}) - } - } - return matches, nil -} - -// Commit applies the in memory edits to the list of committed in memory edits -func (iea *indexEditAccumulatorImpl) Commit(ctx context.Context, nbf *types.NomsBinFormat) error { - if iea.uncommitted.ops > 0 { - if !iea.flushingUncommitted { - // if there are uncommitted changes add them to the committed list of map edits - for _, ht := range iea.uncommitted.adds { - iea.commitEA.AddEdit(ht.key, ht.value) - } - - for _, ht := range iea.uncommitted.deletes { - iea.commitEA.AddEdit(ht.key, nil) - } - } else { - // if we were flushing to the uncommittedEA make the current uncommittedEA the active committedEA and add - // any uncommittedEA IDs that we already flushed - iea.commitEA = iea.uncommittedEA - iea.commitEAId = iea.uncommittedEAId - iea.committedEaIds.Add(iea.uncommittedEaIds.AsSlice()...) - - // reset state to not be flushing uncommitted - iea.uncommittedEA = nil - iea.uncommittedEAId = invalidEaId - iea.uncommittedEaIds = set.NewUint64Set(nil) - iea.lastFlush = 0 - iea.flushingUncommitted = false - } - - // apply in memory uncommitted changes to the committed in memory edits - iea.committed.MergeIn(iea.uncommitted) - - // initialize uncommitted to future in memory edits - iea.uncommitted = newInMemIndexEdits() - } - - return nil -} - -// Rollback rolls back in memory edits until it reaches the state represented by the savedTea -func (iea *indexEditAccumulatorImpl) Rollback(ctx context.Context) error { - // drop uncommitted ea IDs - iea.uncommittedEaIds = set.NewUint64Set(nil) - - if iea.uncommitted.ops > 0 { - iea.uncommitted = newInMemIndexEdits() - - if iea.flushingUncommitted { - _ = iea.uncommittedEA.Close(ctx) - iea.uncommittedEA = nil - iea.uncommittedEAId = invalidEaId - iea.uncommittedEaIds = set.NewUint64Set(nil) - iea.lastFlush = 0 - iea.flushingUncommitted = false - } - } - - return nil -} - -// MaterializeEdits applies the in memory edits to the row data and returns types.Map -func (iea *indexEditAccumulatorImpl) MaterializeEdits(ctx context.Context, nbf *types.NomsBinFormat) (m types.Map, err error) { - err = iea.Commit(ctx, nbf) - if err != nil { - return types.EmptyMap, err - } - - if iea.committed.ops == 0 { - return iea.rowData, nil - } - - committedEP, err := iea.commitEA.FinishedEditing(ctx) - iea.commitEA = nil - if err != nil { - return types.EmptyMap, err - } - - flushedEPs, err := iea.flusher.WaitForIDs(ctx, iea.committedEaIds) - if err != nil { - return types.EmptyMap, err - } - - eps := make([]types.EditProvider, 0, len(flushedEPs)+1) - for i := 0; i < len(flushedEPs); i++ { - eps = append(eps, flushedEPs[i].Edits) - } - eps = append(eps, committedEP) - - defer func() { - for _, ep := range eps { - _ = ep.Close(ctx) - } - }() - - accEdits, err := edits.NewEPMerger(ctx, iea.vr, eps) - if err != nil { - return types.EmptyMap, err - } - - // We are guaranteed that rowData is valid, as we process ieas sequentially. - updatedMap, _, err := types.ApplyEdits(ctx, accEdits, iea.rowData) - if err != nil { - return types.EmptyMap, err - } - - iea.rowData = updatedMap - iea.committed = newInMemIndexEdits() - iea.commitEAId = iea.accumulatorIdx - iea.accumulatorIdx++ - iea.commitEA = edits.NewAsyncSortedEditsWithDefaults(iea.vr) - iea.committedEaIds = set.NewUint64Set(nil) - iea.uncommittedEaIds = set.NewUint64Set(nil) - - return updatedMap, nil -} diff --git a/go/libraries/doltcore/table/editor/index_editor.go b/go/libraries/doltcore/table/editor/index_editor.go deleted file mode 100644 index 2fe907a3f8..0000000000 --- a/go/libraries/doltcore/table/editor/index_editor.go +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright 2020-2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package editor - -import ( - "context" - "fmt" - "sync" - - "github.com/dolthub/go-mysql-server/sql" - - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/types" -) - -const rebuildIndexFlushInterval = 1 << 25 - -var _ error = (*uniqueKeyErr)(nil) - -// uniqueKeyErr is an error that is returned when a unique constraint has been violated. It contains the index key -// (which is the full row). -type uniqueKeyErr struct { - TableTuple types.Tuple - IndexTuple types.Tuple - IndexName string -} - -// Error implements the error interface. -func (u *uniqueKeyErr) Error() string { - keyStr, _ := formatKey(context.Background(), u.IndexTuple) - return fmt.Sprintf("duplicate unique key given: %s", keyStr) -} - -// NOTE: Regarding partial keys and full keys. For this example, let's say that our table has a primary key W, with -// non-pk columns X, Y, and Z. You then declare an index over X and Y (in that order). In the table map containing all of -// the rows for the table, each row is composed of two tuples: the first tuple is called the key, the second tuple is -// called the value. The key is the entire primary key, which in this case is Tuple (tags are ignored for this -// example). The value is the remaining three columns: Tuple. Therefore, a row in the table map is -// Row(Tuple,Tuple). -// -// The index map containing all of the rows for the index also follows this format of key and value tuples. However, -// indexes store all of the columns in the key, and have an empty value tuple. An index key contains the indexed columns -// in the order they were defined, along with any primary keys that were not defined in the index. Thus, our example key -// looks like Tuple. We refer to this key as the full key in the index context, as with the full key you can -// construct an index row, as it's simply adding an empty tuple to the value, i.e. Row(Tuple,Tuple<>). Also with -// a full key, you can find the table row that matches this index row, as the entire primary key (just W) is in the full -// key. -// -// In both the table and index maps, keys are sorted. This means that given X and Y values for the index, we can -// construct a tuple with just those values, Tuple, and find all of the rows in the table with those two values by -// the appended primary key(s). We refer to this prefix of the full key as a partial key. It's easy to think of partial -// keys as just the indexed columns (Tuple), and the full key as the partial key along with the referring primary -// key (Tuple + W = Tuple). - -// IndexEditor takes in changes to an index map and returns the updated map if changes have been made. -// This type is thread-safe, and may be used in a multi-threaded environment. -type IndexEditor struct { - nbf *types.NomsBinFormat - - idxSch schema.Schema - tblSch schema.Schema - idx schema.Index - iea IndexEditAccumulator - stack indexOperationStack - permanentErr error // If this is set then we should always return this error as the IndexEditor is no longer usable - - // This mutex blocks on each operation, so that map reads and updates are serialized - writeMutex *sync.Mutex -} - -// NewIndexEditor creates a new index editor -func NewIndexEditor(ctx context.Context, index schema.Index, indexData types.Map, tableSch schema.Schema, opts Options) *IndexEditor { - ie := &IndexEditor{ - idxSch: index.Schema(), - tblSch: tableSch, - idx: index, - iea: opts.Deaf.NewIndexEA(ctx, indexData), - nbf: indexData.Format(), - permanentErr: nil, - writeMutex: &sync.Mutex{}, - } - return ie -} - -// InsertRow adds the given row to the index. If the row already exists and the index is unique, then an error is returned. -// Otherwise, it is a no-op. -func (ie *IndexEditor) InsertRow(ctx context.Context, key, partialKey types.Tuple, value types.Tuple) error { - return ie.InsertRowWithDupCb(ctx, key, partialKey, value, func(ctx context.Context, uke *uniqueKeyErr) error { - msg, err := formatKey(context.Background(), uke.IndexTuple) - if err != nil { - return err - } - // The only secondary index that can throw unique key errors is a unique index - return sql.NewUniqueKeyErr(msg, !ie.idx.IsUnique(), nil) - }) -} - -// InsertRowWithDupCb adds the given row to the index. If the row already exists and the -// index is unique, then a uniqueKeyErr is passed to |cb|. If |cb| returns a non-nil -// error then the insert is aborted. Otherwise, the insert proceeds. -func (ie *IndexEditor) InsertRowWithDupCb(ctx context.Context, key, partialKey types.Tuple, value types.Tuple, cb func(ctx context.Context, uke *uniqueKeyErr) error) error { - keyHash, err := key.Hash(key.Format()) - if err != nil { - return err - } - partialKeyHash, err := partialKey.Hash(partialKey.Format()) - if err != nil { - return err - } - - ie.writeMutex.Lock() - defer ie.writeMutex.Unlock() - - if ie.permanentErr != nil { - return ie.permanentErr - } - - if ie.idx.IsUnique() { - if matches, err := ie.iea.HasPartial(ctx, ie.idxSch, partialKeyHash, partialKey); err != nil { - return err - } else if len(matches) > 0 { - tableTuple, err := ie.idx.ToTableTuple(ctx, matches[0].key, ie.nbf) - if err != nil { - return err - } - cause := &uniqueKeyErr{tableTuple, partialKey, ie.idx.Name()} - err = cb(ctx, cause) - if err != nil { - return err - } - } - } else { - if rowExists, err := ie.iea.Has(ctx, keyHash, key); err != nil { - return err - } else if rowExists && value.Empty() { - ie.stack.Push(true, types.EmptyTuple(key.Format()), types.EmptyTuple(key.Format()), types.EmptyTuple(value.Format())) - return nil - } - } - - err = ie.iea.Insert(ctx, keyHash, partialKeyHash, key, value) - if err != nil { - return err - } - - ie.stack.Push(true, key, partialKey, value) - return nil -} - -// DeleteRow removes the given row from the index. -func (ie *IndexEditor) DeleteRow(ctx context.Context, key, partialKey, value types.Tuple) error { - keyHash, err := key.Hash(ie.nbf) - if err != nil { - return err - } - partialKeyHash, err := partialKey.Hash(partialKey.Format()) - if err != nil { - return err - } - - ie.writeMutex.Lock() - defer ie.writeMutex.Unlock() - - if ie.permanentErr != nil { - return ie.permanentErr - } - - err = ie.iea.Delete(ctx, keyHash, partialKeyHash, key, value) - if err != nil { - return err - } - - ie.stack.Push(false, key, partialKey, value) - return nil -} - -// HasPartial returns whether the index editor has the given partial key. -func (ie *IndexEditor) HasPartial(ctx context.Context, partialKey types.Tuple) (bool, error) { - partialKeyHash, err := partialKey.Hash(partialKey.Format()) - if err != nil { - return false, err - } - - ie.writeMutex.Lock() - defer ie.writeMutex.Unlock() - - if ie.permanentErr != nil { - return false, ie.permanentErr - } - - tpls, err := ie.iea.HasPartial(ctx, ie.idxSch, partialKeyHash, partialKey) - if err != nil { - return false, err - } - return len(tpls) > 0, nil -} - -// Undo will cause the index editor to undo the last operation at the top of the stack. As Insert and Delete are called, -// they are added onto a limited-size stack, and Undo pops an operation off the top and undoes it. So if there was an -// Insert on a key, it will use Delete on that same key. The stack size is very small, therefore too many consecutive -// calls will cause the stack to empty. This should only be called in the event that an operation was performed that -// has failed for other reasons, such as an INSERT on the parent table failing on a separate index editor. In the event -// that Undo is called and there are no operations to undo OR the reverse operation fails (such as memory capacity -// reached), then we set a permanent error as the index editor is in an invalid state that cannot be corrected. -// -// We don't return an error here as Undo will only be called when there's an error in a different editor. We allow the -// user to handle that initial error, as ALL further calls to this IndexEditor will return the error set here. -func (ie *IndexEditor) Undo(ctx context.Context) { - if ie.permanentErr != nil { - return - } - - indexOp, ok := ie.stack.Pop() - if !ok { - panic(fmt.Sprintf("attempted to undo the last operation on index '%s' but failed due to an empty stack", ie.idx.Name())) - } - // If an operation succeeds and does not do anything, then an empty tuple is pushed onto the stack. - if indexOp.fullKey.Empty() { - return - } - - if indexOp.isInsert { - err := ie.DeleteRow(ctx, indexOp.fullKey, indexOp.partialKey, indexOp.value) - if err != nil { - ie.permanentErr = fmt.Errorf("index '%s' is in an invalid and unrecoverable state: "+ - "attempted to undo previous insertion but encountered the following error: %v", - ie.idx.Name(), err) - return - } - } else { - err := ie.InsertRow(ctx, indexOp.fullKey, indexOp.partialKey, indexOp.value) - if err != nil { - ie.permanentErr = fmt.Errorf("index '%s' is in an invalid and unrecoverable state: "+ - "attempted to undo previous deletion but encountered the following error: %v", - ie.idx.Name(), err) - return - } - } -} - -// Map returns a map based on the edits given, if any. -func (ie *IndexEditor) Map(ctx context.Context) (types.Map, error) { - ie.writeMutex.Lock() - defer ie.writeMutex.Unlock() - - if ie.permanentErr != nil { - return types.EmptyMap, ie.permanentErr - } - - return ie.iea.MaterializeEdits(ctx, ie.nbf) -} - -// Index returns this editor's index. -func (ie *IndexEditor) Index() schema.Index { - return ie.idx -} - -// StatementStarted is analogous to the TableEditor implementation, but specific to the IndexEditor. -func (ie *IndexEditor) StatementStarted(ctx context.Context) { -} - -// StatementFinished is analogous to the TableEditor implementation, but specific to the IndexEditor. -func (ie *IndexEditor) StatementFinished(ctx context.Context, errored bool) error { - ie.writeMutex.Lock() - defer ie.writeMutex.Unlock() - - if ie.permanentErr != nil { - return ie.permanentErr - } else if errored { - return ie.iea.Rollback(ctx) - } - - return ie.iea.Commit(ctx, ie.nbf) -} - -// Close is a no-op for an IndexEditor. -func (ie *IndexEditor) Close() error { - return ie.permanentErr -} diff --git a/go/libraries/doltcore/table/editor/index_editor_test.go b/go/libraries/doltcore/table/editor/index_editor_test.go deleted file mode 100644 index c4939ca45c..0000000000 --- a/go/libraries/doltcore/table/editor/index_editor_test.go +++ /dev/null @@ -1,435 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package editor - -import ( - "context" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/dolthub/dolt/go/libraries/doltcore/dbfactory" - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/types" -) - -const ( - // The number of rows we expect the test to end up with - indexEditorConcurrencyFinalCount = 100 -) - -func TestIndexEditorConcurrency(t *testing.T) { - format := types.Format_LD_1 - _, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil) - require.Equal(t, format, vrw.Format()) - require.NoError(t, err) - colColl := schema.NewColCollection( - schema.NewColumn("pk", 0, types.IntKind, true), - schema.NewColumn("v1", 1, types.IntKind, false), - schema.NewColumn("v2", 2, types.IntKind, false)) - tableSch, err := schema.SchemaFromCols(colColl) - require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_concurrency", []string{"v1"}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) - require.NoError(t, err) - indexSch := index.Schema() - emptyMap, err := types.NewMap(context.Background(), vrw) - require.NoError(t, err) - - opts := TestEditorOptions(vrw) - indexEditor := NewIndexEditor(context.Background(), index, emptyMap, tableSch, opts) - wg := &sync.WaitGroup{} - - for j := 0; j < indexEditorConcurrencyFinalCount*2; j++ { - wg.Add(1) - go func(val int) { - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(val), - 1: types.Int(val), - }) - require.NoError(t, err) - fullKey, partialKey, value, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.InsertRow(context.Background(), fullKey, partialKey, value)) - wg.Done() - }(j) - } - wg.Wait() - - for j := 0; j < indexEditorConcurrencyFinalCount; j++ { - wg.Add(1) - go func(val int) { - dOldRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(val), - 1: types.Int(val), - }) - require.NoError(t, err) - dNewRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(val), - 1: types.Int(val + 1), - }) - require.NoError(t, err) - oldFullKey, oldPartialKey, _, err := dOldRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.DeleteRow(context.Background(), oldFullKey, oldPartialKey, types.EmptyTuple(format))) - newFullKey, newPartialKey, newValue, err := dNewRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.InsertRow(context.Background(), newFullKey, newPartialKey, newValue)) - wg.Done() - }(j) - } - - // We let the Updates and Deletes execute at the same time - for j := indexEditorConcurrencyFinalCount; j < indexEditorConcurrencyFinalCount*2; j++ { - wg.Add(1) - go func(val int) { - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(val), - 1: types.Int(val), - }) - require.NoError(t, err) - fullKey, partialKey, _, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.DeleteRow(context.Background(), fullKey, partialKey, types.EmptyTuple(format))) - wg.Done() - }(j) - } - wg.Wait() - - newIndexData, err := indexEditor.Map(context.Background()) - require.NoError(t, err) - if assert.Equal(t, uint64(indexEditorConcurrencyFinalCount), newIndexData.Len()) { - iterIndex := 0 - _ = newIndexData.IterAll(context.Background(), func(key, value types.Value) error { - dReadRow, err := row.FromNoms(indexSch, key.(types.Tuple), value.(types.Tuple)) - require.NoError(t, err) - dReadVals, err := dReadRow.TaggedValues() - require.NoError(t, err) - assert.Equal(t, row.TaggedValues{ - 0: types.Int(iterIndex), - 1: types.Int(iterIndex + 1), - }, dReadVals) - iterIndex++ - return nil - }) - } -} - -func TestIndexEditorConcurrencyPostInsert(t *testing.T) { - format := types.Format_LD_1 - _, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil) - require.Equal(t, format, vrw.Format()) - require.NoError(t, err) - colColl := schema.NewColCollection( - schema.NewColumn("pk", 0, types.IntKind, true), - schema.NewColumn("v1", 1, types.IntKind, false), - schema.NewColumn("v2", 2, types.IntKind, false)) - tableSch, err := schema.SchemaFromCols(colColl) - require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_concurrency", []string{"v1"}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) - require.NoError(t, err) - indexSch := index.Schema() - emptyMap, err := types.NewMap(context.Background(), vrw) - require.NoError(t, err) - - opts := TestEditorOptions(vrw) - indexEditor := NewIndexEditor(context.Background(), index, emptyMap, tableSch, opts) - for i := 0; i < indexEditorConcurrencyFinalCount*2; i++ { - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(i), - 1: types.Int(i), - }) - require.NoError(t, err) - fullKey, partialKey, value, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.InsertRow(context.Background(), fullKey, partialKey, value)) - } - indexData, err := indexEditor.Map(context.Background()) - require.NoError(t, err) - - indexEditor = NewIndexEditor(context.Background(), index, indexData, tableSch, opts) - wg := &sync.WaitGroup{} - - for j := 0; j < indexEditorConcurrencyFinalCount; j++ { - wg.Add(1) - go func(val int) { - dOldRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(val), - 1: types.Int(val), - }) - require.NoError(t, err) - dNewRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(val), - 1: types.Int(val + 1), - }) - require.NoError(t, err) - oldFullKey, oldPartialKey, _, err := dOldRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.DeleteRow(context.Background(), oldFullKey, oldPartialKey, types.EmptyTuple(format))) - newFullKey, newPartialKey, value, err := dNewRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.InsertRow(context.Background(), newFullKey, newPartialKey, value)) - wg.Done() - }(j) - } - - for j := indexEditorConcurrencyFinalCount; j < indexEditorConcurrencyFinalCount*2; j++ { - wg.Add(1) - go func(val int) { - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(val), - 1: types.Int(val), - }) - require.NoError(t, err) - fullKey, partialKey, _, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.DeleteRow(context.Background(), fullKey, partialKey, types.EmptyTuple(format))) - wg.Done() - }(j) - } - wg.Wait() - - newIndexData, err := indexEditor.Map(context.Background()) - require.NoError(t, err) - if assert.Equal(t, uint64(indexEditorConcurrencyFinalCount), newIndexData.Len()) { - iterIndex := 0 - _ = newIndexData.IterAll(context.Background(), func(key, value types.Value) error { - dReadRow, err := row.FromNoms(indexSch, key.(types.Tuple), value.(types.Tuple)) - require.NoError(t, err) - dReadVals, err := dReadRow.TaggedValues() - require.NoError(t, err) - assert.Equal(t, row.TaggedValues{ - 0: types.Int(iterIndex), - 1: types.Int(iterIndex + 1), - }, dReadVals) - iterIndex++ - return nil - }) - } -} - -func TestIndexEditorUniqueMultipleNil(t *testing.T) { - format := types.Format_LD_1 - _, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil) - require.Equal(t, format, vrw.Format()) - require.NoError(t, err) - colColl := schema.NewColCollection( - schema.NewColumn("pk", 0, types.IntKind, true), - schema.NewColumn("v1", 1, types.IntKind, false)) - tableSch, err := schema.SchemaFromCols(colColl) - require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_unique", []string{"v1"}, nil, schema.IndexProperties{IsUnique: true, Comment: ""}) - require.NoError(t, err) - indexSch := index.Schema() - emptyMap, err := types.NewMap(context.Background(), vrw) - require.NoError(t, err) - - opts := TestEditorOptions(vrw) - indexEditor := NewIndexEditor(context.Background(), index, emptyMap, tableSch, opts) - for i := 0; i < 3; i++ { - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.NullValue, - 1: types.Int(i), - }) - require.NoError(t, err) - fullKey, partialKey, value, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.InsertRow(context.Background(), fullKey, partialKey, value)) - } - newIndexData, err := indexEditor.Map(context.Background()) - require.NoError(t, err) - if assert.Equal(t, uint64(3), newIndexData.Len()) { - index := 0 - _ = newIndexData.IterAll(context.Background(), func(key, value types.Value) error { - dReadRow, err := row.FromNoms(indexSch, key.(types.Tuple), value.(types.Tuple)) - require.NoError(t, err) - dReadVals, err := dReadRow.TaggedValues() - require.NoError(t, err) - assert.Equal(t, row.TaggedValues{ - 1: types.Int(index), // We don't encode NULL values - }, dReadVals) - index++ - return nil - }) - } -} - -func TestIndexEditorWriteAfterFlush(t *testing.T) { - format := types.Format_LD_1 - _, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil) - require.Equal(t, format, vrw.Format()) - require.NoError(t, err) - colColl := schema.NewColCollection( - schema.NewColumn("pk", 0, types.IntKind, true), - schema.NewColumn("v1", 1, types.IntKind, false), - schema.NewColumn("v2", 2, types.IntKind, false)) - tableSch, err := schema.SchemaFromCols(colColl) - require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_concurrency", []string{"v1"}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) - require.NoError(t, err) - indexSch := index.Schema() - emptyMap, err := types.NewMap(context.Background(), vrw) - require.NoError(t, err) - - opts := TestEditorOptions(vrw) - indexEditor := NewIndexEditor(context.Background(), index, emptyMap, tableSch, opts) - require.NoError(t, err) - - for i := 0; i < 20; i++ { - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(i), - 1: types.Int(i), - }) - require.NoError(t, err) - fullKey, partialKey, value, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.InsertRow(context.Background(), fullKey, partialKey, value)) - } - - _, err = indexEditor.Map(context.Background()) - require.NoError(t, err) - - for i := 10; i < 20; i++ { - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(i), - 1: types.Int(i), - }) - require.NoError(t, err) - fullKey, partialKey, _, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.DeleteRow(context.Background(), fullKey, partialKey, types.EmptyTuple(format))) - } - - newIndexData, err := indexEditor.Map(context.Background()) - require.NoError(t, err) - if assert.Equal(t, uint64(10), newIndexData.Len()) { - iterIndex := 0 - _ = newIndexData.IterAll(context.Background(), func(key, value types.Value) error { - dReadRow, err := row.FromNoms(indexSch, key.(types.Tuple), value.(types.Tuple)) - require.NoError(t, err) - dReadVals, err := dReadRow.TaggedValues() - require.NoError(t, err) - assert.Equal(t, row.TaggedValues{ - 0: types.Int(iterIndex), - 1: types.Int(iterIndex), - }, dReadVals) - iterIndex++ - return nil - }) - } - - sameIndexData, err := indexEditor.Map(context.Background()) - require.NoError(t, err) - assert.True(t, sameIndexData.Equals(newIndexData)) -} - -func TestIndexEditorUniqueErrorDoesntPersist(t *testing.T) { - format := types.Format_LD_1 - _, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil) - require.Equal(t, format, vrw.Format()) - require.NoError(t, err) - colColl := schema.NewColCollection( - schema.NewColumn("pk", 0, types.IntKind, true), - schema.NewColumn("v1", 1, types.IntKind, false)) - tableSch, err := schema.SchemaFromCols(colColl) - require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_unq", []string{"v1"}, nil, schema.IndexProperties{IsUnique: true, Comment: ""}) - require.NoError(t, err) - indexSch := index.Schema() - emptyMap, err := types.NewMap(context.Background(), vrw) - require.NoError(t, err) - - opts := TestEditorOptions(vrw) - indexEditor := NewIndexEditor(context.Background(), index, emptyMap, tableSch, opts) - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(1), - 1: types.Int(1), - }) - require.NoError(t, err) - fullKey, partialKey, value, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.InsertRow(context.Background(), fullKey, partialKey, value)) - dRow, err = row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(2), - 1: types.Int(1), - }) - require.NoError(t, err) - fullKey, partialKey, value, err = dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.Error(t, indexEditor.InsertRow(context.Background(), fullKey, partialKey, value)) - dRow, err = row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(2), - 1: types.Int(2), - }) - require.NoError(t, err) - fullKey, partialKey, value, err = dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.InsertRow(context.Background(), fullKey, partialKey, value)) -} - -func TestIndexEditorCapacityExceeded(t *testing.T) { - // In the event that we reach the iea capacity on Undo, we need to verify that all code paths fail and remain failing - ctx := context.Background() - format := types.Format_LD_1 - _, vrw, _, err := dbfactory.MemFactory{}.CreateDB(ctx, format, nil, nil) - require.Equal(t, format, vrw.Format()) - require.NoError(t, err) - colColl := schema.NewColCollection( - schema.NewColumn("pk", 0, types.IntKind, true), - schema.NewColumn("v1", 1, types.IntKind, false)) - tableSch, err := schema.SchemaFromCols(colColl) - require.NoError(t, err) - index, err := tableSch.Indexes().AddIndexByColNames("idx_cap", []string{"v1"}, nil, schema.IndexProperties{IsUnique: false, Comment: ""}) - require.NoError(t, err) - indexSch := index.Schema() - emptyMap, err := types.NewMap(ctx, vrw) - require.NoError(t, err) - - opts := Options{Deaf: NewInMemDeafWithMaxCapacity(vrw, 224)} - indexEditor := NewIndexEditor(ctx, index, emptyMap, tableSch, opts) - for i := 0; i < 3; i++ { - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(i), - 1: types.Int(i), - }) - require.NoError(t, err) - fullKey, partialKey, value, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - require.NoError(t, indexEditor.InsertRow(ctx, fullKey, partialKey, value)) - } - - dRow, err := row.New(format, indexSch, row.TaggedValues{ - 0: types.Int(4), - 1: types.Int(4), - }) - require.NoError(t, err) - fullKey, partialKey, value, err := dRow.ReduceToIndexKeys(index, nil) - require.NoError(t, err) - err = indexEditor.InsertRow(ctx, fullKey, partialKey, value) - require.Error(t, err) - require.Equal(t, "capacity exceeded", err.Error()) - indexEditor.Undo(ctx) // This sets the unrecoverable state error, but does not return an error itself - - require.Contains(t, indexEditor.InsertRow(ctx, fullKey, partialKey, value).Error(), "unrecoverable state") - require.Contains(t, indexEditor.DeleteRow(ctx, fullKey, partialKey, value).Error(), "unrecoverable state") - require.Contains(t, indexEditor.StatementFinished(ctx, false).Error(), "unrecoverable state") - require.Contains(t, indexEditor.Close().Error(), "unrecoverable state") - _, err = indexEditor.HasPartial(ctx, partialKey) - require.Contains(t, err.Error(), "unrecoverable state") - _, err = indexEditor.Map(ctx) - require.Contains(t, err.Error(), "unrecoverable state") -} diff --git a/go/libraries/doltcore/table/editor/index_operation_stack.go b/go/libraries/doltcore/table/editor/index_operation_stack.go deleted file mode 100644 index 5e942251c0..0000000000 --- a/go/libraries/doltcore/table/editor/index_operation_stack.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package editor - -import "github.com/dolthub/dolt/go/store/types" - -// indexOperationStack is a limited-size stack, intended for usage with the index editor and its undo functionality. -// As operations are added, the internal array is filled up. Once it is full, new operations replace the oldest ones. -// This reduces memory usage compared to a traditional stack with an unbounded size, as undo should always come -// immediately after an operation is added. -type indexOperationStack struct { - // entries has a length of 4 as an UPDATE on a table is a Delete & Insert on the index, so we double it for safety. - entries [4]indexOperation - // This is the index of the next item we are adding. Add at this index, then increment. - currentIndex uint64 - // Represents the number of items relative to the "stack size". - numOfItems uint64 -} - -// indexOperation is an operation performed by the index editor, along with the key used. -type indexOperation struct { - isInsert bool - fullKey types.Tuple - partialKey types.Tuple - value types.Tuple -} - -// Push adds the given keys to the top of the stack. -func (ios *indexOperationStack) Push(isInsert bool, fullKey, partialKey, value types.Tuple) { - ios.entries[ios.currentIndex].isInsert = isInsert - ios.entries[ios.currentIndex].fullKey = fullKey - ios.entries[ios.currentIndex].partialKey = partialKey - ios.entries[ios.currentIndex].value = value - ios.currentIndex = (ios.currentIndex + 1) % uint64(len(ios.entries)) - ios.numOfItems++ - if ios.numOfItems > uint64(len(ios.entries)) { - ios.numOfItems = uint64(len(ios.entries)) - } -} - -// Pop removes and returns the keys from the top of the stack. Returns false if the stack is empty. -func (ios *indexOperationStack) Pop() (indexOperation, bool) { - if ios.numOfItems == 0 { - return indexOperation{}, false - } - ios.numOfItems-- - ios.currentIndex = (ios.currentIndex - 1) % uint64(len(ios.entries)) - return ios.entries[ios.currentIndex], true -} diff --git a/go/libraries/doltcore/table/editor/index_operation_stack_test.go b/go/libraries/doltcore/table/editor/index_operation_stack_test.go deleted file mode 100644 index 7810240c9d..0000000000 --- a/go/libraries/doltcore/table/editor/index_operation_stack_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package editor - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/dolthub/dolt/go/store/types" -) - -func TestIndexOperationStack(t *testing.T) { - ios := &indexOperationStack{} - require.True(t, len(ios.entries) >= 2) // Entries should always at least have a length of 2 - - ios.Push(true, iosTuple(t, 100, 100), iosTuple(t, 100), iosTuple(t, 0)) - entry, ok := ios.Pop() - require.True(t, ok) - iosTupleComp(t, entry.fullKey, 100, 100) - iosTupleComp(t, entry.partialKey, 100) - iosTupleComp(t, entry.value, 0) - require.True(t, entry.isInsert) - _, ok = ios.Pop() - require.False(t, ok) - - for i := 0; i < len(ios.entries); i++ { - ios.Push(false, iosTuple(t, i, i), iosTuple(t, i), iosTuple(t, i*2)) - } - for i := len(ios.entries) - 1; i >= 0; i-- { - entry, ok = ios.Pop() - require.True(t, ok) - iosTupleComp(t, entry.fullKey, i, i) - iosTupleComp(t, entry.partialKey, i) - iosTupleComp(t, entry.partialKey, i*2) - require.False(t, entry.isInsert) - } - _, ok = ios.Pop() - require.False(t, ok) - - for i := 0; i < (len(ios.entries)*2)+1; i++ { - ios.Push(true, iosTuple(t, i, i), iosTuple(t, i), iosTuple(t, i*2)) - } - for i := len(ios.entries) - 1; i >= 0; i-- { - entry, ok = ios.Pop() - require.True(t, ok) - val := ((len(ios.entries) * 2) + 1) - i - iosTupleComp(t, entry.fullKey, val, val) - iosTupleComp(t, entry.partialKey, val) - iosTupleComp(t, entry.value, val*2) - require.True(t, entry.isInsert) - } - _, ok = ios.Pop() - require.False(t, ok) -} - -func iosTuple(t *testing.T, vals ...int) types.Tuple { - typeVals := make([]types.Value, len(vals)) - for i, val := range vals { - typeVals[i] = types.Int(val) - } - tpl, err := types.NewTuple(types.Format_Default, typeVals...) - if err != nil { - require.NoError(t, err) - } - return tpl -} - -func iosTupleComp(t *testing.T, tpl types.Tuple, vals ...int) bool { - if tpl.Len() != uint64(len(vals)) { - return false - } - iter, err := tpl.Iterator() - require.NoError(t, err) - var i uint64 - var val types.Value - for i, val, err = iter.Next(); i < uint64(len(vals)) && err == nil; i, val, err = iter.Next() { - if !types.Int(vals[i]).Equals(val) { - return false - } - } - require.NoError(t, err) - return true -} diff --git a/go/libraries/doltcore/table/editor/table_edit_accumulator.go b/go/libraries/doltcore/table/editor/table_edit_accumulator.go deleted file mode 100644 index 72365ebf61..0000000000 --- a/go/libraries/doltcore/table/editor/table_edit_accumulator.go +++ /dev/null @@ -1,485 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package editor - -import ( - "context" - "io" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/table" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" - "github.com/dolthub/dolt/go/libraries/utils/set" - "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/types" - "github.com/dolthub/dolt/go/store/types/edits" -) - -const ( - invalidEaId = 0xFFFFFFFF -) - -type doltKVP struct { - k types.Tuple - v types.Tuple -} - -type TableEditAccumulator interface { - // Delete adds a row to be deleted when these edits are eventually applied. Updates are modeled as a delete and an insert - Delete(keyHash hash.Hash, key types.Tuple) error - - // Insert adds a row to be inserted when these edits are eventually applied. Updates are modeled as a delete and an insert. - Insert(keyHash hash.Hash, key types.Tuple, val types.Tuple) error - - // Get returns a *doltKVP if the current TableEditAccumulator contains the given key, or it exists in the row data. - // This assumes that the given hash is for the given key. - Get(ctx context.Context, keyHash hash.Hash, key types.Tuple) (*doltKVP, bool, error) - - // HasPartial returns true if the current TableEditAccumulator contains the given partialKey - HasPartial(ctx context.Context, idxSch schema.Schema, partialKeyHash hash.Hash, partialKey types.Tuple) ([]hashedTuple, error) - - // Commit applies the in memory edits to the list of committed in memory edits - Commit(ctx context.Context, nbf *types.NomsBinFormat) error - - // Rollback rolls back in memory edits until it reaches the state represented by the savedTea - Rollback(ctx context.Context) error - - // MaterializeEdits commits and applies the in memory edits to the row data - MaterializeEdits(ctx context.Context, nbf *types.NomsBinFormat) (m types.Map, err error) -} - -// var for testing -var flushThreshold int64 = 256 * 1024 - -// inMemModifications represent row adds and deletes that have not been written to the underlying storage and only exist -// in memory -type inMemModifications struct { - ops int64 - adds map[hash.Hash]*doltKVP - deletes map[hash.Hash]types.Tuple -} - -// newInMemModifications returns a pointer to a newly created inMemModifications object -func newInMemModifications() *inMemModifications { - return &inMemModifications{ - adds: make(map[hash.Hash]*doltKVP), - deletes: make(map[hash.Hash]types.Tuple), - } -} - -// MergeIn merges changes from another inMemModifications object into this instance -func (mods *inMemModifications) MergeIn(other *inMemModifications) { - for keyHash, key := range other.deletes { - delete(mods.adds, keyHash) - mods.deletes[keyHash] = key - } - - for keyHash, kvp := range other.adds { - delete(mods.deletes, keyHash) - mods.adds[keyHash] = kvp - } - - mods.ops += other.ops -} - -// Get returns whether a key hash has been added as an insert, or a delete in this inMemModifications object. If it is -// an insert the associated KVP is returned as well. -func (mods *inMemModifications) Get(keyHash hash.Hash) (kvp *doltKVP, added, deleted bool) { - kvp, added = mods.adds[keyHash] - - if added { - return kvp, true, false - } - - _, deleted = mods.deletes[keyHash] - - return nil, false, deleted -} - -// tableEditAccumulatorImpl accumulates edits that need to be applied to the table row data. It needs to be able to -// support rollback and commit without having to materialize the types.Map. To do this it tracks committed and uncommitted -// modifications in memory. When a commit occurs the list of uncommitted changes are added to the list of committed changes. -// When a rollback occurs uncommitted changes are dropped. -// -// In addition to the in memory edits, the changes are applied to committedEA when a commit occurs. It is possible -// for the uncommitted changes to become so large that they need to be flushed to disk. At this point we change modes to write all edits -// to a separate map edit accumulator as they occur until the next commit occurs. -type tableEditAccumulatorImpl struct { - vr types.ValueReader - - // initial state of the map - rowData types.Map - - // in memory changes which will be applied to the rowData when the map is materialized - committed *inMemModifications - uncommitted *inMemModifications - - // accumulatorIdx defines the order in which types.EditAccumulators will be applied - accumulatorIdx uint64 - - // flusher manages flushing of the types.EditAccumulators to disk when needed - flusher *edits.DiskEditFlusher - - // committedEaIds tracks ids of edit accumulators which have changes that have been committed - committedEaIds *set.Uint64Set - // uncommittedEAIds tracks ids of edit accumulators which have not been committed yet. - uncommittedEaIds *set.Uint64Set - - // commitEA is the types.EditAccumulator containing the committed changes that are being accumulated currently - commitEA types.EditAccumulator - // commitEAId is the id used for ordering the commitEA with other types.EditAccumulators that will be applied when - // materializing all changes. - commitEAId uint64 - - // flushingUncommitted is a flag that tracks whether we are in a state where we write uncommitted map edits to uncommittedEA - flushingUncommitted bool - // lastFlush is the number of uncommitted ops that had occurred at the time of the last flush - lastFlush int64 - // uncommittedEA is a types.EditAccumulator that we write to as uncommitted edits come in when the number of uncommitted - // edits becomes large - uncommittedEA types.EditAccumulator - // uncommittedEAId is the id used for ordering the uncommittedEA with other types.EditAccumulators that will be applied - // when materializing all changes - uncommittedEAId uint64 -} - -// Get returns a *doltKVP if the current TableEditAccumulator contains the given key, or it exists in the row data. -// This assumes that the given hash is for the given key. -func (tea *tableEditAccumulatorImpl) Get(ctx context.Context, keyHash hash.Hash, key types.Tuple) (*doltKVP, bool, error) { - // in order of the most recent changes to the least recent falling back to what is in the materialized row data - orderedMods := []*inMemModifications{tea.uncommitted, tea.committed} - for _, mods := range orderedMods { - kvp, added, deleted := mods.Get(keyHash) - - if added { - return kvp, true, nil - } else if deleted { - return nil, false, nil - } - } - - v, ok, err := tea.rowData.MaybeGetTuple(ctx, key) - if err != nil { - return nil, false, err - } - if !ok { - return nil, false, nil - } - - return &doltKVP{k: key, v: v}, true, err -} - -func (tea *tableEditAccumulatorImpl) HasPartial(ctx context.Context, idxSch schema.Schema, partialKeyHash hash.Hash, partialKey types.Tuple) ([]hashedTuple, error) { - var err error - var matches []hashedTuple - var mapIter table.ReadCloser = noms.NewNomsRangeReader(tea.vr, idxSch, tea.rowData, []*noms.ReadRange{ - {Start: partialKey, Inclusive: true, Reverse: false, Check: noms.InRangeCheckPartial(partialKey)}}) - defer mapIter.Close(ctx) - var r row.Row - for r, err = mapIter.ReadRow(ctx); err == nil; r, err = mapIter.ReadRow(ctx) { - tplKeyVal, err := r.NomsMapKey(idxSch).Value(ctx) - if err != nil { - return nil, err - } - key := tplKeyVal.(types.Tuple) - tplValVal, err := r.NomsMapValue(idxSch).Value(ctx) - if err != nil { - return nil, err - } - val := tplValVal.(types.Tuple) - keyHash, err := key.Hash(key.Format()) - if err != nil { - return nil, err - } - matches = append(matches, hashedTuple{key, val, keyHash}) - } - - if err != io.EOF { - return nil, err - } - - orderedMods := []*inMemModifications{tea.committed, tea.uncommitted} - for _, mods := range orderedMods { - for i := len(matches) - 1; i >= 0; i-- { - if _, ok := mods.adds[matches[i].hash]; ok { - matches[i] = matches[len(matches)-1] - matches = matches[:len(matches)-1] - } - } - if added, ok := mods.adds[partialKeyHash]; ok { - matches = append(matches, hashedTuple{key: added.k, value: added.v}) - } - } - - return matches, nil -} - -func (tea *tableEditAccumulatorImpl) flushUncommitted() { - // if we are not already actively writing edits to the uncommittedEA then change the state and push all in mem edits - // to a types.EditAccumulator - if !tea.flushingUncommitted { - tea.flushingUncommitted = true - - if tea.commitEA != nil && tea.commitEA.EditsAdded() > 0 { - // if there are uncommitted flushed changes we need to flush the committed changes first - // so they can be applied before the uncommitted flushed changes and future changes can be applied after - tea.committedEaIds.Add(tea.commitEAId) - tea.flusher.Flush(tea.commitEA, tea.commitEAId) - - tea.commitEA = nil - tea.commitEAId = invalidEaId - } - - tea.uncommittedEA = edits.NewAsyncSortedEditsWithDefaults(tea.vr) - tea.uncommittedEAId = tea.accumulatorIdx - tea.accumulatorIdx++ - - for _, kvp := range tea.uncommitted.adds { - tea.uncommittedEA.AddEdit(kvp.k, kvp.v) - } - - for _, key := range tea.uncommitted.deletes { - tea.uncommittedEA.AddEdit(key, nil) - } - } - - // flush uncommitted - tea.lastFlush = tea.uncommitted.ops - tea.uncommittedEaIds.Add(tea.uncommittedEAId) - tea.flusher.Flush(tea.uncommittedEA, tea.uncommittedEAId) - - // initialize a new types.EditAccumulator for additional uncommitted edits to be written to. - tea.uncommittedEA = edits.NewAsyncSortedEditsWithDefaults(tea.vr) - tea.uncommittedEAId = tea.accumulatorIdx - tea.accumulatorIdx++ -} - -// Delete adds a row to be deleted when these edits are eventually applied. Updates are modeled as a delete and an insert -func (tea *tableEditAccumulatorImpl) Delete(keyHash hash.Hash, key types.Tuple) error { - delete(tea.uncommitted.adds, keyHash) - tea.uncommitted.deletes[keyHash] = key - tea.uncommitted.ops++ - - if tea.flushingUncommitted { - tea.uncommittedEA.AddEdit(key, nil) - - if tea.uncommitted.ops-tea.lastFlush > flushThreshold { - tea.flushUncommitted() - } - } else if tea.uncommitted.ops > flushThreshold { - tea.flushUncommitted() - } - - return nil -} - -// Insert adds a row to be inserted when these edits are eventually applied. Updates are modeled as a delete and an insert. -func (tea *tableEditAccumulatorImpl) Insert(keyHash hash.Hash, key types.Tuple, val types.Tuple) error { - delete(tea.uncommitted.deletes, keyHash) - tea.uncommitted.adds[keyHash] = &doltKVP{k: key, v: val} - tea.uncommitted.ops++ - - if tea.flushingUncommitted { - tea.uncommittedEA.AddEdit(key, val) - - if tea.uncommitted.ops-tea.lastFlush > flushThreshold { - tea.flushUncommitted() - } - } else if tea.uncommitted.ops > flushThreshold { - tea.flushUncommitted() - } - - return nil -} - -// Commit applies the in memory edits to the list of committed in memory edits -func (tea *tableEditAccumulatorImpl) Commit(ctx context.Context, nbf *types.NomsBinFormat) error { - if tea.uncommitted.ops > 0 { - if !tea.flushingUncommitted { - // if there are uncommitted changes add them to the committed list of map edits - for _, kvp := range tea.uncommitted.adds { - tea.commitEA.AddEdit(kvp.k, kvp.v) - } - - for _, key := range tea.uncommitted.deletes { - tea.commitEA.AddEdit(key, nil) - } - } else { - // if we were flushing to the uncommittedEA make the current uncommittedEA the active committedEA and add - // any uncommittedEA IDs that we already flushed - tea.commitEA = tea.uncommittedEA - tea.commitEAId = tea.uncommittedEAId - tea.committedEaIds.Add(tea.uncommittedEaIds.AsSlice()...) - - // reset state to not be flushing uncommitted - tea.uncommittedEA = nil - tea.uncommittedEAId = invalidEaId - tea.uncommittedEaIds = set.NewUint64Set(nil) - tea.lastFlush = 0 - tea.flushingUncommitted = false - } - - // apply in memory uncommitted changes to the committed in memory edits - tea.committed.MergeIn(tea.uncommitted) - - // initialize uncommitted to future in memory edits - tea.uncommitted = newInMemModifications() - } - - return nil -} - -// Rollback rolls back in memory edits until it reaches the state represented by the savedTea -func (tea *tableEditAccumulatorImpl) Rollback(ctx context.Context) error { - // drop uncommitted ea IDs - tea.uncommittedEaIds = set.NewUint64Set(nil) - - if tea.uncommitted.ops > 0 { - tea.uncommitted = newInMemModifications() - - if tea.flushingUncommitted { - _ = tea.uncommittedEA.Close(ctx) - tea.uncommittedEA = nil - tea.uncommittedEAId = invalidEaId - tea.uncommittedEaIds = set.NewUint64Set(nil) - tea.lastFlush = 0 - tea.flushingUncommitted = false - } - } - - return nil -} - -// MaterializeEdits applies the in memory edits to the row data and returns types.Map -func (tea *tableEditAccumulatorImpl) MaterializeEdits(ctx context.Context, nbf *types.NomsBinFormat) (m types.Map, err error) { - // In the case where the current edits become so large that they need to be flushed to disk, the committed edits will also be flushed - // to disk first before the uncommitted edits. When commit gets run now the uncommitted edits will then become committed edits, - // but they need to be applied after the flushed edits. So in the loop below where we build the list of EditProviders the newly - // committed edits must be applied last. - err = tea.Commit(ctx, nbf) - if err != nil { - return types.EmptyMap, err - } - - if tea.committed.ops == 0 { - return tea.rowData, nil - } - - committedEP, err := tea.commitEA.FinishedEditing(ctx) - tea.commitEA = nil - if err != nil { - return types.EmptyMap, err - } - - flushedEPs, err := tea.flusher.WaitForIDs(ctx, tea.committedEaIds) - if err != nil { - return types.EmptyMap, err - } - - eps := make([]types.EditProvider, 0, len(flushedEPs)+1) - for i := 0; i < len(flushedEPs); i++ { - eps = append(eps, flushedEPs[i].Edits) - } - eps = append(eps, committedEP) - - defer func() { - for _, ep := range eps { - _ = ep.Close(ctx) - } - }() - - accEdits, err := edits.NewEPMerger(ctx, tea.vr, eps) - if err != nil { - return types.EmptyMap, err - } - - // We are guaranteed that rowData is valid, as we process teas sequentially. - updatedMap, _, err := types.ApplyEdits(ctx, accEdits, tea.rowData) - if err != nil { - return types.EmptyMap, err - } - - tea.rowData = updatedMap - tea.committed = newInMemModifications() - tea.commitEAId = tea.accumulatorIdx - tea.accumulatorIdx++ - tea.commitEA = edits.NewAsyncSortedEditsWithDefaults(tea.vr) - tea.committedEaIds = set.NewUint64Set(nil) - tea.uncommittedEaIds = set.NewUint64Set(nil) - - return updatedMap, nil -} - -// DbEaFactory is an interface for a factory object used to make table and index edit accumulators -type DbEaFactory interface { - // NewTableEA creates a TableEditAccumulator - NewTableEA(ctx context.Context, rowData types.Map) TableEditAccumulator - // NewIndexEA creates an IndexEditAccumulator - NewIndexEA(ctx context.Context, rowData types.Map) IndexEditAccumulator -} - -type dbEaFactory struct { - directory string - vrw types.ValueReadWriter -} - -// NewDbEaFactory creates a DbEaFatory which uses the provided directory to hold temp files -func NewDbEaFactory(directory string, vrw types.ValueReadWriter) DbEaFactory { - return &dbEaFactory{ - directory: directory, - vrw: vrw, - } -} - -// NewTableEA creates a TableEditAccumulator -func (deaf *dbEaFactory) NewTableEA(ctx context.Context, rowData types.Map) TableEditAccumulator { - return &tableEditAccumulatorImpl{ - vr: deaf.vrw, - rowData: rowData, - committed: newInMemModifications(), - uncommitted: newInMemModifications(), - accumulatorIdx: 1, - flusher: edits.NewDiskEditFlusher(ctx, deaf.directory, deaf.vrw), - committedEaIds: set.NewUint64Set(nil), - uncommittedEaIds: set.NewUint64Set(nil), - commitEA: edits.NewAsyncSortedEditsWithDefaults(deaf.vrw), - commitEAId: 0, - flushingUncommitted: false, - lastFlush: 0, - uncommittedEA: nil, - uncommittedEAId: invalidEaId, - } -} - -// NewIndexEA creates an IndexEditAccumulator -func (deaf *dbEaFactory) NewIndexEA(ctx context.Context, rowData types.Map) IndexEditAccumulator { - return &indexEditAccumulatorImpl{ - vr: deaf.vrw, - rowData: rowData, - committed: newInMemIndexEdits(), - uncommitted: newInMemIndexEdits(), - commitEA: edits.NewAsyncSortedEditsWithDefaults(deaf.vrw), - commitEAId: 0, - accumulatorIdx: 1, - flusher: edits.NewDiskEditFlusher(ctx, deaf.directory, deaf.vrw), - committedEaIds: set.NewUint64Set(nil), - uncommittedEaIds: set.NewUint64Set(nil), - flushingUncommitted: false, - lastFlush: 0, - uncommittedEA: nil, - uncommittedEAId: invalidEaId, - } -} diff --git a/go/libraries/doltcore/table/editor/table_edit_accumulator_test.go b/go/libraries/doltcore/table/editor/table_edit_accumulator_test.go deleted file mode 100644 index cc192dd568..0000000000 --- a/go/libraries/doltcore/table/editor/table_edit_accumulator_test.go +++ /dev/null @@ -1,269 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package editor - -import ( - "context" - "os" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/dolthub/dolt/go/store/hash" - "github.com/dolthub/dolt/go/store/types" -) - -var emptyTpl = types.EmptyTuple(types.Format_LD_1) - -func newTestTEAF() *dbEaFactory { - dir := os.TempDir() - - return &dbEaFactory{ - directory: dir, - vrw: types.NewMemoryValueStore(), - } -} - -func newTuple(t *testing.T, vals ...types.Value) types.Tuple { - tpl, err := types.NewTuple(types.Format_LD_1, vals...) - require.NoError(t, err) - - return tpl -} - -func teaInsert(t *testing.T, tea TableEditAccumulator, key types.Tuple) { - h, err := key.Hash(types.Format_LD_1) - require.NoError(t, err) - - tea.Insert(h, key, emptyTpl) -} - -func teaDelete(t *testing.T, tea TableEditAccumulator, key types.Tuple) { - h, err := key.Hash(types.Format_LD_1) - require.NoError(t, err) - - tea.Delete(h, key) -} - -func requireGet(ctx context.Context, t *testing.T, tea TableEditAccumulator, key types.Tuple, expected bool) { - h, err := key.Hash(types.Format_LD_1) - require.NoError(t, err) - _, has, err := tea.Get(ctx, h, key) - require.NoError(t, err) - require.Equal(t, expected, has) -} - -func TestIndexEditAccumulatorStableOrder(t *testing.T) { - origFlushThreshold := flushThreshold - defer func() { - indexFlushThreshold = origFlushThreshold - }() - indexFlushThreshold = 1 - - ctx := context.Background() - nbf := types.Format_LD_1 - teaf := newTestTEAF() - m, err := types.NewMap(ctx, teaf.vrw) - require.NoError(t, err) - iea := teaf.NewIndexEA(ctx, m).(*indexEditAccumulatorImpl) - - h := func(k types.Tuple) hash.Hash { - h, err := k.Hash(nbf) - require.NoError(t, err) - return h - } - - k1 := newTuple(t, types.Int(0)) - k2 := newTuple(t, types.Int(1)) - - err = iea.Insert(ctx, h(k1), h(k1), k1, emptyTpl) - require.NoError(t, err) - err = iea.Insert(ctx, h(k2), h(k1), k2, emptyTpl) - require.NoError(t, err) - - err = iea.Delete(ctx, h(k1), h(k1), k1, k1) - require.NoError(t, err) - err = iea.Delete(ctx, h(k2), h(k2), k2, k2) - require.NoError(t, err) - - err = iea.Insert(ctx, h(k1), h(k1), k1, k1) - require.NoError(t, err) - - err = iea.Commit(ctx, nbf) - require.NoError(t, err) - - m, err = iea.MaterializeEdits(ctx, nbf) - require.NoError(t, err) - require.Equal(t, uint64(1), m.Len()) -} - -func TestTableEditAccumulatorStableOrder(t *testing.T) { - origFlushThreshold := flushThreshold - defer func() { - flushThreshold = origFlushThreshold - }() - flushThreshold = 2 - - ctx := context.Background() - nbf := types.Format_LD_1 - teaf := newTestTEAF() - m, err := types.NewMap(ctx, teaf.vrw) - require.NoError(t, err) - tea := teaf.NewTableEA(ctx, m).(*tableEditAccumulatorImpl) - - h := func(k types.Tuple) hash.Hash { - h, err := k.Hash(nbf) - require.NoError(t, err) - return h - } - - k1 := newTuple(t, types.Int(0)) - k2 := newTuple(t, types.Int(1)) - err = tea.Delete(h(k1), k1) - require.NoError(t, err) - err = tea.Delete(h(k2), k2) - require.NoError(t, err) - - err = tea.Insert(h(k1), k1, emptyTpl) - require.NoError(t, err) - err = tea.Insert(h(k2), k2, emptyTpl) - require.NoError(t, err) - - err = tea.Commit(ctx, nbf) - require.NoError(t, err) - - m, err = tea.MaterializeEdits(ctx, nbf) - require.NoError(t, err) - require.Equal(t, uint64(2), m.Len()) -} - -func TestGet(t *testing.T) { - ctx := context.Background() - nbf := types.Format_LD_1 - teaf := newTestTEAF() - m, err := types.NewMap(ctx, teaf.vrw) - require.NoError(t, err) - tea := teaf.NewTableEA(ctx, m).(*tableEditAccumulatorImpl) - - key1 := newTuple(t, types.Int(1)) - key2 := newTuple(t, types.Int(2)) - key3 := newTuple(t, types.Int(3)) - key4 := newTuple(t, types.Int(4)) - key5 := newTuple(t, types.Int(5)) - key6 := newTuple(t, types.Int(6)) - - // test uncommitted - requireGet(ctx, t, tea, key1, false) - teaInsert(t, tea, key1) - requireGet(ctx, t, tea, key1, true) - err = tea.Rollback(ctx) - require.NoError(t, err) - requireGet(ctx, t, tea, key1, false) - - // test uncommitted flushed - teaInsert(t, tea, key1) - requireGet(ctx, t, tea, key1, true) - tea.flushUncommitted() - requireGet(ctx, t, tea, key1, true) - err = tea.Rollback(ctx) - require.NoError(t, err) - requireGet(ctx, t, tea, key1, false) - - // test committed - teaInsert(t, tea, key1) - err = tea.Commit(ctx, nbf) - require.NoError(t, err) - requireGet(ctx, t, tea, key1, true) - - // edits in committed and uncommitted - requireGet(ctx, t, tea, key2, false) - teaInsert(t, tea, key2) - requireGet(ctx, t, tea, key1, true) - requireGet(ctx, t, tea, key2, true) - err = tea.Rollback(ctx) - require.NoError(t, err) - requireGet(ctx, t, tea, key1, true) - requireGet(ctx, t, tea, key2, false) - - // edits in committed and uncommitted flushed - teaInsert(t, tea, key2) - requireGet(ctx, t, tea, key1, true) - requireGet(ctx, t, tea, key2, true) - tea.flushUncommitted() - requireGet(ctx, t, tea, key1, true) - requireGet(ctx, t, tea, key2, true) - err = tea.Rollback(ctx) - require.NoError(t, err) - requireGet(ctx, t, tea, key1, true) - requireGet(ctx, t, tea, key2, false) - - // edits in committed, uncommitted and uncommitted flushed - requireGet(ctx, t, tea, key3, false) - teaInsert(t, tea, key2) - tea.flushUncommitted() - teaInsert(t, tea, key3) - requireGet(ctx, t, tea, key1, true) - requireGet(ctx, t, tea, key2, true) - requireGet(ctx, t, tea, key3, true) - err = tea.Rollback(ctx) - require.NoError(t, err) - requireGet(ctx, t, tea, key1, true) - requireGet(ctx, t, tea, key2, false) - requireGet(ctx, t, tea, key3, false) - - // edits everywhere materialized - teaInsert(t, tea, key2) - tea.flushUncommitted() - teaInsert(t, tea, key3) - requireGet(ctx, t, tea, key1, true) - requireGet(ctx, t, tea, key2, true) - requireGet(ctx, t, tea, key3, true) - - // edits in materialized data - _, err = tea.MaterializeEdits(ctx, nbf) - require.NoError(t, err) - requireGet(ctx, t, tea, key1, true) - requireGet(ctx, t, tea, key2, true) - requireGet(ctx, t, tea, key3, true) - - // edits everywhere - teaDelete(t, tea, key1) - teaInsert(t, tea, key4) - err = tea.Commit(ctx, nbf) - require.NoError(t, err) - requireGet(ctx, t, tea, key1, false) - requireGet(ctx, t, tea, key4, true) - teaDelete(t, tea, key2) - teaInsert(t, tea, key5) - tea.flushUncommitted() - requireGet(ctx, t, tea, key2, false) - requireGet(ctx, t, tea, key5, true) - teaInsert(t, tea, key6) - requireGet(ctx, t, tea, key1, false) - requireGet(ctx, t, tea, key2, false) - requireGet(ctx, t, tea, key3, true) - requireGet(ctx, t, tea, key4, true) - requireGet(ctx, t, tea, key5, true) - requireGet(ctx, t, tea, key6, true) - - _, err = tea.MaterializeEdits(ctx, nbf) - require.NoError(t, err) - requireGet(ctx, t, tea, key1, false) - requireGet(ctx, t, tea, key2, false) - requireGet(ctx, t, tea, key3, true) - requireGet(ctx, t, tea, key4, true) - requireGet(ctx, t, tea, key5, true) - requireGet(ctx, t, tea, key6, true) -} diff --git a/go/libraries/doltcore/table/typed/noms/doc.go b/go/libraries/doltcore/table/typed/noms/doc.go deleted file mode 100644 index bcc3bbaa85..0000000000 --- a/go/libraries/doltcore/table/typed/noms/doc.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2019 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package nbf provides TableReadCloser and TableWriteCloser implementations for working with dolt tables in noms. -package noms diff --git a/go/libraries/doltcore/table/typed/noms/range_reader.go b/go/libraries/doltcore/table/typed/noms/range_reader.go deleted file mode 100644 index 9b0c9d2f6e..0000000000 --- a/go/libraries/doltcore/table/typed/noms/range_reader.go +++ /dev/null @@ -1,372 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package noms - -import ( - "context" - "errors" - "fmt" - "io" - - "github.com/dolthub/go-mysql-server/sql" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/types" -) - -// InRangeCheck evaluates tuples to determine whether they are valid and/or should be skipped. -type InRangeCheck interface { - // Check is a call made as the reader reads through values to check that the next value either being read is valid - // and whether it should be skipped or returned. - Check(ctx context.Context, vr types.ValueReader, tuple types.Tuple) (valid bool, skip bool, err error) -} - -// InRangeCheckAlways will always return that the given tuple is valid and not to be skipped. -type InRangeCheckAlways struct{} - -func (InRangeCheckAlways) Check(context.Context, types.ValueReader, types.Tuple) (valid bool, skip bool, err error) { - return true, false, nil -} - -func (InRangeCheckAlways) String() string { - return "Always" -} - -// InRangeCheckNever will always return that the given tuple is not valid. -type InRangeCheckNever struct{} - -func (InRangeCheckNever) Check(context.Context, types.ValueReader, types.Tuple) (valid bool, skip bool, err error) { - return false, false, nil -} - -func (InRangeCheckNever) String() string { - return "Never" -} - -// InRangeCheckPartial will check if the given tuple contains the aliased tuple as a partial key. -type InRangeCheckPartial types.Tuple - -func (ircp InRangeCheckPartial) Check(_ context.Context, vr types.ValueReader, t types.Tuple) (valid bool, skip bool, err error) { - return t.StartsWith(types.Tuple(ircp)), false, nil -} - -func (ircp InRangeCheckPartial) String() string { - return fmt.Sprintf("StartsWith(%v)", types.Tuple(ircp).HumanReadableString()) -} - -// ReadRange represents a range of values to be read -type ReadRange struct { - // Start is a Dolt map key which is the starting point (or ending point if Reverse is true) - Start types.Tuple - // Inclusive says whether the Start key should be included in the range. - Inclusive bool - // Reverse says if the range should be read in reverse (from high to low) instead of the default (low to high) - Reverse bool - // Check is a callb made as the reader reads through values to check that the next value being read is in the range. - Check InRangeCheck -} - -func (rr *ReadRange) String() string { - return fmt.Sprintf("ReadRange[Start: %v, Inclusive: %t, Reverse %t, Check: %v]", rr.Start.HumanReadableString(), rr.Inclusive, rr.Reverse, rr.Check) -} - -// NewRangeEndingAt creates a range with a starting key which will be iterated in reverse -func NewRangeEndingAt(key types.Tuple, inRangeCheck InRangeCheck) *ReadRange { - return &ReadRange{ - Start: key, - Inclusive: true, - Reverse: true, - Check: inRangeCheck, - } -} - -// NewRangeEndingBefore creates a range starting before the provided key iterating in reverse -func NewRangeEndingBefore(key types.Tuple, inRangeCheck InRangeCheck) *ReadRange { - return &ReadRange{ - Start: key, - Inclusive: false, - Reverse: true, - Check: inRangeCheck, - } -} - -// NewRangeStartingAt creates a range with a starting key -func NewRangeStartingAt(key types.Tuple, inRangeCheck InRangeCheck) *ReadRange { - return &ReadRange{ - Start: key, - Inclusive: true, - Reverse: false, - Check: inRangeCheck, - } -} - -// NewRangeStartingAfter creates a range starting after the provided key -func NewRangeStartingAfter(key types.Tuple, inRangeCheck InRangeCheck) *ReadRange { - return &ReadRange{ - Start: key, - Inclusive: false, - Reverse: false, - Check: inRangeCheck, - } -} - -// NomsRangeReader reads values in one or more ranges from a map -type NomsRangeReader struct { - vr types.ValueReader - sch schema.Schema - m types.Map - ranges []*ReadRange - idx int - itr types.MapIterator - currCheck InRangeCheck - cardCounter *CardinalityCounter -} - -// NewNomsRangeReader creates a NomsRangeReader -func NewNomsRangeReader(vr types.ValueReader, sch schema.Schema, m types.Map, ranges []*ReadRange) *NomsRangeReader { - return &NomsRangeReader{ - vr, - sch, - m, - ranges, - 0, - nil, - nil, - NewCardinalityCounter(), - } -} - -// GetSchema gets the schema of the rows being read. -func (nrr *NomsRangeReader) GetSchema() schema.Schema { - return nrr.sch -} - -// ReadRow reads a row from a table. If there is a bad row the returned error will be non nil, and calling -// IsBadRow(err) will be return true. This is a potentially non-fatal error and callers can decide if they want to -// continue on a bad row, or fail. -func (nrr *NomsRangeReader) ReadRow(ctx context.Context) (row.Row, error) { - k, v, err := nrr.ReadKV(ctx) - - if err != nil { - return nil, err - } - - return row.FromNoms(nrr.sch, k, v) -} - -func (nrr *NomsRangeReader) ReadKey(ctx context.Context) (types.Tuple, error) { - k, _, err := nrr.ReadKV(ctx) - - return k, err -} - -func (nrr *NomsRangeReader) ReadKV(ctx context.Context) (types.Tuple, types.Tuple, error) { - var err error - var k types.Tuple - var v types.Tuple - for nrr.itr != nil || nrr.idx < len(nrr.ranges) { - if !nrr.cardCounter.empty() { - if nrr.cardCounter.done() { - nrr.cardCounter.reset() - } else { - return nrr.cardCounter.next() - } - } - - if nrr.itr == nil { - r := nrr.ranges[nrr.idx] - nrr.idx++ - - if r.Reverse { - nrr.itr, err = nrr.m.IteratorBackFrom(ctx, r.Start) - } else { - nrr.itr, err = nrr.m.IteratorFrom(ctx, r.Start) - } - if err != nil { - return types.Tuple{}, types.Tuple{}, err - } - - nrr.currCheck = r.Check - - k, v, err = nrr.itr.NextTuple(ctx) - - if err == nil && !r.Inclusive { - var res int - res, err = r.Start.Compare(ctx, nrr.vr.Format(), k) - if err == nil && res == 0 { - k, v, err = nrr.itr.NextTuple(ctx) - } - } - } else { - k, v, err = nrr.itr.NextTuple(ctx) - } - - if err != nil && err != io.EOF { - return types.Tuple{}, types.Tuple{}, err - } - - if err != io.EOF { - valid, skip, err := nrr.currCheck.Check(ctx, nrr.vr, k) - if err != nil { - return types.Tuple{}, types.Tuple{}, err - } - - if valid { - if skip { - continue - } - if !v.Empty() { - nrr.cardCounter.updateWithKV(k, v) - if !nrr.cardCounter.empty() && !nrr.cardCounter.done() { - return nrr.cardCounter.next() - } - } - return k, v, nil - } - } - - nrr.itr = nil - nrr.currCheck = nil - } - - return types.Tuple{}, types.Tuple{}, io.EOF -} - -// VerifySchema checks that the incoming schema matches the schema from the existing table -func (nrr *NomsRangeReader) VerifySchema(outSch schema.Schema) (bool, error) { - return schema.VerifyInSchema(nrr.sch, outSch) -} - -// Close should release resources being held -func (nrr *NomsRangeReader) Close(ctx context.Context) error { - return nil -} - -// SqlRowFromTuples constructs a go-mysql-server/sql.Row from Noms tuples. -func SqlRowFromTuples(sch schema.Schema, key, val types.Tuple) (sql.Row, error) { - allCols := sch.GetAllCols() - colVals := make(sql.Row, allCols.Size()) - - keySl, err := key.AsSlice() - if err != nil { - return nil, err - } - valSl, err := val.AsSlice() - if err != nil { - return nil, err - } - - for _, sl := range []types.TupleValueSlice{keySl, valSl} { - var convErr error - err := row.IterPkTuple(sl, func(tag uint64, val types.Value) (stop bool, err error) { - if idx, ok := allCols.TagToIdx[tag]; ok { - col := allCols.GetByIndex(idx) - colVals[idx], convErr = col.TypeInfo.ConvertNomsValueToValue(val) - - if convErr != nil { - return false, err - } - } - - return false, nil - }) - - if err != nil { - return nil, err - } - } - - return sql.NewRow(colVals...), nil -} - -type CardinalityCounter struct { - key *types.Tuple - value *types.Tuple - card int - idx int -} - -func NewCardinalityCounter() *CardinalityCounter { - return &CardinalityCounter{ - nil, - nil, - -1, - -1, - } -} - -func (cc *CardinalityCounter) updateWithKV(k, v types.Tuple) error { - if !v.Empty() { - cardTagVal, err := v.Get(0) - if err != nil { - return err - } - cardTag, ok := cardTagVal.(types.Uint) - if !ok { - return errors.New("index cardinality invalid tag type") - } - - if uint64(cardTag) != schema.KeylessRowCardinalityTag { - return errors.New("index cardinality tag invalid") - } - - cardVal, err := v.Get(1) - if err != nil { - return err - } - card, ok := cardVal.(types.Uint) - if !ok { - return errors.New("index cardinality value invalid type") - } - if int(card) > 1 { - cc.card = int(card) - cc.idx = 0 - cc.key = &k - cc.value = &v - return nil - } else { - cc.card = -1 - cc.idx = -1 - cc.key = nil - cc.value = nil - } - } - return nil -} - -func (cc *CardinalityCounter) empty() bool { - return cc.key == nil || cc.value == nil -} - -func (cc *CardinalityCounter) done() bool { - return cc.card < 1 || cc.idx >= cc.card -} - -func (cc *CardinalityCounter) next() (types.Tuple, types.Tuple, error) { - if cc.key == nil || cc.value == nil { - return types.Tuple{}, types.Tuple{}, errors.New("cannot increment empty cardinality counter") - } - cc.idx++ - return *cc.key, *cc.value, nil - -} - -func (cc *CardinalityCounter) reset() { - cc.card = -1 - cc.idx = -1 - cc.key = nil - cc.value = nil -} diff --git a/go/libraries/doltcore/table/typed/noms/range_reader_test.go b/go/libraries/doltcore/table/typed/noms/range_reader_test.go deleted file mode 100644 index 1b09dbcc54..0000000000 --- a/go/libraries/doltcore/table/typed/noms/range_reader_test.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2019 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package noms - -import ( - "context" - "io" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/chunks" - "github.com/dolthub/dolt/go/store/types" -) - -var rangeReaderTests = []struct { - name string - ranges []*ReadRange - expectKeys []int64 -}{ - { - "test range ending at", - []*ReadRange{NewRangeEndingAt(mustTuple(10), greaterThanCheck(2))}, - []int64{10, 8, 6, 4}, - }, - { - "test range ending before", - []*ReadRange{NewRangeEndingBefore(mustTuple(10), greaterThanCheck(2))}, - []int64{8, 6, 4}, - }, - { - "test range starting at", - []*ReadRange{NewRangeStartingAt(mustTuple(10), lessThanCheck(20))}, - []int64{10, 12, 14, 16, 18}, - }, - { - "test range starting after", - []*ReadRange{NewRangeStartingAfter(mustTuple(10), lessThanCheck(20))}, - []int64{12, 14, 16, 18}, - }, - { - "test range iterating to the end", - []*ReadRange{NewRangeStartingAt(mustTuple(100), lessThanCheck(200))}, - []int64{100}, - }, - { - "test multiple ranges", - []*ReadRange{ - NewRangeEndingBefore(mustTuple(10), greaterThanCheck(2)), - NewRangeStartingAt(mustTuple(10), lessThanCheck(20)), - }, - []int64{8, 6, 4, 10, 12, 14, 16, 18}, - }, - { - "test empty range starting after", - []*ReadRange{NewRangeStartingAfter(mustTuple(100), lessThanCheck(200))}, - []int64(nil), - }, - { - "test empty range starting at", - []*ReadRange{NewRangeStartingAt(mustTuple(101), lessThanCheck(200))}, - []int64(nil), - }, - { - "test empty range ending before", - []*ReadRange{NewRangeEndingBefore(mustTuple(0), greaterThanCheck(-100))}, - []int64(nil), - }, - { - "test empty range ending at", - []*ReadRange{NewRangeEndingAt(mustTuple(-1), greaterThanCheck(-100))}, - []int64(nil), - }, -} - -func mustTuple(id int64) types.Tuple { - t, err := types.NewTuple(types.Format_Default, types.Uint(pkTag), types.Int(id)) - - if err != nil { - panic(err) - } - - return t -} - -func TestRangeReader(t *testing.T) { - ctx := context.Background() - colColl := schema.NewColCollection( - schema.NewColumn("id", pkTag, types.IntKind, true), - schema.NewColumn("val", valTag, types.IntKind, false)) - - sch, err := schema.SchemaFromCols(colColl) - require.NoError(t, err) - - storage := &chunks.MemoryStorage{} - vrw := types.NewValueStore(storage.NewView()) - m, err := types.NewMap(ctx, vrw) - assert.NoError(t, err) - - me := m.Edit() - for i := 0; i <= 100; i += 2 { - k, err := types.NewTuple(vrw.Format(), types.Uint(pkTag), types.Int(i)) - require.NoError(t, err) - - v, err := types.NewTuple(vrw.Format(), types.Uint(valTag), types.Int(100-i)) - require.NoError(t, err) - - me.Set(k, v) - } - - m, err = me.Map(ctx) - assert.NoError(t, err) - - for _, test := range rangeReaderTests { - t.Run(test.name, func(t *testing.T) { - ctx := context.Background() - rd := NewNomsRangeReader(vrw, sch, m, test.ranges) - - var keys []int64 - for { - r, err := rd.ReadRow(ctx) - - if err == io.EOF { - break - } - - assert.NoError(t, err) - col0, ok := r.GetColVal(0) - assert.True(t, ok) - - keys = append(keys, int64(col0.(types.Int))) - } - - err = rd.Close(ctx) - assert.NoError(t, err) - - assert.Equal(t, test.expectKeys, keys) - }) - } -} - -func TestRangeReaderOnEmptyMap(t *testing.T) { - ctx := context.Background() - colColl := schema.NewColCollection( - schema.NewColumn("id", pkTag, types.IntKind, true), - schema.NewColumn("val", valTag, types.IntKind, false)) - - sch, err := schema.SchemaFromCols(colColl) - require.NoError(t, err) - - storage := &chunks.MemoryStorage{} - vrw := types.NewValueStore(storage.NewView()) - m, err := types.NewMap(ctx, vrw) - assert.NoError(t, err) - - for _, test := range rangeReaderTests { - t.Run(test.name, func(t *testing.T) { - ctx := context.Background() - rd := NewNomsRangeReader(vrw, sch, m, test.ranges) - - r, err := rd.ReadRow(ctx) - assert.Equal(t, io.EOF, err) - assert.Nil(t, r) - }) - } -} - -type greaterThanCheck int64 - -func (n greaterThanCheck) Check(ctx context.Context, _ types.ValueReader, k types.Tuple) (valid bool, skip bool, err error) { - col0, err := k.Get(1) - - if err != nil { - panic(err) - } - - return int64(col0.(types.Int)) > int64(n), false, nil -} - -type lessThanCheck int64 - -func (n lessThanCheck) Check(ctx context.Context, _ types.ValueReader, k types.Tuple) (valid bool, skip bool, err error) { - col0, err := k.Get(1) - - if err != nil { - panic(err) - } - - return int64(col0.(types.Int)) < int64(n), false, nil -} diff --git a/go/libraries/doltcore/table/typed/noms/reader.go b/go/libraries/doltcore/table/typed/noms/reader.go deleted file mode 100644 index 674cc76948..0000000000 --- a/go/libraries/doltcore/table/typed/noms/reader.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2019 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package noms - -import ( - "context" - "io" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/types" -) - -type StatsCB func(stats types.AppliedEditStats) - -// NomsMapReader is a TableReader that reads rows from a noms table which is stored in a types.Map where the key is -// a types.Value and the value is a types.Tuple of field values. -type NomsMapReader struct { - sch schema.Schema - itr types.MapIterator -} - -// NewNomsMapReader creates a NomsMapReader for a given noms types.Map -func NewNomsMapReader(ctx context.Context, m types.Map, sch schema.Schema) (*NomsMapReader, error) { - itr, err := m.Iterator(ctx) - - if err != nil { - return nil, err - } - - return &NomsMapReader{sch, itr}, nil -} - -// GetSchema gets the schema of the rows that this reader will return -func (nmr *NomsMapReader) GetSchema() schema.Schema { - return nmr.sch -} - -// ReadRow reads a row from a table. If there is a bad row the returned error will be non nil, and callin IsBadRow(err) -// will be return true. This is a potentially non-fatal error and callers can decide if they want to continue on a bad row, or fail. -func (nmr *NomsMapReader) ReadRow(ctx context.Context) (row.Row, error) { - key, val, err := nmr.itr.Next(ctx) - - if err != nil { - return nil, err - } else if key == nil { - return nil, io.EOF - } - - return row.FromNoms(nmr.sch, key.(types.Tuple), val.(types.Tuple)) -} - -// Close should release resources being held -func (nmr *NomsMapReader) Close(ctx context.Context) error { - nmr.itr = nil - return nil -} - -// VerifySchema checks that the incoming schema matches the schema from the existing table -func (nmr *NomsMapReader) VerifySchema(outSch schema.Schema) (bool, error) { - return schema.VerifyInSchema(nmr.sch, outSch) -} diff --git a/go/libraries/doltcore/table/typed/noms/reader_for_keys.go b/go/libraries/doltcore/table/typed/noms/reader_for_keys.go deleted file mode 100644 index f4ec11c550..0000000000 --- a/go/libraries/doltcore/table/typed/noms/reader_for_keys.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package noms - -import ( - "context" - "io" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/types" -) - -// KeyIterator is an interface for iterating through a collection of keys -type KeyIterator interface { - // Next returns the next key in the collection. When all keys are exhausted nil, io.EOF must be returned. - Next() (types.Value, error) -} - -// SliceOfKeysIterator is a KeyIterator implementation backed by a slice of keys which are iterated in order -type SliceOfKeysIterator struct { - keys []types.Tuple - idx int -} - -// Next returns the next key in the slice. When all keys are exhausted nil, io.EOF is be returned. -func (sokItr *SliceOfKeysIterator) Next() (types.Value, error) { - if sokItr.idx < len(sokItr.keys) { - k := sokItr.keys[sokItr.idx] - sokItr.idx++ - - return k, nil - } - - return nil, io.EOF -} - -// NomsMapReaderForKeys implements TableReadCloser -type NomsMapReaderForKeys struct { - sch schema.Schema - m types.Map - keyItr KeyIterator -} - -// NewNomsMapReaderForKeys creates a NomsMapReaderForKeys for a given noms types.Map, and a list of keys -func NewNomsMapReaderForKeys(m types.Map, sch schema.Schema, keys []types.Tuple) *NomsMapReaderForKeys { - return NewNomsMapReaderForKeyItr(m, sch, &SliceOfKeysIterator{keys, 0}) -} - -// NewNomsMapReaderForKeyItr creates a NomsMapReaderForKeys for a given noms types.Map, and a list of keys -func NewNomsMapReaderForKeyItr(m types.Map, sch schema.Schema, keyItr KeyIterator) *NomsMapReaderForKeys { - return &NomsMapReaderForKeys{sch, m, keyItr} -} - -// GetSchema gets the schema of the rows being read. -func (nmr *NomsMapReaderForKeys) GetSchema() schema.Schema { - return nmr.sch -} - -// ReadRow reads a row from a table. If there is a bad row the returned error will be non nil, and calling -// IsBadRow(err) will be return true. This is a potentially non-fatal error and callers can decide if they want to -// continue on a bad row, or fail. -func (nmr *NomsMapReaderForKeys) ReadRow(ctx context.Context) (row.Row, error) { - var key types.Value - var value types.Value - var err error - for value == nil { - key, err = nmr.keyItr.Next() - - if err != nil { - return nil, err - } - - v, ok, err := nmr.m.MaybeGet(ctx, key) - - if err != nil { - return nil, err - } - - if ok { - value = v - } - } - - return row.FromNoms(nmr.sch, key.(types.Tuple), value.(types.Tuple)) -} - -// VerifySchema checks that the incoming schema matches the schema from the existing table -func (nmr *NomsMapReaderForKeys) VerifySchema(outSch schema.Schema) (bool, error) { - return schema.VerifyInSchema(nmr.sch, outSch) -} - -// Close should release resources being held -func (nmr *NomsMapReaderForKeys) Close(ctx context.Context) error { - return nil -} diff --git a/go/libraries/doltcore/table/typed/noms/reader_for_keys_test.go b/go/libraries/doltcore/table/typed/noms/reader_for_keys_test.go deleted file mode 100644 index 85e1f687a7..0000000000 --- a/go/libraries/doltcore/table/typed/noms/reader_for_keys_test.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package noms - -import ( - "context" - "io" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/chunks" - "github.com/dolthub/dolt/go/store/types" -) - -const ( - pkTag uint64 = iota - valTag -) - -func TestReaderForKeys(t *testing.T) { - ctx := context.Background() - colColl := schema.NewColCollection( - schema.NewColumn("id", pkTag, types.IntKind, true), - schema.NewColumn("val", valTag, types.IntKind, false)) - - sch, err := schema.SchemaFromCols(colColl) - require.NoError(t, err) - - storage := &chunks.MemoryStorage{} - vrw := types.NewValueStore(storage.NewView()) - m, err := types.NewMap(ctx, vrw) - assert.NoError(t, err) - - me := m.Edit() - for i := 0; i <= 100; i += 2 { - k, err := types.NewTuple(vrw.Format(), types.Uint(pkTag), types.Int(i)) - require.NoError(t, err) - - v, err := types.NewTuple(vrw.Format(), types.Uint(valTag), types.Int(100-i)) - require.NoError(t, err) - - me.Set(k, v) - } - - m, err = me.Map(ctx) - assert.NoError(t, err) - - tests := []struct { - name string - keys []int - expected []int - }{ - { - name: "tens", - keys: []int{10, 20, 30, 40, 50, 60, 70, 80, 90}, - expected: []int{10, 20, 30, 40, 50, 60, 70, 80, 90}, - }, - { - name: "fives", - keys: []int{5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95}, - expected: []int{10, 20, 30, 40, 50, 60, 70, 80, 90}, - }, - { - name: "empty", - keys: []int{}, - expected: []int{}, - }, - { - name: "no keys that are in the map", - keys: []int{-5, -3, -1, 1, 3, 5, 102, 104, 106}, - expected: []int{}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctx := context.Background() - rd := NewNomsMapReaderForKeys(m, sch, intKeysToTupleKeys(t, vrw.Format(), test.keys)) - - var rows []row.Row - for { - r, err := rd.ReadRow(ctx) - - if err == io.EOF { - break - } - - assert.NoError(t, err) - rows = append(rows, r) - } - - testAgainstExpected(t, rows, test.expected) - rd.Close(ctx) - }) - } -} - -func intKeysToTupleKeys(t *testing.T, nbf *types.NomsBinFormat, keys []int) []types.Tuple { - tupleKeys := make([]types.Tuple, len(keys)) - - for i, key := range keys { - tuple, err := types.NewTuple(nbf, types.Uint(pkTag), types.Int(key)) - require.NoError(t, err) - - tupleKeys[i] = tuple - } - - return tupleKeys -} - -func testAgainstExpected(t *testing.T, rows []row.Row, expected []int) { - assert.Equal(t, len(expected), len(rows)) - for i, r := range rows { - k, ok := r.GetColVal(pkTag) - require.True(t, ok) - v, ok := r.GetColVal(valTag) - require.True(t, ok) - - kn := int(k.(types.Int)) - vn := int(v.(types.Int)) - - expectedK := expected[i] - expectedV := 100 - expectedK - - assert.Equal(t, expectedK, kn) - assert.Equal(t, expectedV, vn) - } -} diff --git a/go/libraries/doltcore/table/untyped/xlsx/marshaling.go b/go/libraries/doltcore/table/untyped/xlsx/marshaling.go index 3f9d03ab20..f8b151aa94 100644 --- a/go/libraries/doltcore/table/untyped/xlsx/marshaling.go +++ b/go/libraries/doltcore/table/untyped/xlsx/marshaling.go @@ -27,21 +27,6 @@ import ( var ErrTableNameMatchSheetName = errors.New("table name must match excel sheet name.") -func UnmarshalFromXLSX(path string) ([][][]string, error) { - data, err := openFile(path) - - if err != nil { - return nil, err - } - - dataSlice, err := data.ToSlice() - if err != nil { - return nil, err - } - - return dataSlice, nil -} - func openFile(path string) (*xlsx.File, error) { data, err := xlsx.OpenFile(path) diff --git a/go/libraries/doltcore/table/untyped/xlsx/reader.go b/go/libraries/doltcore/table/untyped/xlsx/reader.go index 1460a47876..16dab2815e 100644 --- a/go/libraries/doltcore/table/untyped/xlsx/reader.go +++ b/go/libraries/doltcore/table/untyped/xlsx/reader.go @@ -41,35 +41,6 @@ type XLSXReader struct { vrw types.ValueReadWriter } -func OpenXLSXReaderFromBinary(ctx context.Context, vrw types.ValueReadWriter, r io.ReadCloser, info *XLSXFileInfo) (*XLSXReader, error) { - br := bufio.NewReaderSize(r, ReadBufSize) - - contents, err := io.ReadAll(r) - if err != nil { - return nil, err - } - - colStrs, err := getColHeadersFromBinary(contents, info.SheetName) - if err != nil { - return nil, err - } - - data, err := getXlsxRowsFromBinary(contents, info.SheetName) - if err != nil { - return nil, err - } - - _, sch := untyped.NewUntypedSchema(colStrs...) - - decodedRows, err := decodeXLSXRows(data, sch) - if err != nil { - r.Close() - return nil, err - } - - return &XLSXReader{r, br, info, sch, 0, decodedRows, vrw}, nil -} - func OpenXLSXReader(ctx context.Context, vrw types.ValueReadWriter, path string, fs filesys.ReadableFS, info *XLSXFileInfo) (*XLSXReader, error) { r, err := fs.OpenForRead(path) @@ -110,16 +81,6 @@ func getColHeadersFromPath(path string, sheetName string) ([]string, error) { return colHeaders, nil } -func getColHeadersFromBinary(content []byte, sheetName string) ([]string, error) { - data, err := getXlsxRowsFromBinary(content, sheetName) - if err != nil { - return nil, err - } - - colHeaders := data[0][0] - return colHeaders, nil -} - // GetSchema gets the schema of the rows that this reader will return func (xlsxr *XLSXReader) GetSchema() schema.Schema { return xlsxr.sch diff --git a/go/libraries/utils/buffer/buffer.go b/go/libraries/utils/buffer/buffer.go deleted file mode 100644 index 9fb3216016..0000000000 --- a/go/libraries/utils/buffer/buffer.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package buffer - -import ( - "io" - - "github.com/dolthub/dolt/go/libraries/utils/iohelp" -) - -type DynamicBuffer struct { - blocks [][]byte - blockSize int -} - -func New(blockSize int) *DynamicBuffer { - return &DynamicBuffer{blockSize: blockSize} -} - -func (buf *DynamicBuffer) Append(bytes []byte) { - blockIdx := len(buf.blocks) - 1 - - var space int - var pos int - if blockIdx >= 0 { - currBlock := buf.blocks[blockIdx] - pos = len(currBlock) - space = cap(currBlock) - pos - } - - for len(bytes) > 0 { - if space == 0 { - for len(bytes) >= buf.blockSize { - buf.blocks = append(buf.blocks, bytes[:buf.blockSize]) - bytes = bytes[buf.blockSize:] - blockIdx++ - } - - if len(bytes) == 0 { - return - } - - buf.blocks = append(buf.blocks, make([]byte, 0, buf.blockSize)) - pos = 0 - space = buf.blockSize - blockIdx++ - } - - n := len(bytes) - if n > space { - n = space - } - - buf.blocks[blockIdx] = buf.blocks[blockIdx][:pos+n] - copy(buf.blocks[blockIdx][pos:], bytes[:n]) - bytes = bytes[n:] - space -= n - pos += n - } -} - -func (buf *DynamicBuffer) Close() *BufferIterator { - itr := &BufferIterator{blocks: buf.blocks} - buf.blocks = nil - - return itr -} - -type BufferIterator struct { - blocks [][]byte - i int -} - -func (itr *BufferIterator) Next() ([]byte, error) { - if itr.i >= len(itr.blocks) { - return nil, io.EOF - } - next := itr.blocks[itr.i] - itr.i++ - - return next, nil -} - -func (itr *BufferIterator) NumBlocks() int { - return len(itr.blocks) -} - -func (itr *BufferIterator) FlushTo(wr io.Writer) error { - for { - data, err := itr.Next() - - if err == io.EOF { - return nil - } else if err != nil { - return err - } - - err = iohelp.WriteAll(wr, data) - - if err != nil { - return err - } - } -} - -func (itr *BufferIterator) AsReader() io.Reader { - return &bufferIteratorReader{ - itr: itr, - } -} - -type bufferIteratorReader struct { - itr *BufferIterator - currBuff []byte -} - -func (b *bufferIteratorReader) Read(dest []byte) (n int, err error) { - if len(b.currBuff) == 0 { - b.currBuff, err = b.itr.Next() - - if err != nil { - return 0, err - } - } - - destSize := len(dest) - toCopy := b.currBuff - if len(b.currBuff) > destSize { - toCopy = b.currBuff[:destSize] - } - - n = copy(dest, toCopy) - b.currBuff = b.currBuff[n:] - - return n, err -} diff --git a/go/libraries/utils/buffer/buffer_test.go b/go/libraries/utils/buffer/buffer_test.go deleted file mode 100644 index 003501da98..0000000000 --- a/go/libraries/utils/buffer/buffer_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2021 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package buffer - -import ( - "bytes" - "math/rand" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestDynamicBuffer(t *testing.T) { - const blockSize = 53 - - rand := rand.New(rand.NewSource(time.Now().UnixNano())) - for i := 0; i < 100; i++ { - n := 1000 + rand.Int63()%10000 - t.Run(strconv.FormatInt(n, 10), func(t *testing.T) { - data := make([]byte, n) - read, err := rand.Read(data) - require.NoError(t, err) - require.Equal(t, int(n), read) - - buf := New(blockSize) - buf.Append(data) - itr := buf.Close() - - reassembled := bytes.NewBuffer(nil) - err = itr.FlushTo(reassembled) - require.NoError(t, err) - require.Equal(t, data, reassembled.Bytes()) - }) - } -} diff --git a/go/serial/workingset.fbs b/go/serial/workingset.fbs index 84b5ee7530..ce1d48fcc4 100644 --- a/go/serial/workingset.fbs +++ b/go/serial/workingset.fbs @@ -67,6 +67,10 @@ table RebaseState { // The rebasing_started field indicates if execution of the rebase plan has been started or not. Once execution of the // plan has been started, the last_attempted_step field holds a reference to the most recent plan step attempted. rebasing_started:bool; + + // When set to true, the rebase process will skip performing commit + // verification if it would otherwise run. + skip_verification:bool; } // KEEP THIS IN SYNC WITH fileidentifiers.go diff --git a/go/store/blobstore/oci.go b/go/store/blobstore/oci.go index c9a030d0a2..6e0749fedc 100644 --- a/go/store/blobstore/oci.go +++ b/go/store/blobstore/oci.go @@ -22,7 +22,6 @@ import ( "io" "math" "net/http" - "os" "path" "github.com/oracle/oci-go-sdk/v65/common" @@ -44,23 +43,6 @@ type toUpload struct { type uploadFunc func(ctx context.Context, objectName, uploadID string, partNumber int, contentLength int64, reader io.Reader) (objectstorage.CommitMultipartUploadPartDetails, error) -type tempLocalObject struct { - f *os.File - path string -} - -var _ io.ReadCloser = &tempLocalObject{} - -func (t *tempLocalObject) Read(p []byte) (int, error) { - return t.f.Read(p) -} - -func (t *tempLocalObject) Close() error { - err := t.f.Close() - os.Remove(t.path) - return err -} - // OCIBlobstore provides an OCI implementation of the Blobstore interface type OCIBlobstore struct { provider common.ConfigurationProvider diff --git a/go/store/config/resolver.go b/go/store/config/resolver.go index 7a733e862e..914cbdfd07 100644 --- a/go/store/config/resolver.go +++ b/go/store/config/resolver.go @@ -26,7 +26,6 @@ import ( "fmt" "strings" - "github.com/dolthub/dolt/go/store/chunks" "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/prolly/tree" "github.com/dolthub/dolt/go/store/spec" @@ -140,16 +139,6 @@ func (r *Resolver) GetDatabase(ctx context.Context, str string) (datas.Database, return sp.GetDatabase(ctx), sp.GetVRW(ctx), sp.GetNodeStore(ctx), nil } -// Resolve string to a chunkstore. Like ResolveDatabase, but returns the underlying ChunkStore -func (r *Resolver) GetChunkStore(ctx context.Context, str string) (chunks.ChunkStore, error) { - dbc := r.DbConfigForDbSpec(str) - sp, err := spec.ForDatabaseOpts(r.verbose(ctx, str, dbc.Url), specOptsForConfig(r.config, dbc)) - if err != nil { - return nil, err - } - return sp.NewChunkStore(ctx), nil -} - // Resolve string to a dataset. If a config is present, // - if no db prefix is present, assume the default db // - if the db prefix is an alias, replace it @@ -178,9 +167,3 @@ func (r *Resolver) GetPath(ctx context.Context, str string) (datas.Database, typ return sp.GetDatabase(ctx), sp.GetVRW(ctx), value, nil } - -// GetDatabaseSpecForPath returns the database and a VRW for the path given, but does not attempt to load a value -func (r *Resolver) GetDatabaseSpecForPath(ctx context.Context, str string) (spec.Spec, error) { - specStr, dbc := r.ResolvePathSpecAndGetDbConfig(str) - return spec.ForPathOpts(r.verbose(ctx, str, specStr), specOptsForConfig(r.config, dbc)) -} diff --git a/go/store/datas/commit.go b/go/store/datas/commit.go index 02e71885e7..4d32b474bd 100644 --- a/go/store/datas/commit.go +++ b/go/store/datas/commit.go @@ -31,7 +31,6 @@ import ( "github.com/dolthub/dolt/go/gen/fb/serial" "github.com/dolthub/dolt/go/store/chunks" - "github.com/dolthub/dolt/go/store/d" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/nomdl" "github.com/dolthub/dolt/go/store/prolly/tree" @@ -741,13 +740,6 @@ func makeCommitStructType(metaType, parentsType, parentsListType, parentsClosure } } -func getRefElementType(t *types.Type) *types.Type { - // precondition checks - d.PanicIfFalse(t.TargetKind() == types.RefKind) - - return t.Desc.(types.CompoundDesc).ElemTypes[0] -} - func firstError(l, r error) error { if l != nil { return l diff --git a/go/store/datas/commit_test.go b/go/store/datas/commit_test.go index 5369a03ab2..a12036e508 100644 --- a/go/store/datas/commit_test.go +++ b/go/store/datas/commit_test.go @@ -47,13 +47,6 @@ func mustHead(ds Dataset) types.Value { return s } -func mustHeight(ds Dataset) uint64 { - h, ok, err := ds.MaybeHeight() - d.PanicIfError(err) - d.PanicIfFalse(ok) - return h -} - func mustHeadValue(ds Dataset) types.Value { val, ok, err := ds.MaybeHeadValue() if err != nil { @@ -70,11 +63,6 @@ func mustString(str string, err error) string { return str } -func mustStruct(st types.Struct, err error) types.Struct { - d.PanicIfError(err) - return st -} - func mustSet(s types.Set, err error) types.Set { d.PanicIfError(err) return s @@ -85,11 +73,6 @@ func mustList(l types.List, err error) types.List { return l } -func mustMap(m types.Map, err error) types.Map { - d.PanicIfError(err) - return m -} - func mustParentsClosure(t *testing.T, exists bool) func(types.Ref, bool, error) types.Ref { return func(r types.Ref, got bool, err error) types.Ref { t.Helper() @@ -114,11 +97,6 @@ func mustValue(val types.Value, err error) types.Value { return val } -func mustTuple(val types.Tuple, err error) types.Tuple { - d.PanicIfError(err) - return val -} - func TestNewCommit(t *testing.T) { assert := assert.New(t) @@ -279,20 +257,6 @@ func mustCommitToTargetHashes(vrw types.ValueReadWriter, commits ...types.Value) return ret } -// Convert list of Struct's to List -func toRefList(vrw types.ValueReadWriter, commits ...types.Struct) (types.List, error) { - l, err := types.NewList(context.Background(), vrw) - if err != nil { - return types.EmptyList, err - } - - le := l.Edit() - for _, p := range commits { - le = le.Append(mustRef(types.NewRef(p, vrw.Format()))) - } - return le.List(context.Background()) -} - func commonAncWithSetClosure(ctx context.Context, c1, c2 *Commit, vr1, vr2 types.ValueReader, ns1, ns2 tree.NodeStore) (a hash.Hash, ok bool, err error) { closure, err := NewSetCommitClosure(ctx, vr1, c1) if err != nil { diff --git a/go/store/datas/dataset.go b/go/store/datas/dataset.go index 75f94f16c5..bebb2a1d8f 100644 --- a/go/store/datas/dataset.go +++ b/go/store/datas/dataset.go @@ -169,6 +169,7 @@ type RebaseState struct { commitBecomesEmptyHandling uint8 emptyCommitHandling uint8 rebasingStarted bool + skipVerification bool } func (rs *RebaseState) PreRebaseWorkingAddr() hash.Hash { @@ -206,6 +207,10 @@ func (rs *RebaseState) EmptyCommitHandling(_ context.Context) uint8 { return rs.emptyCommitHandling } +func (rs *RebaseState) SkipVerification(_ context.Context) bool { + return rs.skipVerification +} + type MergeState struct { preMergeWorkingAddr *hash.Hash fromCommitAddr *hash.Hash @@ -457,6 +462,7 @@ func (h serialWorkingSetHead) HeadWorkingSet() (*WorkingSetHead, error) { rebaseState.EmptyCommitHandling(), rebaseState.LastAttemptedStep(), rebaseState.RebasingStarted(), + rebaseState.SkipVerification(), ) } @@ -525,74 +531,10 @@ func newStatisticHead(sm types.SerialMessage, addr hash.Hash) serialStashListHea return serialStashListHead{sm, addr} } -type statisticsHead struct { - msg types.SerialMessage - addr hash.Hash -} - -var _ dsHead = statisticsHead{} - -// TypeName implements dsHead -func (s statisticsHead) TypeName() string { - return "Statistics" -} - -// Addr implements dsHead -func (s statisticsHead) Addr() hash.Hash { - return s.addr -} - -// HeadTag implements dsHead -func (s statisticsHead) HeadTag() (*TagMeta, hash.Hash, error) { - return nil, hash.Hash{}, errors.New("HeadTag called on statistic") -} - -// HeadWorkingSet implements dsHead -func (s statisticsHead) HeadWorkingSet() (*WorkingSetHead, error) { - return nil, errors.New("HeadWorkingSet called on statistic") -} - -// value implements dsHead -func (s statisticsHead) value() types.Value { - return s.msg -} - func newTupleHead(sm types.SerialMessage, addr hash.Hash) serialStashListHead { return serialStashListHead{sm, addr} } -type tupleHead struct { - msg types.SerialMessage - addr hash.Hash -} - -var _ dsHead = tupleHead{} - -// TypeName implements dsHead -func (s tupleHead) TypeName() string { - return "Tuple" -} - -// Addr implements dsHead -func (s tupleHead) Addr() hash.Hash { - return s.addr -} - -// HeadTag implements dsHead -func (s tupleHead) HeadTag() (*TagMeta, hash.Hash, error) { - return nil, hash.Hash{}, errors.New("HeadTag called on tuple") -} - -// HeadWorkingSet implements dsHead -func (s tupleHead) HeadWorkingSet() (*WorkingSetHead, error) { - return nil, errors.New("HeadWorkingSet called on statistic") -} - -// value implements dsHead -func (s tupleHead) value() types.Value { - return s.msg -} - // Dataset is a named value within a Database. Different head values may be stored in a dataset. Most commonly, this is // a commit, but other values are also supported in some cases. type Dataset struct { diff --git a/go/store/datas/stashlist.go b/go/store/datas/stashlist.go index 7418ea7d7f..3654b5af1a 100644 --- a/go/store/datas/stashlist.go +++ b/go/store/datas/stashlist.go @@ -142,18 +142,6 @@ func (s *StashList) getStashAtIdx(ctx context.Context, idx int) (hash.Hash, erro return stash.addr, nil } -// IsStashList determines whether the types.Value is a stash list object. -func IsStashList(v types.Value) (bool, error) { - if _, ok := v.(types.Struct); ok { - // this should not return true as stash is not supported for old format - return false, nil - } else if sm, ok := v.(types.SerialMessage); ok { - return serial.GetFileID(sm) == serial.StashListFileID, nil - } else { - return false, nil - } -} - // GetStashAtIdx returns hash address of stash at given index in the stash list. func GetStashAtIdx(ctx context.Context, ns tree.NodeStore, val types.Value, idx int) (hash.Hash, error) { stashList, err := getExistingStashList(ctx, ns, val) diff --git a/go/store/datas/statistics.go b/go/store/datas/statistics.go index e5de7cfe78..4bd4e443ef 100644 --- a/go/store/datas/statistics.go +++ b/go/store/datas/statistics.go @@ -49,18 +49,6 @@ func (s *Statistics) Count() (int, error) { return s.m.Count() } -// IsStatistic determines whether the types.Value is a stash list object. -func IsStatistic(v types.Value) (bool, error) { - if _, ok := v.(types.Struct); ok { - // this should not return true as stash is not supported for old format - return false, nil - } else if sm, ok := v.(types.SerialMessage); ok { - return serial.GetFileID(sm) == serial.StatisticFileID, nil - } else { - return false, nil - } -} - // LoadStatistics attempts to dereference a database's statistics Dataset into a typed Statistics object. func LoadStatistics(ctx context.Context, nbf *types.NomsBinFormat, ns tree.NodeStore, vr types.ValueReader, ds Dataset) (*Statistics, error) { if !nbf.UsesFlatbuffers() { diff --git a/go/store/datas/tuple.go b/go/store/datas/tuple.go index 947c7a0ea0..2016eb2ec3 100644 --- a/go/store/datas/tuple.go +++ b/go/store/datas/tuple.go @@ -35,18 +35,6 @@ func (t Tuple) Bytes() []byte { return t.val } -// IsTuple determines whether the types.Value is a tuple -func IsTuple(v types.Value) (bool, error) { - if _, ok := v.(types.Struct); ok { - // this should not return true as stash is not supported for old format - return false, nil - } else if sm, ok := v.(types.SerialMessage); ok { - return serial.GetFileID(sm) == serial.StatisticFileID, nil - } else { - return false, nil - } -} - // LoadTuple attempts to dereference a database's Tuple Dataset into a typed Tuple object. func LoadTuple(ctx context.Context, nbf *types.NomsBinFormat, ns tree.NodeStore, vr types.ValueReader, ds Dataset) (*Tuple, error) { if !nbf.UsesFlatbuffers() { diff --git a/go/store/datas/workingset.go b/go/store/datas/workingset.go index 05ec22dce3..f9e784ae2d 100755 --- a/go/store/datas/workingset.go +++ b/go/store/datas/workingset.go @@ -196,6 +196,7 @@ func workingset_flatbuffer(working hash.Hash, staged *hash.Hash, mergeState *Mer serial.RebaseStateAddEmptyCommitHandling(builder, rebaseState.emptyCommitHandling) serial.RebaseStateAddLastAttemptedStep(builder, rebaseState.lastAttemptedStep) serial.RebaseStateAddRebasingStarted(builder, rebaseState.rebasingStarted) + serial.RebaseStateAddSkipVerification(builder, rebaseState.skipVerification) rebaseStateOffset = serial.RebaseStateEnd(builder) } @@ -264,7 +265,7 @@ func NewMergeState( } } -func NewRebaseState(preRebaseWorkingRoot hash.Hash, commitAddr hash.Hash, branch string, commitBecomesEmptyHandling uint8, emptyCommitHandling uint8, lastAttemptedStep float32, rebasingStarted bool) *RebaseState { +func NewRebaseState(preRebaseWorkingRoot hash.Hash, commitAddr hash.Hash, branch string, commitBecomesEmptyHandling uint8, emptyCommitHandling uint8, lastAttemptedStep float32, rebasingStarted bool, skipVerification bool) *RebaseState { return &RebaseState{ preRebaseWorkingAddr: &preRebaseWorkingRoot, ontoCommitAddr: &commitAddr, @@ -273,6 +274,7 @@ func NewRebaseState(preRebaseWorkingRoot hash.Hash, commitAddr hash.Hash, branch emptyCommitHandling: emptyCommitHandling, lastAttemptedStep: lastAttemptedStep, rebasingStarted: rebasingStarted, + skipVerification: skipVerification, } } diff --git a/go/store/nbs/aws_table_persister_test.go b/go/store/nbs/aws_table_persister_test.go index 71ceaf724e..17ceb398a6 100644 --- a/go/store/nbs/aws_table_persister_test.go +++ b/go/store/nbs/aws_table_persister_test.go @@ -34,7 +34,6 @@ import ( "github.com/stretchr/testify/require" dherrors "github.com/dolthub/dolt/go/libraries/utils/errors" - "github.com/dolthub/dolt/go/store/hash" ) func randomChunks(t *testing.T, r *rand.Rand, sz int) [][]byte { @@ -159,30 +158,6 @@ func TestAWSTablePersisterPersist(t *testing.T) { }) } -type waitOnStoreTableCache struct { - readers map[hash.Hash]io.ReaderAt - mu sync.RWMutex - storeWG sync.WaitGroup -} - -func (mtc *waitOnStoreTableCache) checkout(h hash.Hash) (io.ReaderAt, error) { - mtc.mu.RLock() - defer mtc.mu.RUnlock() - return mtc.readers[h], nil -} - -func (mtc *waitOnStoreTableCache) checkin(h hash.Hash) error { - return nil -} - -func (mtc *waitOnStoreTableCache) store(h hash.Hash, data io.Reader, size uint64) error { - defer mtc.storeWG.Done() - mtc.mu.Lock() - defer mtc.mu.Unlock() - mtc.readers[h] = data.(io.ReaderAt) - return nil -} - type failingFakeS3 struct { *fakeS3 mu sync.Mutex diff --git a/go/store/nbs/journal_index_record.go b/go/store/nbs/journal_index_record.go index 6fbb9557e4..19509465c1 100644 --- a/go/store/nbs/journal_index_record.go +++ b/go/store/nbs/journal_index_record.go @@ -18,11 +18,9 @@ import ( "bufio" "encoding/binary" "errors" - "fmt" "hash/crc32" "io" - "github.com/dolthub/dolt/go/store/d" "github.com/dolthub/dolt/go/store/hash" ) @@ -93,91 +91,6 @@ func journalIndexRecordSize(idx []byte) (recordSz uint32) { return } -func writeJournalIndexRecord(buf []byte, root hash.Hash, start, end uint64, idx []byte) (n uint32) { - //defer trace.StartRegion(ctx, "writeJournalIndexRecord").End() - - // length - l := journalIndexRecordSize(idx) - writeUint32(buf[:indexRecLenSz], l) - n += indexRecLenSz - // last root - buf[n] = byte(lastRootIndexRecTag) - n += indexRecTagSz - copy(buf[n:], root[:]) - n += indexRecLastRootSz - // start offset - buf[n] = byte(startOffsetIndexRecTag) - n += indexRecTagSz - writeUint64(buf[n:], start) - n += indexRecOffsetSz - // end offset - buf[n] = byte(endOffsetIndexRecTag) - n += indexRecTagSz - writeUint64(buf[n:], end) - n += indexRecOffsetSz - // kind - buf[n] = byte(kindIndexRecTag) - n += indexRecTagSz - buf[n] = byte(tableIndexRecKind) - n += indexRecKindSz - // payload - buf[n] = byte(payloadIndexRecTag) - n += indexRecTagSz - copy(buf[n:], idx) - n += uint32(len(idx)) - // checksum - writeUint32(buf[n:], crc(buf[:n])) - n += indexRecChecksumSz - d.PanicIfFalse(l == n) - return -} - -func readJournalIndexRecord(buf []byte) (rec indexRec, err error) { - rec.length = readUint32(buf) - buf = buf[indexRecLenSz:] - for len(buf) > indexRecChecksumSz { - tag := indexRecTag(buf[0]) - buf = buf[indexRecTagSz:] - switch tag { - case lastRootIndexRecTag: - copy(rec.lastRoot[:], buf) - buf = buf[indexRecLastRootSz:] - case startOffsetIndexRecTag: - rec.start = readUint64(buf) - buf = buf[indexRecOffsetSz:] - case endOffsetIndexRecTag: - rec.end = readUint64(buf) - buf = buf[indexRecOffsetSz:] - case kindIndexRecTag: - rec.kind = indexRecKind(buf[0]) - buf = buf[indexRecKindSz:] - case payloadIndexRecTag: - sz := len(buf) - indexRecChecksumSz - rec.payload = buf[:sz] - buf = buf[sz:] - case unknownIndexRecTag: - fallthrough - default: - err = fmt.Errorf("unknown record field tag: %d", tag) - return - } - } - rec.checksum = readUint32(buf[:indexRecChecksumSz]) - return -} - -func validateIndexRecord(buf []byte) bool { - if len(buf) < (indexRecLenSz + indexRecChecksumSz) { - return false - } - off := readUint32(buf) - if int(off) > len(buf) { - return false - } - off -= indexRecChecksumSz - return crc(buf[:off]) == readUint32(buf[off:]) -} - type lookupMeta struct { batchStart int64 batchEnd int64 diff --git a/go/store/nbs/journal_index_record_test.go b/go/store/nbs/journal_index_record_test.go index 15007d4c88..34c3ebbce5 100644 --- a/go/store/nbs/journal_index_record_test.go +++ b/go/store/nbs/journal_index_record_test.go @@ -115,19 +115,3 @@ func TestRoundTripIndexLookupsMeta(t *testing.T) { // do a bunch of iters // use processIndexRecords2 to read back, make sure roots/checksums are consistent, counts, etc } - -func makeLookups(cnt int) (lookups []lookup) { - lookups = make([]lookup, cnt) - buf := make([]byte, cnt*hash.ByteLen) - rand.Read(buf) - var off uint64 - for i := range lookups { - copy(lookups[i].a[:], buf) - buf = buf[hash.ByteLen:] - lookups[i].r.Offset = off - l := rand.Uint32() % 1024 - lookups[i].r.Length = l - off += uint64(l) - } - return -} diff --git a/go/store/nbs/journal_writer.go b/go/store/nbs/journal_writer.go index 3cd5b45e5e..8478acca44 100644 --- a/go/store/nbs/journal_writer.go +++ b/go/store/nbs/journal_writer.go @@ -699,14 +699,6 @@ func (idx rangeIndex) novelCount() int { return len(idx.novel) } -func (idx rangeIndex) novelLookups() (lookups []lookup) { - lookups = make([]lookup, 0, len(idx.novel)) - for a, r := range idx.novel { - lookups = append(lookups, lookup{a: toAddr16(a), r: r}) - } - return -} - func (idx rangeIndex) flatten(ctx context.Context) rangeIndex { defer trace.StartRegion(ctx, "flatten journal index").End() trace.Logf(ctx, "map index cached count", "%d", len(idx.cached)) diff --git a/go/store/nbs/journal_writer_test.go b/go/store/nbs/journal_writer_test.go index 546c3f0526..b8e7897c88 100644 --- a/go/store/nbs/journal_writer_test.go +++ b/go/store/nbs/journal_writer_test.go @@ -16,7 +16,6 @@ package nbs import ( "context" - "encoding/base32" "math/rand" "os" "path/filepath" @@ -405,13 +404,6 @@ func TestJournalIndexBootstrap(t *testing.T) { } } -var encoding = base32.NewEncoding("0123456789abcdefghijklmnopqrstuv") - -// encode returns the base32 encoding in the Dolt alphabet. -func encode(data []byte) string { - return encoding.EncodeToString(data) -} - func randomCompressedChunks(cnt int) (compressed map[hash.Hash]CompressedChunk) { compressed = make(map[hash.Hash]CompressedChunk) var buf []byte diff --git a/go/store/nbs/mem_table.go b/go/store/nbs/mem_table.go index 473583d7ba..87c4eb3428 100644 --- a/go/store/nbs/mem_table.go +++ b/go/store/nbs/mem_table.go @@ -212,14 +212,6 @@ func (mt *memTable) getManyCompressed(ctx context.Context, eg *errgroup.Group, r return remaining, gcBehavior_Continue, nil } -func (mt *memTable) extract(ctx context.Context, chunks chan<- extractRecord) error { - for _, hrec := range mt.order { - chunks <- extractRecord{a: *hrec.a, data: mt.chunks[*hrec.a], err: nil} - } - - return nil -} - func (mt *memTable) write(haver chunkReader, keeper keeperF, stats *Stats) (name hash.Hash, data []byte, splitOffset uint64, chunkCount uint32, gcb gcBehavior, err error) { gcb = gcBehavior_Continue numChunks := uint64(len(mt.order)) diff --git a/go/store/prolly/commit_closure.go b/go/store/prolly/commit_closure.go index 8345d22b31..9f9c98a69c 100644 --- a/go/store/prolly/commit_closure.go +++ b/go/store/prolly/commit_closure.go @@ -122,13 +122,6 @@ func (c CommitClosure) ContainsKey(ctx context.Context, h hash.Hash, height uint return c.closure.Has(ctx, k) } -func DecodeCommitClosureKey(key []byte) (height uint64, addr hash.Hash) { - height = binary.LittleEndian.Uint64(key) - addr = hash.New(key[prefixWidth:]) - - return -} - func (c CommitClosure) AsHashSet(ctx context.Context) (hash.HashSet, error) { closureIter, err := c.IterAllReverse(ctx) if err != nil { diff --git a/go/store/prolly/tuple_range_iter.go b/go/store/prolly/tuple_range_iter.go index 71323de13f..4f843bb7a8 100644 --- a/go/store/prolly/tuple_range_iter.go +++ b/go/store/prolly/tuple_range_iter.go @@ -35,7 +35,6 @@ type rangeIter[K, V ~[]byte] interface { var _ rangeIter[val.Tuple, val.Tuple] = &tree.OrderedTreeIter[val.Tuple, val.Tuple]{} var _ rangeIter[val.Tuple, val.Tuple] = &memRangeIter{} -var _ rangeIter[val.Tuple, val.Tuple] = emptyIter{} // mutableMapIter iterates over a Range of Tuples. type mutableMapIter[K, V ~[]byte, O tree.Ordering[K]] struct { @@ -88,16 +87,6 @@ func (it mutableMapIter[K, V, O]) Next(ctx context.Context) (key K, value V, err } } -func (it mutableMapIter[K, V, O]) currentKeys() (memKey, proKey K) { - if it.memory != nil { - memKey, _ = it.memory.Current() - } - if it.prolly != nil { - proKey, _ = it.prolly.Current() - } - return -} - func (it mutableMapIter[K, V, O]) compareKeys(ctx context.Context, memKey, proKey K) int { if memKey == nil { return 1 @@ -179,16 +168,6 @@ func (it *memRangeIter) Iterate(ctx context.Context) (err error) { } } -type emptyIter struct{} - -func (e emptyIter) Next(context.Context) (val.Tuple, val.Tuple, error) { - return nil, nil, io.EOF -} - -func (e emptyIter) Iterate(ctx context.Context) (err error) { return } - -func (e emptyIter) Current() (key, value val.Tuple) { return } - type filteredIter struct { iter MapIter rng Range diff --git a/go/store/prolly/tuple_range_iter_test.go b/go/store/prolly/tuple_range_iter_test.go index 7b03987ff3..8c3bb53f67 100644 --- a/go/store/prolly/tuple_range_iter_test.go +++ b/go/store/prolly/tuple_range_iter_test.go @@ -95,8 +95,8 @@ func testIterRange(t *testing.T, om testMap, tuples [][2]val.Tuple) { } for _, test := range tests { - //s := fmt.Sprintf(test.testRange.format()) - //fmt.Println(s) + // s := fmt.Sprintf(test.testRange.format()) + // fmt.Println(s) iter, err := om.IterRange(ctx, test.testRange) require.NoError(t, err) @@ -500,38 +500,6 @@ func testIterOrdinalRangeWithBounds(t *testing.T, om Map, tuples [][2]val.Tuple, }) } -func testIterKeyRange(t *testing.T, m Map, tuples [][2]val.Tuple) { - ctx := context.Background() - - t.Run("RandomKeyRange", func(t *testing.T) { - bounds := generateInserts(t, m, m.keyDesc, m.valDesc, 2) - start, stop := bounds[0][0], bounds[1][0] - if m.keyDesc.Compare(ctx, start, stop) > 0 { - start, stop = stop, start - } - kR := keyRange{kd: m.keyDesc, start: start, stop: stop} - - var expectedKeys []val.Tuple - for _, kv := range tuples { - if kR.includes(kv[0]) { - expectedKeys = append(expectedKeys, kv[0]) - } - } - - itr, err := m.IterKeyRange(ctx, start, stop) - require.NoError(t, err) - - for _, eK := range expectedKeys { - k, _, err := itr.Next(ctx) - require.NoError(t, err) - assert.Equal(t, eK, k) - } - - _, _, err = itr.Next(ctx) - require.Equal(t, io.EOF, err) - }) -} - func iterOrdinalRange(t *testing.T, ctx context.Context, iter MapIter) (actual [][2]val.Tuple) { for { k, v, err := iter.Next(ctx) diff --git a/go/store/spec/util.go b/go/store/spec/util.go index 7c224983bc..05d65c80de 100644 --- a/go/store/spec/util.go +++ b/go/store/spec/util.go @@ -23,7 +23,6 @@ package spec import ( "github.com/dolthub/dolt/go/store/d" - "github.com/dolthub/dolt/go/store/hash" ) func CreateDatabaseSpecString(protocol, db string) string { @@ -35,7 +34,3 @@ func CreateValueSpecString(protocol, db, path string) string { d.Chk.NoError(err) return Spec{Protocol: protocol, DatabaseName: db, Path: p}.String() } - -func CreateHashSpecString(protocol, db string, h hash.Hash) string { - return Spec{Protocol: protocol, DatabaseName: db, Path: AbsolutePath{Hash: h}}.String() -} diff --git a/go/store/store_test.go b/go/store/store_test.go index a4faaa835f..52776b88a1 100644 --- a/go/store/store_test.go +++ b/go/store/store_test.go @@ -16,19 +16,16 @@ package store import ( "context" - "io" "math/rand" "os" "sync" "testing" "time" - "github.com/dolthub/go-mysql-server/sql" "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/index" "github.com/dolthub/dolt/go/store/datas" "github.com/dolthub/dolt/go/store/nbs" "github.com/dolthub/dolt/go/store/prolly/tree" @@ -208,46 +205,6 @@ func BenchmarkSimulatedCoveringIndex(b *testing.B) { } } -func BenchmarkMapItr(b *testing.B) { - ctx := context.Background() - generateTestData(ctx) - - require.True(b, b.N < numRows, "b.N:%d >= numRows:%d", b.N, numRows) - - _, vals := readTupleFromDB(ctx, b, simIdxBenchDataset) - m := vals[0].(types.Map) - - itr, err := m.RangeIterator(ctx, 0, uint64(b.N)) - require.NoError(b, err) - - var closeFunc func() error - if cl, ok := itr.(io.Closer); ok { - closeFunc = cl.Close - } - - sch, err := schema.SchemaFromCols(schema.NewColCollection(testDataCols...)) - require.NoError(b, err) - - dmItr := index.NewDoltMapIter(itr.NextTuple, closeFunc, index.NewKVToSqlRowConverterForCols(m.Format(), sch, nil)) - sqlCtx := sql.NewContext(ctx) - - b.ResetTimer() - for { - var r sql.Row - r, err = dmItr.Next(sqlCtx) - - if r == nil || err != nil { - break - } - } - b.StopTimer() - - if err != io.EOF { - require.NoError(b, err) - } - _ = dmItr.Close(sqlCtx) -} - /*func BenchmarkFullScan(b *testing.B) { const dir = "dolt directory containing db with table to scan" const branch = "master" diff --git a/go/store/types/apply_map_edits.go b/go/store/types/apply_map_edits.go index f9e7d7b1b8..60d5eea11a 100644 --- a/go/store/types/apply_map_edits.go +++ b/go/store/types/apply_map_edits.go @@ -99,28 +99,15 @@ type AppliedEditStats struct { NonExistentDeletes int64 } -// Add adds two AppliedEditStats structures member by member. -func (stats AppliedEditStats) Add(other AppliedEditStats) AppliedEditStats { - return AppliedEditStats{ - Additions: stats.Additions + other.Additions, - Modifications: stats.Modifications + other.Modifications, - SameVal: stats.SameVal + other.SameVal, - Deletions: stats.Deletions + other.Deletions, - NonExistentDeletes: stats.NonExistentDeletes + other.NonExistentDeletes, - } -} - // ApplyEdits applies all the edits to a given Map and returns the resulting map, and some statistics about the edits // that were applied. -func ApplyEdits(ctx context.Context, edits EditProvider, m Map) (Map, AppliedEditStats, error) { +func ApplyEdits(ctx context.Context, edits EditProvider, m Map) (Map, error) { return ApplyNEdits(ctx, edits, m, -1) } -func ApplyNEdits(ctx context.Context, edits EditProvider, m Map, numEdits int64) (Map, AppliedEditStats, error) { - var stats AppliedEditStats - +func ApplyNEdits(ctx context.Context, edits EditProvider, m Map, numEdits int64) (Map, error) { if edits.ReachedEOF() { - return m, stats, nil // no edits + return m, nil // no edits } var seq sequence = m.orderedSequence @@ -168,12 +155,10 @@ func ApplyNEdits(ctx context.Context, edits EditProvider, m Map, numEdits int64) } if existingValue == nil && kv.value == nil { - stats.NonExistentDeletes++ continue // already non-present } if existingValue != nil && kv.value != nil && existingValue.Equals(kv.value) { - stats.SameVal++ continue // same value } @@ -193,15 +178,11 @@ func ApplyNEdits(ctx context.Context, edits EditProvider, m Map, numEdits int64) } if existingValue != nil { - stats.Modifications++ err := ch.Skip(ctx) if ae.SetIfError(err) { continue } - - } else { - stats.Additions++ } if kv.value != nil { @@ -219,20 +200,20 @@ func ApplyNEdits(ctx context.Context, edits EditProvider, m Map, numEdits int64) } if ae.IsSet() { - return EmptyMap, AppliedEditStats{}, ae.Get() + return EmptyMap, ae.Get() } if ch == nil { - return m, stats, nil // no edits required application + return m, nil // no edits required application } seq, err := ch.Done(ctx) if err != nil { - return EmptyMap, AppliedEditStats{}, err + return EmptyMap, err } - return newMap(seq.(orderedSequence)), stats, nil + return newMap(seq.(orderedSequence)), nil } // prepWorker will wait for work to be read from a channel, then iterate over all of the edits finding the appropriate diff --git a/go/store/types/float_util.go b/go/store/types/float_util.go index 2799ada5b7..793cbf906b 100644 --- a/go/store/types/float_util.go +++ b/go/store/types/float_util.go @@ -22,7 +22,6 @@ package types import ( - "fmt" "math" ) @@ -34,53 +33,3 @@ func Round(v Value) Value { return val } } - -func Increment(v Value) Value { - switch val := v.(type) { - case Int: - return Int(int64(val) + 1) - case Uint: - return Uint(uint64(val) + 1) - case Float: - return Float(float64(val) + 1) - default: - return val - } -} - -func float64IsInt(f float64) bool { - return math.Trunc(f) == f -} - -// convert float64 to int64 where f == i * 2^exp -func float64ToIntExp(f float64) (int64, int) { - if math.IsNaN(f) || math.IsInf(f, 0) { - panic(fmt.Errorf("%v is not a supported number", f)) - } - - if f == 0 { - return 0, 0 - } - - isNegative := math.Signbit(f) - f = math.Abs(f) - - frac, exp := math.Frexp(f) - // frac is [.5, 1) - // Move frac up until it is an integer. - for !float64IsInt(frac) { - frac *= 2 - exp-- - } - - if isNegative { - frac *= -1 - } - - return int64(frac), exp -} - -// fracExpToFloat returns frac * 2 ** exp -func fracExpToFloat(frac int64, exp int) float64 { - return float64(frac) * math.Pow(2, float64(exp)) -} diff --git a/go/store/types/geometry.go b/go/store/types/geometry.go index c2c5f2d09b..905674bbc1 100644 --- a/go/store/types/geometry.go +++ b/go/store/types/geometry.go @@ -149,11 +149,3 @@ func (v Geometry) skip(nbf *NomsBinFormat, b *binaryNomsReader) { func (v Geometry) HumanReadableString() string { return v.Inner.HumanReadableString() } - -func EncodeGeometryWKB(v Geometry) ([]byte, error) { - wr := &binaryNomsWriter{make([]byte, 128), 0} - if err := v.writeTo(wr, nil); err != nil { - return nil, err - } - return wr.data()[1:], nil // trim NomsKind -} diff --git a/go/store/types/incremental_test.go b/go/store/types/incremental_test.go index 6203807631..6a5831c525 100644 --- a/go/store/types/incremental_test.go +++ b/go/store/types/incremental_test.go @@ -94,90 +94,3 @@ func TestIncrementalLoadList(t *testing.T) { assert.Equal(expectedCount+chunkReads[i], cs.Reads()) } } - -func SkipTestIncrementalLoadSet(t *testing.T) { - assert := assert.New(t) - ts := &chunks.TestStorage{} - cs := ts.NewView() - vs := NewValueStore(cs) - - expected, err := NewSet(context.Background(), vs, getTestVals(vs)...) - require.NoError(t, err) - ref, err := vs.WriteValue(context.Background(), expected) - require.NoError(t, err) - refHash := ref.TargetHash() - - actualVar, err := vs.ReadValue(context.Background(), refHash) - require.NoError(t, err) - actual := actualVar.(Set) - - expectedCount := cs.Reads() - assert.Equal(1, expectedCount) - err = actual.Iter(context.Background(), func(v Value) (bool, error) { - expectedCount += isEncodedOutOfLine(v) - assert.Equal(expectedCount, cs.Reads()) - return false, nil - }) - - require.NoError(t, err) -} - -func SkipTestIncrementalLoadMap(t *testing.T) { - assert := assert.New(t) - ts := &chunks.TestStorage{} - cs := ts.NewView() - vs := NewValueStore(cs) - - expected, err := NewMap(context.Background(), vs, getTestVals(vs)...) - require.NoError(t, err) - ref, err := vs.WriteValue(context.Background(), expected) - require.NoError(t, err) - refHash := ref.TargetHash() - - actualVar, err := vs.ReadValue(context.Background(), refHash) - require.NoError(t, err) - actual := actualVar.(Map) - - expectedCount := cs.Reads() - assert.Equal(1, expectedCount) - err = actual.Iter(context.Background(), func(k, v Value) (bool, error) { - expectedCount += isEncodedOutOfLine(k) - expectedCount += isEncodedOutOfLine(v) - assert.Equal(expectedCount, cs.Reads()) - return false, nil - }) - require.NoError(t, err) -} - -func SkipTestIncrementalAddRef(t *testing.T) { - assert := assert.New(t) - ts := &chunks.TestStorage{} - cs := ts.NewView() - vs := NewValueStore(cs) - - expectedItem := Float(42) - ref, err := vs.WriteValue(context.Background(), expectedItem) - require.NoError(t, err) - - expected, err := NewList(context.Background(), vs, ref) - require.NoError(t, err) - ref, err = vs.WriteValue(context.Background(), expected) - require.NoError(t, err) - actualVar, err := vs.ReadValue(context.Background(), ref.TargetHash()) - require.NoError(t, err) - - assert.Equal(1, cs.Reads()) - assert.True(expected.Equals(actualVar)) - - actual := actualVar.(List) - actualItem, err := actual.Get(context.Background(), 0) - require.NoError(t, err) - assert.Equal(2, cs.Reads()) - assert.True(expectedItem.Equals(actualItem)) - - // do it again to make sure caching works. - actualItem, err = actual.Get(context.Background(), 0) - require.NoError(t, err) - assert.Equal(2, cs.Reads()) - assert.True(expectedItem.Equals(actualItem)) -} diff --git a/go/store/types/json.go b/go/store/types/json.go index 85a308c838..c62e15cd85 100644 --- a/go/store/types/json.go +++ b/go/store/types/json.go @@ -42,16 +42,6 @@ func NewJSONDoc(nbf *NomsBinFormat, vrw ValueReadWriter, value Value) (JSON, err return JSON{valueImpl{vrw, nbf, w.data(), nil}}, nil } -func NewTestJSONDoc(nbf *NomsBinFormat, vrw ValueReadWriter, buf []byte) (JSON, error) { - w := newBinaryNomsWriter() - if err := JSONKind.writeTo(&w, nbf); err != nil { - return emptyJSONDoc(nbf), err - } - - w.writeString(string(buf)) - return JSON{valueImpl{vrw, nbf, w.data(), nil}}, nil -} - // emptyJSONDoc creates and empty JSON value. func emptyJSONDoc(nbf *NomsBinFormat) JSON { w := newBinaryNomsWriter() @@ -148,12 +138,6 @@ func (t JSON) Kind() NomsKind { return JSONKind } -func (t JSON) decoderSkipToFields() (valueDecoder, uint64) { - dec := t.decoder() - dec.skipKind() - return dec, uint64(1) -} - // Len implements the Value interface. func (t JSON) Len() uint64 { // TODO(andy): is this ever 0? diff --git a/go/store/types/map.go b/go/store/types/map.go index 17469fd6b6..50c50ad026 100644 --- a/go/store/types/map.go +++ b/go/store/types/map.go @@ -26,8 +26,6 @@ import ( "errors" "fmt" - "golang.org/x/sync/errgroup" - "github.com/dolthub/dolt/go/store/d" "github.com/dolthub/dolt/go/store/hash" ) @@ -98,98 +96,6 @@ func NewMap(ctx context.Context, vrw ValueReadWriter, kv ...Value) (Map, error) return newMap(seq.(orderedSequence)), nil } -// NewStreamingMap takes an input channel of values and returns a value that -// will produce a finished Map when |.Wait()| is called. Values sent to the -// input channel must be alternating keys and values. (e.g. k1, v1, k2, -// v2...). Moreover keys need to be added to the channel in Noms sortorder, -// adding key values to the input channel out of order will result in an error. -// Once the input channel is closed by the caller, a finished Map will be -// available from the |Wait| call. -// -// See graph_builder.go for building collections with values that are not in -// order. -func NewStreamingMap(ctx context.Context, vrw ValueReadWriter, kvs <-chan Value) *StreamingMap { - d.PanicIfTrue(vrw == nil) - sm := &StreamingMap{} - sm.eg, sm.egCtx = errgroup.WithContext(ctx) - sm.eg.Go(func() error { - m, err := readMapInput(sm.egCtx, vrw, kvs) - sm.m = m - return err - }) - return sm -} - -type StreamingMap struct { - eg *errgroup.Group - egCtx context.Context - m Map -} - -func (sm *StreamingMap) Wait() (Map, error) { - err := sm.eg.Wait() - return sm.m, err -} - -// Done returns a signal channel which is closed once the StreamingMap is no -// longer reading from the key/values channel. A send to the key/value channel -// should be in a select with a read from this channel to ensure that the send -// does not deadlock. -func (sm *StreamingMap) Done() <-chan struct{} { - return sm.egCtx.Done() -} - -func readMapInput(ctx context.Context, vrw ValueReadWriter, kvs <-chan Value) (Map, error) { - ch, err := newEmptyMapSequenceChunker(ctx, vrw) - if err != nil { - return EmptyMap, err - } - - var lastK Value - nextIsKey := true - var k Value -LOOP: - for { - select { - case v, ok := <-kvs: - if !ok { - break LOOP - } - if nextIsKey { - k = v - - if lastK != nil { - isLess, err := lastK.Less(ctx, vrw.Format(), k) - if err != nil { - return EmptyMap, err - } - if !isLess { - return EmptyMap, ErrKeysNotOrdered - } - } - lastK = k - nextIsKey = false - } else { - _, err := ch.Append(ctx, mapEntry{key: k, value: v}) - if err != nil { - return EmptyMap, err - } - - nextIsKey = true - } - case <-ctx.Done(): - return EmptyMap, ctx.Err() - } - } - - seq, err := ch.Done(ctx) - if err != nil { - return EmptyMap, err - } - - return newMap(seq.(orderedSequence)), nil -} - // Diff computes the diff from |last| to |m| using the top-down algorithm, // which completes as fast as possible while taking longer to return early // results than left-to-right. diff --git a/go/store/types/map_editor.go b/go/store/types/map_editor.go index 0c25a0cdb6..0c23f77963 100644 --- a/go/store/types/map_editor.go +++ b/go/store/types/map_editor.go @@ -76,8 +76,7 @@ func (med *MapEditor) Map(ctx context.Context) (Map, error) { return EmptyMap, err } - m, _, err := ApplyEdits(ctx, edits, med.m) - return m, err + return ApplyEdits(ctx, edits, med.m) } // Set adds an edit diff --git a/go/store/types/map_iterator.go b/go/store/types/map_iterator.go index 0bef6c0451..1bb8ec035f 100644 --- a/go/store/types/map_iterator.go +++ b/go/store/types/map_iterator.go @@ -39,16 +39,6 @@ type MapIterator interface { Next(ctx context.Context) (k, v Value, err error) } -type EmptyMapIterator struct{} - -func (mtItr EmptyMapIterator) Next(ctx context.Context) (k, v Value, err error) { - return nil, nil, nil -} - -func (mtItr EmptyMapIterator) NextTuple(ctx context.Context) (k, v Tuple, err error) { - return Tuple{}, Tuple{}, io.EOF -} - // mapIterator can efficiently iterate through a Noms Map. type mapIterator struct { sequenceIter sequenceIterator @@ -136,49 +126,3 @@ func (m Map) RangeIterator(ctx context.Context, startIdx, endIdx uint64) (MapTup return &mapRangeIter{collItr: collItr}, nil } - -// LimitingMapIterator iterates |iter| only returning up to |limit| results. -type LimitingMapIterator struct { - iter MapIterator - limit uint64 - cnt uint64 -} - -var _ MapIterator = (*LimitingMapIterator)(nil) - -// NewLimitingMapIterator returns a *LimitingMapIterator. -func NewLimitingMapIterator(iter MapIterator, limit uint64) *LimitingMapIterator { - return &LimitingMapIterator{ - iter: iter, - limit: limit, - } -} - -// Next implements MapIterator. -func (l *LimitingMapIterator) Next(ctx context.Context) (k, v Value, err error) { - if l.cnt == l.limit { - return nil, nil, nil - } - k, v, err = l.iter.Next(ctx) - if err != nil { - return nil, nil, err - } - if k == nil { - return nil, nil, nil - } - l.cnt++ - return -} - -// NextTuple implements MapIterator. -func (l *LimitingMapIterator) NextTuple(ctx context.Context) (k, v Tuple, err error) { - if l.cnt == l.limit { - return Tuple{}, Tuple{}, io.EOF - } - k, v, err = l.iter.NextTuple(ctx) - if err != nil { - return Tuple{}, Tuple{}, err - } - l.cnt++ - return -} diff --git a/go/store/types/map_test.go b/go/store/types/map_test.go index 19f2d8ecb0..cfedc11f4f 100644 --- a/go/store/types/map_test.go +++ b/go/store/types/map_test.go @@ -27,7 +27,6 @@ import ( "fmt" "math/rand" "sort" - "sync" "testing" "time" @@ -322,67 +321,6 @@ func newMapTestSuite(size uint, expectChunkCount int, expectPrependChunkDiff int } } -func (suite *mapTestSuite) createStreamingMap(vs *ValueStore) Map { - kvChan := make(chan Value) - streamingMap := NewStreamingMap(context.Background(), vs, kvChan) - for _, entry := range suite.elems.entries.entries { - kvChan <- entry.key - kvChan <- entry.value - } - close(kvChan) - m, err := streamingMap.Wait() - suite.NoError(err) - return m -} - -func (suite *mapTestSuite) TestStreamingMap() { - vs := newTestValueStore() - defer vs.Close() - m := suite.createStreamingMap(vs) - suite.True(suite.validate(m), "map not valid") -} - -func (suite *mapTestSuite) TestStreamingMapOrder() { - vs := newTestValueStore() - defer vs.Close() - - entries := mapEntrySlice{make([]mapEntry, len(suite.elems.entries.entries))} - copy(entries.entries, suite.elems.entries.entries) - entries.entries[0], entries.entries[1] = entries.entries[1], entries.entries[0] - - kvChan := make(chan Value, len(entries.entries)*2) - for _, e := range entries.entries { - kvChan <- e.key - kvChan <- e.value - } - close(kvChan) - - sm := NewStreamingMap(context.Background(), vs, kvChan) - _, err := sm.Wait() - - suite.Assert().EqualError(err, ErrKeysNotOrdered.Error()) -} - -func (suite *mapTestSuite) TestStreamingMap2() { - wg := sync.WaitGroup{} - vs := newTestValueStore() - defer vs.Close() - - wg.Add(2) - var m1, m2 Map - go func() { - m1 = suite.createStreamingMap(vs) - wg.Done() - }() - go func() { - m2 = suite.createStreamingMap(vs) - wg.Done() - }() - wg.Wait() - suite.True(suite.validate(m1), "map 'm1' not valid") - suite.True(suite.validate(m2), "map 'm2' not valid") -} - func TestMapSuite4K(t *testing.T) { suite.Run(t, newMapTestSuite(12, 5, 2, 2, newNumber)) } diff --git a/go/store/types/read_geometry.go b/go/store/types/read_geometry.go index d3b23a014f..d6e075eacc 100644 --- a/go/store/types/read_geometry.go +++ b/go/store/types/read_geometry.go @@ -200,10 +200,6 @@ func DeserializeEWKBHeader(buf []byte) (uint32, bool, uint32, error) { return types.DeserializeEWKBHeader(buf) } -func DeserializeWKBHeader(buf []byte) (bool, uint32, error) { - return types.DeserializeWKBHeader(buf) -} - func DeserializePoint(buf []byte, isBig bool, srid uint32) types.Point { p, _, err := types.DeserializePoint(buf, isBig, srid) if err != nil { diff --git a/go/store/types/value.go b/go/store/types/value.go index 7d48301f65..18148bf001 100644 --- a/go/store/types/value.go +++ b/go/store/types/value.go @@ -171,60 +171,6 @@ type valueReadWriter interface { valueReadWriter() ValueReadWriter } -type TupleSlice []Tuple - -func (vs TupleSlice) Equals(other TupleSlice) bool { - if len(vs) != len(other) { - return false - } - - for i, v := range vs { - if !v.Equals(other[i]) { - return false - } - } - - return true -} - -func (vs TupleSlice) Contains(nbf *NomsBinFormat, v Tuple) bool { - for _, v := range vs { - if v.Equals(v) { - return true - } - } - return false -} - -type TupleSort struct { - Tuples []Tuple -} - -func (vs TupleSort) Len() int { - return len(vs.Tuples) -} - -func (vs TupleSort) Swap(i, j int) { - vs.Tuples[i], vs.Tuples[j] = vs.Tuples[j], vs.Tuples[i] -} - -func (vs TupleSort) Less(ctx context.Context, nbf *NomsBinFormat, i, j int) (bool, error) { - res, err := vs.Tuples[i].TupleCompare(ctx, nbf, vs.Tuples[j]) - if err != nil { - return false, err - } - - return res < 0, nil -} - -func (vs TupleSort) Equals(other TupleSort) bool { - return TupleSlice(vs.Tuples).Equals(other.Tuples) -} - -func (vs TupleSort) Contains(nbf *NomsBinFormat, v Tuple) bool { - return TupleSlice(vs.Tuples).Contains(nbf, v) -} - type valueImpl struct { vrw ValueReadWriter nbf *NomsBinFormat diff --git a/go/store/types/value_store.go b/go/store/types/value_store.go index 214e1ebb1d..c92772d114 100644 --- a/go/store/types/value_store.go +++ b/go/store/types/value_store.go @@ -130,11 +130,6 @@ func newTestValueStore() *ValueStore { return NewValueStore(ts.NewViewWithDefaultFormat()) } -func newTestValueStore_LD_1() *ValueStore { - ts := &chunks.TestStorage{} - return NewValueStore(ts.NewView()) -} - // NewMemoryValueStore creates a simple struct that satisfies ValueReadWriter // and is backed by a chunks.TestStore. Used for dolt operations outside of noms. func NewMemoryValueStore() *ValueStore { diff --git a/integration-tests/bats/commit_verification.bats b/integration-tests/bats/commit_verification.bats new file mode 100644 index 0000000000..7a0d9b7b86 --- /dev/null +++ b/integration-tests/bats/commit_verification.bats @@ -0,0 +1,253 @@ +#!/usr/bin/env bats +load $BATS_TEST_DIRNAME/helper/common.bash + +setup() { + setup_common + + dolt sql </dev/null 2>&1; then + skip "git not installed" + fi + + tempDir="$(mktemp -d "${BATS_TMPDIR:-/tmp}/dolt-sql-server-clone-git.XXXXXX")" + trap 'rm -rf "$tempDir"' EXIT + cd "$tempDir" + + # Set up a bare git remote whose path ends with .git and seed it with a branch. + mkdir first_dolt_remote.git + git init --bare first_dolt_remote.git + seed_dir="$(mktemp -d "${BATS_TMPDIR:-/tmp}/seed-repo.XXXXXX")" + ( + set -euo pipefail + trap 'rm -rf "$seed_dir"' EXIT + cd "$seed_dir" + git init >/dev/null + git config user.email "bats@email.fake" + git config user.name "Bats Tests" + echo "seed" > README + git add README + git commit -m "seed" >/dev/null + git branch -M main + git remote add origin "$tempDir/first_dolt_remote.git" + git push origin main >/dev/null + ) + + # Push dolt data to the git remote. + mkdir src + cd src + dolt init + dolt sql -q "create table test(pk int primary key, v int);" + dolt sql -q "insert into test values (1, 111);" + dolt add . + dolt commit -m "seed dolt" + dolt remote add origin "$tempDir/first_dolt_remote.git" + dolt push origin main + + # Start an empty server and clone into it via the stored procedure. + cd "$tempDir" + mkdir empty_server + cd empty_server + start_sql_server + + dolt sql -q "create database hostdb" + run dolt --use-db hostdb sql -q "call dolt_clone('$tempDir/first_dolt_remote.git'); show databases;" + [ "$status" -eq 0 ] + [[ "$output" =~ "first_dolt_remote" ]] || false + [[ ! "$output" =~ "first_dolt_remote.git" ]] || false + + run dolt --use-db first_dolt_remote sql -q "select v from test where pk=1;" -r csv + [ "$status" -eq 0 ] + [[ "$output" =~ "111" ]] || false +} + @test "sql-server: loglevels are case insensitive" { # assert that loglevel on command line is not case sensitive cd repo1