mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-04-24 19:49:43 -05:00
Code Issues Packages Projects Releases 157 Wiki Activity

Merge pull request #10481 from dolthub/zachmu/kill-noms2

Browse Source 
[no-review-notes] Removed most of the remaining LD_1 specific codepaths and types
This commit is contained in:
Zach Musgrave
2026-02-11 14:58:55 -08:00
committed by GitHub
parent 3c2969b73a 2d0493d1f1
commit da07d80472
68 changed files with 277 additions and 30588 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/cmd/dolt/commands/migrate.go
-117
View File
@@ -1,117 +0,0 @@
// Copyright 2019 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package commands
import (
"context"
"os"
eventsapi "github.com/dolthub/eventsapi_schema/dolt/services/eventsapi/v1alpha1"
"github.com/dolthub/dolt/go/cmd/dolt/cli"
"github.com/dolthub/dolt/go/cmd/dolt/errhand"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/migrate"
"github.com/dolthub/dolt/go/libraries/utils/argparser"
"github.com/dolthub/dolt/go/store/types"
)
const (
migrationPrompt = `Run "dolt migrate" to update this database to the latest data format`
migrationMsg = "Migrating database to the latest data format"
migrateDropConflictsFlag = "drop-conflicts"
)
var migrateDocs = cli.CommandDocumentationContent{
ShortDesc: "Executes a database migration to use the latest Dolt data format.",
LongDesc: `Migrate is a multi-purpose command to update the data format of a Dolt database. Over time, development
on Dolt requires changes to the on-disk data format. These changes are necessary to improve Database performance and
correctness. Migrating to the latest format is therefore necessary for compatibility with the latest Dolt clients, and
to take advantage of the newly released Dolt features.`,
Synopsis: []string{
"[ --push ] [ --pull ]",
},
}
type MigrateCmd struct{}
// Name is returns the name of the Dolt cli command. This is what is used on the command line to invoke the command
func (cmd MigrateCmd) Name() string {
return "migrate"
}
// Description returns a description of the command
func (cmd MigrateCmd) Description() string {
return migrateDocs.ShortDesc
}
func (cmd MigrateCmd) Docs() *cli.CommandDocumentation {
return nil
}
func (cmd MigrateCmd) ArgParser() *argparser.ArgParser {
ap := argparser.NewArgParserWithMaxArgs(cmd.Name(), 0)
ap.SupportsFlag(migrateDropConflictsFlag, "", "Drop any conflicts visited during the migration")
return ap
}
// EventType returns the type of the event to log
func (cmd MigrateCmd) EventType() eventsapi.ClientEventType {
return eventsapi.ClientEventType_MIGRATE
}
// Exec executes the command
func (cmd MigrateCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv, cliCtx cli.CliContext) int {
ap := cmd.ArgParser()
help, usage := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, migrateDocs, ap))
apr := cli.ParseArgsOrDie(ap, args, help)
dropConflicts := apr.Contains(migrateDropConflictsFlag)
if err := MigrateDatabase(ctx, dEnv, dropConflicts); err != nil {
verr := errhand.BuildDError("migration failed").AddCause(err).Build()
return HandleVErrAndExitCode(verr, usage)
}
os.Exit(0)
return 0 // unreachable
}
// MigrateDatabase migrates the NomsBinFormat of |dEnv.DoltDB(ctx)|.
func MigrateDatabase(ctx context.Context, dEnv *env.DoltEnv, dropConflicts bool) error {
menv, err := migrate.NewEnvironment(ctx, dEnv)
if err != nil {
return err
}
menv.DropConflicts = dropConflicts
if curr := menv.Existing.DoltDB(ctx).Format(); types.IsFormat_DOLT(curr) {
cli.Println("database is already migrated")
return nil
}
p, err := menv.Migration.FS.Abs(".")
if err != nil {
return err
}
cli.Println("migrating database at tmp dir: ", p)
err = migrate.TraverseDAG(ctx, menv, menv.Existing.DoltDB(ctx), menv.Migration.DoltDB(ctx))
if err != nil {
return err
}
return migrate.SwapChunkStores(ctx, menv)
}
go/cmd/dolt/dolt.go
-2
View File
@@ -74,7 +74,6 @@ var commandsWithoutCliCtx = []cli.Command{
credcmds.Commands,
cvcmds.Commands,
commands.SendMetricsCmd{},
commands.MigrateCmd{},
indexcmds.Commands,
commands.ReadTablesCmd{},
commands.FilterBranchCmd{},
@@ -96,7 +95,6 @@ var commandsWithoutGlobalArgSupport = []cli.Command{
commands.InitCmd{},
commands.CloneCmd{},
docscmds.Commands,
commands.MigrateCmd{},
commands.ReadTablesCmd{},
commands.LoginCmd{},
credcmds.Commands,
go/cmd/dolt/doltcmd/doltcmd.go
-1
View File
@@ -68,7 +68,6 @@ var doltSubCommands = []cli.Command{
commands.BlameCmd{},
cvcmds.Commands,
commands.SendMetricsCmd{},
commands.MigrateCmd{},
indexcmds.Commands,
commands.ReadTablesCmd{},
commands.GarbageCollectionCmd{},
go/libraries/doltcore/diff/async_differ.go
-16
View File
@@ -22,27 +22,11 @@ import (
"golang.org/x/sync/errgroup"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/utils/async"
"github.com/dolthub/dolt/go/store/diff"
"github.com/dolthub/dolt/go/store/types"
)
func NewRowDiffer(ctx context.Context, format *types.NomsBinFormat, fromSch, toSch schema.Schema, buf int) RowDiffer {
ad := NewAsyncDiffer(buf)
// Returns an EmptyRowDiffer if the two schemas are not diffable.
if !schema.ArePrimaryKeySetsDiffable(format, fromSch, toSch) {
return &EmptyRowDiffer{}
}
if schema.IsKeyless(fromSch) || schema.IsKeyless(toSch) {
return &keylessDiffer{AsyncDiffer: ad}
}
return ad
}
// todo: make package private
type AsyncDiffer struct {
diffChan chan diff.Difference
go/libraries/doltcore/diff/diff_source.go
-111
View File
@@ -14,17 +14,6 @@
package diff
import (
"errors"
"io"
"time"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/rowconv"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/types"
)
const (
From = "from"
To = "to"
@@ -37,103 +26,3 @@ func ToColNamer(name string) string {
func FromColNamer(name string) string {
return From + "_" + name
}
type RowDiffSource struct {
ad RowDiffer
joiner *rowconv.Joiner
oldRowConv *rowconv.RowConverter
newRowConv *rowconv.RowConverter
warnFn rowconv.WarnFunction
}
func NewRowDiffSource(ad RowDiffer, joiner *rowconv.Joiner, warnFn rowconv.WarnFunction) *RowDiffSource {
return &RowDiffSource{
ad: ad,
joiner: joiner,
oldRowConv: rowconv.IdentityConverter,
newRowConv: rowconv.IdentityConverter,
warnFn: warnFn,
}
}
func (rdRd *RowDiffSource) AddInputRowConversion(oldConv, newConv *rowconv.RowConverter) {
rdRd.oldRowConv = oldConv
rdRd.newRowConv = newConv
}
// GetSchema gets the schema of the rows that this reader will return
func (rdRd *RowDiffSource) GetSchema() schema.Schema {
return rdRd.joiner.GetSchema()
}
// NextDiff reads a row from a table. If there is a bad row the returned error will be non nil, and calling IsBadRow(err)
// will be return true. This is a potentially non-fatal error and callers can decide if they want to continue on a bad row, or fail.
func (rdRd *RowDiffSource) NextDiff() (row.Row, error) {
diffs, hasMore, err := rdRd.ad.GetDiffs(1, time.Second)
if err != nil {
return nil, err
}
if len(diffs) == 0 {
if !hasMore {
return nil, io.EOF
}
return nil, errors.New("timeout")
}
if len(diffs) != 1 {
panic("only a single diff requested, multiple returned. bug in AsyncDiffer")
}
d := diffs[0]
rows := make(map[string]row.Row)
if d.OldValue != nil {
sch := rdRd.joiner.SchemaForName(From)
if !rdRd.oldRowConv.IdentityConverter {
sch = rdRd.oldRowConv.SrcSch
}
oldRow, err := row.FromNoms(sch, d.KeyValue.(types.Tuple), d.OldValue.(types.Tuple))
if err != nil {
return nil, err
}
rows[From], err = rdRd.oldRowConv.ConvertWithWarnings(oldRow, rdRd.warnFn)
if err != nil {
return nil, err
}
}
if d.NewValue != nil {
sch := rdRd.joiner.SchemaForName(To)
if !rdRd.newRowConv.IdentityConverter {
sch = rdRd.newRowConv.SrcSch
}
newRow, err := row.FromNoms(sch, d.KeyValue.(types.Tuple), d.NewValue.(types.Tuple))
if err != nil {
return nil, err
}
rows[To], err = rdRd.newRowConv.ConvertWithWarnings(newRow, rdRd.warnFn)
if err != nil {
return nil, err
}
}
joinedRow, err := rdRd.joiner.Join(rows)
if err != nil {
return nil, err
}
return joinedRow, nil
}
// Close should release resources being held
func (rdRd *RowDiffSource) Close() error {
rdRd.ad.Close()
return nil
}
go/libraries/doltcore/diff/diff_stat.go
+4 -38
View File
@@ -62,11 +62,8 @@ func Stat(ctx context.Context, ch chan DiffStatProgress, from, to durable.Index,
return fmt.Errorf("cannot perform a diff between keyless and keyed schema")
}
if types.IsFormat_DOLT(from.Format()) {
return diffProllyTrees(ctx, ch, keyless, from, to, fromSch, toSch)
}
return diffNomsMaps(ctx, ch, keyless, from, to, fromSch, toSch)
types.AssertFormat_DOLT(from.Format())
return diffProllyTrees(ctx, ch, keyless, from, to, fromSch, toSch)
}
// StatForTableDelta pushes diff stat progress messages for the table delta given to the channel given
@@ -88,7 +85,7 @@ func StatForTableDelta(ctx context.Context, ch chan DiffStatProgress, td TableDe
return errhand.BuildDError("cannot retrieve schema for table %s", td.ToName).AddCause(err).Build()
}
if !schema.ArePrimaryKeySetsDiffable(td.Format(), fromSch, toSch) {
if !schema.ArePrimaryKeySetsDiffable(fromSch, toSch) {
return fmt.Errorf("failed to compute diff stat for table %s: %w", td.CurName(), ErrPrimaryKeySetChanged)
}
@@ -102,11 +99,7 @@ func StatForTableDelta(ctx context.Context, ch chan DiffStatProgress, td TableDe
return err
}
if types.IsFormat_DOLT(td.Format()) {
return diffProllyTrees(ctx, ch, keyless, fromRows, toRows, fromSch, toSch)
} else {
return diffNomsMaps(ctx, ch, keyless, fromRows, toRows, fromSch, toSch)
}
return diffProllyTrees(ctx, ch, keyless, fromRows, toRows, fromSch, toSch)
}
func diffProllyTrees(ctx context.Context, ch chan DiffStatProgress, keyless bool, from, to durable.Index, fromSch, toSch schema.Schema) error {
@@ -175,33 +168,6 @@ func diffProllyTrees(ctx context.Context, ch chan DiffStatProgress, keyless bool
return nil
}
func diffNomsMaps(ctx context.Context, ch chan DiffStatProgress, keyless bool, fromRows durable.Index, toRows durable.Index, fromSch, toSch schema.Schema) error {
var rpr nomsReporter
if keyless {
rpr = reportNomsKeylessChanges
} else {
fc, err := fromRows.Count()
if err != nil {
return err
}
cfc := uint64(len(fromSch.GetAllCols().GetColumns())) * fc
tc, err := toRows.Count()
if err != nil {
return err
}
ctc := uint64(len(toSch.GetAllCols().GetColumns())) * tc
rpr = reportNomsPkChanges
ch <- DiffStatProgress{
OldRowSize: fc,
NewRowSize: tc,
OldCellSize: cfc,
NewCellSize: ctc,
}
}
return statWithReporter(ctx, ch, durable.NomsMapFromIndex(fromRows), durable.NomsMapFromIndex(toRows), rpr, fromSch, toSch)
}
func statWithReporter(ctx context.Context, ch chan DiffStatProgress, from, to types.Map, rpr nomsReporter, fromSch, toSch schema.Schema) (err error) {
ad := NewAsyncDiffer(1024)
ad.Start(ctx, from, to)
go/libraries/doltcore/diff/table_deltas.go
+1 -12
View File
@@ -594,7 +594,7 @@ func (td TableDelta) HasDataChanged(ctx context.Context) (bool, error) {
}
func (td TableDelta) HasPrimaryKeySetChanged() bool {
return !schema.ArePrimaryKeySetsDiffable(td.Format(), td.FromSch, td.ToSch)
return !schema.ArePrimaryKeySetsDiffable(td.FromSch, td.ToSch)
}
func (td TableDelta) HasChanges() (bool, error) {
@@ -658,17 +658,6 @@ func (td TableDelta) GetSchemas(ctx context.Context) (from, to schema.Schema, er
return td.FromSch, td.ToSch, nil
}
// Format returns the format of the tables in this delta.
func (td TableDelta) Format() *types.NomsBinFormat {
if td.FromRootObject != nil || td.ToRootObject != nil {
return types.Format_DOLT
}
if td.FromTable != nil {
return td.FromTable.Format()
}
return td.ToTable.Format()
}
func (td TableDelta) IsKeyless(ctx context.Context) (bool, error) {
f, t, err := td.GetSchemas(ctx)
if err != nil {
go/libraries/doltcore/doltdb/doltdb_test.go
+1 -35
View File
@@ -27,7 +27,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/libraries/utils/test"
@@ -81,39 +80,6 @@ func CreateTestTable(vrw types.ValueReadWriter, ns tree.NodeStore, tSchema schem
return tbl, nil
}
func createTestRowData(t *testing.T, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) durable.Index {
if types.Format_Default == types.Format_DOLT {
idx, err := durable.NewEmptyPrimaryIndex(context.Background(), vrw, ns, sch)
require.NoError(t, err)
return idx
}
vals := []row.TaggedValues{
{idTag: types.UUID(id0), firstTag: types.String("bill"), lastTag: types.String("billerson"), ageTag: types.Uint(53)},
{idTag: types.UUID(id1), firstTag: types.String("eric"), lastTag: types.String("ericson"), isMarriedTag: types.Bool(true), ageTag: types.Uint(21)},
{idTag: types.UUID(id2), firstTag: types.String("john"), lastTag: types.String("johnson"), isMarriedTag: types.Bool(false), ageTag: types.Uint(53)},
{idTag: types.UUID(id3), firstTag: types.String("robert"), lastTag: types.String("robertson"), ageTag: types.Uint(36)},
}
var err error
rows := make([]row.Row, len(vals))
m, err := types.NewMap(context.Background(), vrw)
assert.NoError(t, err)
ed := m.Edit()
for i, val := range vals {
r, err := row.New(vrw.Format(), sch, val)
require.NoError(t, err)
rows[i] = r
ed = ed.Set(r.NomsMapKey(sch), r.NomsMapValue(sch))
}
m, err = ed.Map(context.Background())
assert.NoError(t, err)
return durable.IndexFromNomsMap(m, vrw, ns)
}
func TestIsValidTableName(t *testing.T) {
assert.True(t, IsValidTableName("a"))
assert.True(t, IsValidTableName("a1"))
@@ -271,7 +237,7 @@ func TestLDNoms(t *testing.T) {
}
}
//read the empty repo back and add a new table. Write the value, but don't commit
// read the empty repo back and add a new table. Write the value, but don't commit
var valHash hash.Hash
var tbl *Table
{
go/libraries/doltcore/doltdb/durable/artifact_index.go
+4 -22
View File
@@ -30,7 +30,6 @@ import (
type ArtifactIndex interface {
HashOf() (hash.Hash, error)
Count() (uint64, error)
Format() *types.NomsBinFormat
HasConflicts(ctx context.Context) (bool, error)
// ConflictCount returns the number of conflicts
ConflictCount(ctx context.Context) (uint64, error)
@@ -43,24 +42,15 @@ type ArtifactIndex interface {
// RefFromArtifactIndex persists |idx| and returns the types.Ref targeting it.
func RefFromArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, idx ArtifactIndex) (types.Ref, error) {
switch idx.Format() {
case types.Format_LD_1:
panic("TODO")
case types.Format_DOLT:
b := shim.ValueFromMap(idx.(prollyArtifactIndex).index)
return refFromNomsValue(ctx, vrw, b)
default:
return types.Ref{}, errNbfUnknown
}
b := shim.ValueFromMap(idx.(prollyArtifactIndex).index)
return vrw.WriteValue(ctx, b)
}
// NewEmptyArtifactIndex returns an ArtifactIndex with no artifacts.
func NewEmptyArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, tableSch schema.Schema) (ArtifactIndex, error) {
switch vrw.Format() {
case types.Format_LD_1:
panic("TODO")
panic("Unsupported format " + vrw.Format().VersionString())
case types.Format_DOLT:
kd := tableSch.GetKeyDescriptor(ns)
@@ -85,10 +75,6 @@ func ProllyMapFromArtifactIndex(i ArtifactIndex) prolly.ArtifactMap {
return i.(prollyArtifactIndex).index
}
func artifactIndexFromRef(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, tableSch schema.Schema, r types.Ref) (ArtifactIndex, error) {
return artifactIndexFromAddr(ctx, vrw, ns, tableSch, r.TargetHash())
}
func artifactIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, tableSch schema.Schema, addr hash.Hash) (ArtifactIndex, error) {
v, err := vrw.MustReadValue(ctx, addr)
if err != nil {
@@ -97,7 +83,7 @@ func artifactIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tr
switch vrw.Format() {
case types.Format_LD_1:
panic("TODO")
panic("Unsupported format " + vrw.Format().VersionString())
case types.Format_DOLT:
root, fileId, err := shim.NodeFromValue(v)
@@ -129,10 +115,6 @@ func (i prollyArtifactIndex) Count() (uint64, error) {
return uint64(c), err
}
func (i prollyArtifactIndex) Format() *types.NomsBinFormat {
return i.index.Format()
}
func (i prollyArtifactIndex) HasConflicts(ctx context.Context) (bool, error) {
return i.index.HasArtifactOfType(ctx, prolly.ArtifactTypeConflict)
}
go/libraries/doltcore/doltdb/durable/conflict_index.go
-113
View File
@@ -1,113 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package durable
import (
"context"
"fmt"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
type ConflictIndex interface {
HashOf() (hash.Hash, error)
Count() uint64
Format() *types.NomsBinFormat
}
// RefFromConflictIndex persists |idx| and returns the types.Ref targeting it.
func RefFromConflictIndex(ctx context.Context, vrw types.ValueReadWriter, idx ConflictIndex) (types.Ref, error) {
switch idx.Format() {
case types.Format_LD_1:
return refFromNomsValue(ctx, vrw, idx.(nomsConflictIndex).index)
case types.Format_DOLT:
return types.Ref{}, fmt.Errorf("__DOLT__ conflicts should be stored in ArtifactIndex")
default:
return types.Ref{}, errNbfUnknown
}
}
// NewEmptyConflictIndex returns an ConflictIndex with no rows.
func NewEmptyConflictIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, oursSch, theirsSch, baseSch schema.Schema) (ConflictIndex, error) {
switch vrw.Format() {
case types.Format_LD_1:
m, err := types.NewMap(ctx, vrw)
if err != nil {
return nil, err
}
return ConflictIndexFromNomsMap(m, vrw), nil
case types.Format_DOLT:
return nil, fmt.Errorf("__DOLT__ conflicts should be stored in ArtifactIndex")
default:
return nil, errNbfUnknown
}
}
func ConflictIndexFromNomsMap(m types.Map, vrw types.ValueReadWriter) ConflictIndex {
return nomsConflictIndex{
index: m,
vrw: vrw,
}
}
func NomsMapFromConflictIndex(i ConflictIndex) types.Map {
return i.(nomsConflictIndex).index
}
func conflictIndexFromRef(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, ourSch, theirSch, baseSch schema.Schema, r types.Ref) (ConflictIndex, error) {
return conflictIndexFromAddr(ctx, vrw, ns, ourSch, theirSch, baseSch, r.TargetHash())
}
func conflictIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, ourSch, theirSch, baseSch schema.Schema, addr hash.Hash) (ConflictIndex, error) {
v, err := vrw.ReadValue(ctx, addr)
if err != nil {
return nil, err
}
switch vrw.Format() {
case types.Format_LD_1:
return ConflictIndexFromNomsMap(v.(types.Map), vrw), nil
case types.Format_DOLT:
return nil, fmt.Errorf("__DOLT__ conflicts should be stored in ArtifactIndex")
default:
return nil, errNbfUnknown
}
}
type nomsConflictIndex struct {
index types.Map
vrw types.ValueReadWriter
}
func (i nomsConflictIndex) HashOf() (hash.Hash, error) {
return i.index.Hash(i.vrw.Format())
}
func (i nomsConflictIndex) Count() uint64 {
return i.index.Len()
}
func (i nomsConflictIndex) Format() *types.NomsBinFormat {
return i.vrw.Format()
}
go/libraries/doltcore/doltdb/durable/index.go
+6 -180
View File
@@ -71,10 +71,6 @@ type IndexSet interface {
// PutIndex puts an index into the set.
PutIndex(ctx context.Context, name string, idx Index) (IndexSet, error)
// PutNomsIndex puts a noms index into the set.
// todo(andy): this is a temporary stop-gap while abstracting types.Map
PutNomsIndex(ctx context.Context, name string, idx types.Map) (IndexSet, error)
// DropIndex removes an index from the set.
DropIndex(ctx context.Context, name string) (IndexSet, error)
@@ -86,11 +82,10 @@ type IndexSet interface {
func RefFromIndex(ctx context.Context, vrw types.ValueReadWriter, idx Index) (types.Ref, error) {
switch idx.Format() {
case types.Format_LD_1:
return refFromNomsValue(ctx, vrw, idx.(nomsIndex).index)
panic("Unsupported format " + idx.Format().VersionString())
case types.Format_DOLT:
b := shim.ValueFromMap(MapFromIndex(idx))
return refFromNomsValue(ctx, vrw, b)
return vrw.WriteValue(ctx, b)
default:
return types.Ref{}, errNbfUnknown
@@ -111,7 +106,7 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
switch vrw.Format() {
case types.Format_LD_1:
return IndexFromNomsMap(v.(types.Map), vrw, ns), nil
panic("Unsupported format " + vrw.Format().VersionString())
case types.Format_DOLT:
m, err := shim.MapInterfaceFromValue(ctx, v, sch, ns, isKeylessTable)
@@ -146,11 +141,7 @@ func NewEmptyIndexFromTableSchema(ctx context.Context, vrw types.ValueReadWriter
func newEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, isVector bool, isKeylessSecondary bool) (Index, error) {
switch vrw.Format() {
case types.Format_LD_1:
m, err := types.NewMap(ctx, vrw)
if err != nil {
return nil, err
}
return IndexFromNomsMap(m, vrw, ns), nil
panic("Unsupported format " + vrw.Format().VersionString())
case types.Format_DOLT:
kd, vd := sch.GetMapDescriptors(ns)
@@ -188,14 +179,6 @@ func NewEmptyProximityIndex(ctx context.Context, ns tree.NodeStore, kd, vd *val.
return IndexFromProximityMap(m), nil
}
type nomsIndex struct {
index types.Map
vrw types.ValueReadWriter
ns tree.NodeStore
}
var _ Index = nomsIndex{}
func IterAllIndexes(
ctx context.Context,
sch schema.Schema,
@@ -214,60 +197,6 @@ func IterAllIndexes(
return nil
}
// NomsMapFromIndex unwraps the Index and returns the underlying types.Map.
func NomsMapFromIndex(i Index) types.Map {
return i.(nomsIndex).index
}
// IndexFromNomsMap wraps a types.Map and returns it as an Index.
func IndexFromNomsMap(m types.Map, vrw types.ValueReadWriter, ns tree.NodeStore) Index {
return nomsIndex{
index: m,
vrw: vrw,
ns: ns,
}
}
var _ Index = nomsIndex{}
// HashOf implements Index.
func (i nomsIndex) HashOf() (hash.Hash, error) {
return i.index.Hash(i.vrw.Format())
}
// Count implements Index.
func (i nomsIndex) Count() (uint64, error) {
return i.index.Len(), nil
}
// Empty implements Index.
func (i nomsIndex) Empty() (bool, error) {
return i.index.Len() == 0, nil
}
// Format implements Index.
func (i nomsIndex) Format() *types.NomsBinFormat {
return i.vrw.Format()
}
// bytes implements Index.
func (i nomsIndex) bytes() ([]byte, error) {
rowschunk, err := types.EncodeValue(i.index, i.vrw.Format())
if err != nil {
return nil, err
}
return rowschunk.Data(), nil
}
func (i nomsIndex) AddColumnToRows(ctx context.Context, newCol string, newSchema schema.Schema) (Index, error) {
// no-op for noms indexes because of tag-based mapping
return i, nil
}
func (i nomsIndex) DebugString(ctx context.Context, ns tree.NodeStore, schema schema.Schema) string {
panic("Not implemented")
}
type prollyIndex struct {
index prolly.Map
}
@@ -432,14 +361,7 @@ func NewIndexSet(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeSto
return doltDevIndexSet{vrw, ns, emptyam}, nil
}
empty, err := types.NewMap(ctx, vrw)
if err != nil {
return nil, err
}
return nomsIndexSet{
indexes: empty,
vrw: vrw,
}, nil
panic("Unsupported format " + vrw.Format().VersionString())
}
func NewIndexSetWithEmptyIndexes(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) (IndexSet, error) {
@@ -460,98 +382,6 @@ func NewIndexSetWithEmptyIndexes(ctx context.Context, vrw types.ValueReadWriter,
return s, nil
}
type nomsIndexSet struct {
indexes types.Map
vrw types.ValueReadWriter
ns tree.NodeStore
}
var _ IndexSet = nomsIndexSet{}
// HashOf implements IndexSet.
func (s nomsIndexSet) HashOf() (hash.Hash, error) {
return s.indexes.Hash(s.vrw.Format())
}
// HasIndex implements IndexSet.
func (s nomsIndexSet) HasIndex(ctx context.Context, name string) (bool, error) {
_, ok, err := s.indexes.MaybeGet(ctx, types.String(name))
if err != nil {
return false, err
}
return ok, nil
}
// GetIndex implements IndexSet.
func (s nomsIndexSet) GetIndex(ctx context.Context, tableSch schema.Schema, idxSch schema.Schema, name string) (Index, error) {
v, ok, err := s.indexes.MaybeGet(ctx, types.String(name))
if !ok {
err = fmt.Errorf("index %s not found in IndexSet", name)
}
if err != nil {
return nil, err
}
idx := tableSch.Indexes().GetByName(name)
if idx == nil {
return nil, fmt.Errorf("index not found: %s", name)
}
return indexFromRef(ctx, s.vrw, s.ns, idxSch, v.(types.Ref))
}
// PutNomsIndex implements IndexSet.
func (s nomsIndexSet) PutNomsIndex(ctx context.Context, name string, idx types.Map) (IndexSet, error) {
return s.PutIndex(ctx, name, IndexFromNomsMap(idx, s.vrw, s.ns))
}
// PutIndex implements IndexSet.
func (s nomsIndexSet) PutIndex(ctx context.Context, name string, idx Index) (IndexSet, error) {
ref, err := RefFromIndex(ctx, s.vrw, idx)
if err != nil {
return nil, err
}
im, err := s.indexes.Edit().Set(types.String(name), ref).Map(ctx)
if err != nil {
return nil, err
}
return nomsIndexSet{indexes: im, vrw: s.vrw, ns: s.ns}, nil
}
// DropIndex implements IndexSet.
func (s nomsIndexSet) DropIndex(ctx context.Context, name string) (IndexSet, error) {
im, err := s.indexes.Edit().Remove(types.String(name)).Map(ctx)
if err != nil {
return nil, err
}
return nomsIndexSet{indexes: im, vrw: s.vrw, ns: s.ns}, nil
}
func (s nomsIndexSet) RenameIndex(ctx context.Context, oldName, newName string) (IndexSet, error) {
v, ok, err := s.indexes.MaybeGet(ctx, types.String(oldName))
if !ok {
err = fmt.Errorf("index %s not found in IndexSet", oldName)
}
if err != nil {
return nil, err
}
edit := s.indexes.Edit()
im, err := edit.Set(types.String(newName), v).Remove(types.String(oldName)).Map(ctx)
if err != nil {
return nil, err
}
return nomsIndexSet{indexes: im, vrw: s.vrw, ns: s.ns}, nil
}
func mapFromIndexSet(ic IndexSet) types.Map {
return ic.(nomsIndexSet).indexes
}
type doltDevIndexSet struct {
vrw types.ValueReadWriter
ns tree.NodeStore
@@ -604,11 +434,7 @@ func (is doltDevIndexSet) PutIndex(ctx context.Context, name string, idx Index)
return nil, err
}
return doltDevIndexSet{is.vrw, is.ns, am}, nil
}
func (is doltDevIndexSet) PutNomsIndex(ctx context.Context, name string, idx types.Map) (IndexSet, error) {
return is.PutIndex(ctx, name, IndexFromNomsMap(idx, is.vrw, is.ns))
return doltDevIndexSet{vrw: is.vrw, ns: is.ns, am: am}, nil
}
func (is doltDevIndexSet) DropIndex(ctx context.Context, name string) (IndexSet, error) {
go/libraries/doltcore/doltdb/durable/table.go
+26 -738
View File
@@ -15,7 +15,6 @@
package durable
import (
"bytes"
"context"
"errors"
"fmt"
@@ -23,7 +22,6 @@ import (
flatbuffers "github.com/dolthub/flatbuffers/v23/go"
"github.com/dolthub/dolt/go/gen/fb/serial"
"github.com/dolthub/dolt/go/libraries/doltcore/conflict"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding"
"github.com/dolthub/dolt/go/store/hash"
@@ -35,26 +33,8 @@ import (
"github.com/dolthub/dolt/go/store/val"
)
const (
tableStructName = "table"
schemaRefKey = "schema_ref"
tableRowsKey = "rows"
artifactsKey = "artifacts"
conflictsKey = "conflicts"
conflictSchemasKey = "conflict_schemas"
constraintViolationsKey = "constraint_violations"
indexesKey = "indexes"
autoIncrementKey = "auto_increment"
)
var (
ErrUnknownAutoIncrementValue = fmt.Errorf("auto increment set for non-numeric column type")
)
var (
errNbfUnknown = fmt.Errorf("unknown NomsBinFormat")
errNbfUnsupported = fmt.Errorf("operation unsupported for NomsBinFormat")
errNbfUnknown = fmt.Errorf("unknown NomsBinFormat")
)
// Table is a Dolt table that can be persisted.
@@ -62,9 +42,6 @@ type Table interface {
// HashOf returns the hash.Hash of this table.
HashOf() (hash.Hash, error)
// Format returns the types.NomsBinFormat for this table.
Format() *types.NomsBinFormat
// GetSchemaHash returns the hash.Hash of this table's schema.
GetSchemaHash(ctx context.Context) (hash.Hash, error)
// GetSchema returns this table's schema.
@@ -89,20 +66,6 @@ type Table interface {
// SetArtifacts sets the merge artifacts for this table.
SetArtifacts(ctx context.Context, artifacts ArtifactIndex) (Table, error)
// GetConflicts returns the merge conflicts for this table.
GetConflicts(ctx context.Context) (conflict.ConflictSchema, ConflictIndex, error)
// HasConflicts returns true if this table has conflicts.
HasConflicts(ctx context.Context) (bool, error)
// SetConflicts sets the merge conflicts for this table.
SetConflicts(ctx context.Context, sch conflict.ConflictSchema, conflicts ConflictIndex) (Table, error)
// ClearConflicts removes all merge conflicts for this table.
ClearConflicts(ctx context.Context) (Table, error)
// GetConstraintViolations returns the constraint violations for this table.
GetConstraintViolations(ctx context.Context) (types.Map, error)
// SetConstraintViolations sets the constraint violations for this table.
SetConstraintViolations(ctx context.Context, violations types.Map) (Table, error)
// GetAutoIncrement returns the AUTO_INCREMENT sequence value for this table.
GetAutoIncrement(ctx context.Context) (uint64, error)
// SetAutoIncrement sets the AUTO_INCREMENT sequence value for this table.
@@ -112,574 +75,52 @@ type Table interface {
DebugString(ctx context.Context, ns tree.NodeStore) string
}
type nomsTable struct {
vrw types.ValueReadWriter
ns tree.NodeStore
tableStruct types.Struct
}
var _ Table = nomsTable{}
var sharePool = pool.NewBuffPool()
// NewNomsTable makes a new Table.
func NewNomsTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows types.Map, indexes IndexSet, autoIncVal types.Value) (Table, error) {
return NewTable(ctx, vrw, ns, sch, nomsIndex{index: rows, vrw: vrw}, indexes, autoIncVal)
}
// NewTable returns a new Table.
func NewTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows Index, indexes IndexSet, autoIncVal types.Value) (Table, error) {
if vrw.Format().UsesFlatbuffers() {
return newDoltDevTable(ctx, vrw, ns, sch, rows, indexes, autoIncVal)
}
schVal, err := encoding.MarshalSchema(ctx, vrw, sch)
if err != nil {
return nil, err
}
schemaRef, err := refFromNomsValue(ctx, vrw, schVal)
if err != nil {
return nil, err
}
rowsRef, err := RefFromIndex(ctx, vrw, rows)
if err != nil {
return nil, err
}
if indexes == nil {
indexes, err = NewIndexSet(ctx, vrw, ns)
if err != nil {
return nil, err
}
}
indexesRef, err := refFromNomsValue(ctx, vrw, mapFromIndexSet(indexes))
if err != nil {
return nil, err
}
sd := types.StructData{
schemaRefKey: schemaRef,
tableRowsKey: rowsRef,
indexesKey: indexesRef,
}
if schema.HasAutoIncrement(sch) && autoIncVal != nil {
sd[autoIncrementKey] = autoIncVal
}
tableStruct, err := types.NewStruct(vrw.Format(), tableStructName, sd)
if err != nil {
return nil, err
}
return nomsTable{vrw, ns, tableStruct}, nil
panic("Unsupported format: " + vrw.Format().VersionString())
}
// TableFromAddr deserializes the table in the chunk at |addr|.
func TableFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, addr hash.Hash) (Table, error) {
types.AssertFormat_DOLT(vrw.Format())
val, err := vrw.MustReadValue(ctx, addr)
if err != nil {
return nil, err
}
if !vrw.Format().UsesFlatbuffers() {
st, ok := val.(types.Struct)
if !ok {
err = errors.New("table ref is unexpected noms value")
return nil, err
}
return nomsTable{vrw: vrw, tableStruct: st, ns: ns}, nil
} else {
sm, ok := val.(types.SerialMessage)
if !ok {
err = errors.New("table ref is unexpected noms value; not SerialMessage")
return nil, err
}
id := serial.GetFileID(sm)
if id != serial.TableFileID {
err = errors.New("table ref is unexpected noms value; GetFileID == " + id)
return nil, err
}
st, err := serial.TryGetRootAsTable([]byte(sm), serial.MessagePrefixSz)
if err != nil {
return nil, err
}
return doltDevTable{vrw, ns, st}, nil
sm, ok := val.(types.SerialMessage)
if !ok {
err = errors.New("table ref is unexpected noms value; not SerialMessage")
return nil, err
}
}
// RefFromNomsTable serialized |table|, and returns its types.Ref.
func RefFromNomsTable(ctx context.Context, table Table) (types.Ref, error) {
nt, ok := table.(nomsTable)
if ok {
return refFromNomsValue(ctx, nt.vrw, nt.tableStruct)
id := serial.GetFileID(sm)
if id != serial.TableFileID {
err = errors.New("table ref is unexpected noms value; GetFileID == " + id)
return nil, err
}
ddt := table.(doltDevTable)
return refFromNomsValue(ctx, ddt.vrw, ddt.nomsValue())
st, err := serial.TryGetRootAsTable([]byte(sm), serial.MessagePrefixSz)
if err != nil {
return nil, err
}
return doltDevTable{vrw, ns, st}, nil
}
// VrwFromTable returns the types.ValueReadWriter used by |t|.
// todo(andy): this is a temporary method that will be removed when there is a
// general-purpose abstraction to replace types.ValueReadWriter.
func VrwFromTable(t Table) types.ValueReadWriter {
if nt, ok := t.(nomsTable); ok {
return nt.vrw
} else {
ddt := t.(doltDevTable)
return ddt.vrw
}
ddt := t.(doltDevTable)
return ddt.vrw
}
func NodeStoreFromTable(t Table) tree.NodeStore {
if nt, ok := t.(nomsTable); ok {
return nt.ns
} else {
ddt := t.(doltDevTable)
return ddt.ns
}
}
// valueReadWriter returns the valueReadWriter for this table.
func (t nomsTable) valueReadWriter() types.ValueReadWriter {
return t.vrw
}
// HashOf implements Table.
func (t nomsTable) HashOf() (hash.Hash, error) {
return t.tableStruct.Hash(t.vrw.Format())
}
// Format returns the types.NomsBinFormat for this index.
func (t nomsTable) Format() *types.NomsBinFormat {
return t.vrw.Format()
}
// GetSchema implements Table.
func (t nomsTable) GetSchema(ctx context.Context) (schema.Schema, error) {
schemaRefVal, _, err := t.tableStruct.MaybeGet(schemaRefKey)
if err != nil {
return nil, err
}
schemaRef := schemaRefVal.(types.Ref)
return schemaFromRef(ctx, t.vrw, schemaRef)
}
// GetSchemaHash implements Table.
func (t nomsTable) GetSchemaHash(ctx context.Context) (hash.Hash, error) {
r, _, err := t.tableStruct.MaybeGet(schemaRefKey)
if err != nil {
return hash.Hash{}, err
}
return r.(types.Ref).TargetHash(), nil
}
// SetSchema implements Table.
func (t nomsTable) SetSchema(ctx context.Context, sch schema.Schema) (Table, error) {
newSchemaVal, err := encoding.MarshalSchema(ctx, t.vrw, sch)
if err != nil {
return nil, err
}
schRef, err := refFromNomsValue(ctx, t.vrw, newSchemaVal)
if err != nil {
return nil, err
}
newTableStruct, err := t.tableStruct.Set(schemaRefKey, schRef)
if err != nil {
return nil, err
}
return nomsTable{t.vrw, t.ns, newTableStruct}, nil
}
// SetTableRows implements Table.
func (t nomsTable) SetTableRows(ctx context.Context, updatedRows Index) (Table, error) {
rowsRef, err := RefFromIndex(ctx, t.vrw, updatedRows)
if err != nil {
return nil, err
}
updatedSt, err := t.tableStruct.Set(tableRowsKey, rowsRef)
if err != nil {
return nil, err
}
return nomsTable{t.vrw, t.ns, updatedSt}, nil
}
// GetTableRows implements Table.
func (t nomsTable) GetTableRows(ctx context.Context) (Index, error) {
val, _, err := t.tableStruct.MaybeGet(tableRowsKey)
if err != nil {
return nil, err
}
sch, err := t.GetSchema(ctx)
if err != nil {
return nil, err
}
return indexFromRef(ctx, t.vrw, t.ns, sch, val.(types.Ref))
}
func (t nomsTable) GetTableRowsWithDescriptors(ctx context.Context, kd, vd *val.TupleDesc) (Index, error) {
return nil, fmt.Errorf("nomsTable does not implement GetTableRowsWithDescriptors")
}
// GetIndexes implements Table.
func (t nomsTable) GetIndexes(ctx context.Context) (IndexSet, error) {
iv, ok, err := t.tableStruct.MaybeGet(indexesKey)
if err != nil {
return nil, err
}
if !ok {
return NewIndexSet(ctx, t.vrw, t.ns)
}
im, err := iv.(types.Ref).TargetValue(ctx, t.vrw)
if err != nil {
return nil, err
}
return nomsIndexSet{
indexes: im.(types.Map),
vrw: t.vrw,
ns: t.ns,
}, nil
}
// SetIndexes implements Table.
func (t nomsTable) SetIndexes(ctx context.Context, indexes IndexSet) (Table, error) {
if indexes == nil {
var err error
indexes, err = NewIndexSet(ctx, t.vrw, t.ns)
if err != nil {
return nil, err
}
}
indexesRef, err := refFromNomsValue(ctx, t.vrw, mapFromIndexSet(indexes))
if err != nil {
return nil, err
}
newTableStruct, err := t.tableStruct.Set(indexesKey, indexesRef)
if err != nil {
return nil, err
}
return nomsTable{t.vrw, t.ns, newTableStruct}, nil
}
// GetArtifacts implements Table.
func (t nomsTable) GetArtifacts(ctx context.Context) (ArtifactIndex, error) {
if t.Format() != types.Format_DOLT {
panic("artifacts not implemented for old storage format")
}
sch, err := t.GetSchema(ctx)
if err != nil {
return nil, err
}
val, ok, err := t.tableStruct.MaybeGet(artifactsKey)
if err != nil {
return nil, err
}
if !ok {
return NewEmptyArtifactIndex(ctx, t.vrw, t.ns, sch)
}
return artifactIndexFromRef(ctx, t.vrw, t.ns, sch, val.(types.Ref))
}
// SetArtifacts implements Table.
func (t nomsTable) SetArtifacts(ctx context.Context, artifacts ArtifactIndex) (Table, error) {
if t.Format() != types.Format_DOLT {
panic("artifacts not implemented for old storage format")
}
ref, err := RefFromArtifactIndex(ctx, t.vrw, artifacts)
if err != nil {
return nil, err
}
updated, err := t.tableStruct.Set(artifactsKey, ref)
if err != nil {
return nil, err
}
return nomsTable{t.vrw, t.ns, updated}, nil
}
// HasConflicts implements Table.
func (t nomsTable) HasConflicts(ctx context.Context) (bool, error) {
_, ok, err := t.tableStruct.MaybeGet(conflictSchemasKey)
return ok, err
}
// GetConflicts implements Table.
func (t nomsTable) GetConflicts(ctx context.Context) (conflict.ConflictSchema, ConflictIndex, error) {
schemasVal, ok, err := t.tableStruct.MaybeGet(conflictSchemasKey)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
if !ok {
sch, err := t.GetSchema(ctx)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
empty, err := NewEmptyConflictIndex(ctx, t.vrw, t.ns, sch, sch, sch)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
return conflict.ConflictSchema{}, empty, nil
}
schemas, err := conflict.ConflictSchemaFromValue(ctx, t.vrw, schemasVal)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
conflictsVal, _, err := t.tableStruct.MaybeGet(conflictsKey)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
if conflictsVal == nil {
confIndex, err := NewEmptyConflictIndex(ctx, t.vrw, t.ns, schemas.Schema, schemas.MergeSchema, schemas.Base)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
return conflict.ConflictSchema{}, confIndex, nil
}
i, err := conflictIndexFromRef(ctx, t.vrw, t.ns, schemas.Schema, schemas.MergeSchema, schemas.Base, conflictsVal.(types.Ref))
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
return schemas, i, nil
}
// SetConflicts implements Table.
func (t nomsTable) SetConflicts(ctx context.Context, schemas conflict.ConflictSchema, conflictData ConflictIndex) (Table, error) {
if t.Format() == types.Format_DOLT {
panic("should use artifacts")
}
conflictsRef, err := RefFromConflictIndex(ctx, t.vrw, conflictData)
if err != nil {
return nil, err
}
tpl, err := conflict.ValueFromConflictSchema(ctx, t.vrw, schemas)
if err != nil {
return nil, err
}
updatedSt, err := t.tableStruct.Set(conflictSchemasKey, tpl)
if err != nil {
return nil, err
}
updatedSt, err = updatedSt.Set(conflictsKey, conflictsRef)
if err != nil {
return nil, err
}
return nomsTable{t.vrw, t.ns, updatedSt}, nil
}
// GetConflictSchemas implements Table.
func (t nomsTable) GetConflictSchemas(ctx context.Context) (base, sch, mergeSch schema.Schema, err error) {
schemasVal, ok, err := t.tableStruct.MaybeGet(conflictSchemasKey)
if err != nil {
return nil, nil, nil, err
}
if ok {
schemas, err := conflict.ConflictFromTuple(schemasVal.(types.Tuple))
if err != nil {
return nil, nil, nil, err
}
baseRef := schemas.Base.(types.Ref)
valRef := schemas.Value.(types.Ref)
mergeRef := schemas.MergeValue.(types.Ref)
var baseSch, sch, mergeSch schema.Schema
if baseSch, err = schemaFromRef(ctx, t.vrw, baseRef); err == nil {
if sch, err = schemaFromRef(ctx, t.vrw, valRef); err == nil {
mergeSch, err = schemaFromRef(ctx, t.vrw, mergeRef)
}
}
return baseSch, sch, mergeSch, err
}
return nil, nil, nil, nil
}
// ClearConflicts implements Table.
func (t nomsTable) ClearConflicts(ctx context.Context) (Table, error) {
if t.Format() == types.Format_DOLT {
panic("should use artifacts")
}
tSt, err := t.tableStruct.Delete(conflictSchemasKey)
if err != nil {
return nil, err
}
tSt, err = tSt.Delete(conflictsKey)
if err != nil {
return nil, err
}
return nomsTable{t.vrw, t.ns, tSt}, nil
}
// GetConstraintViolations implements Table.
func (t nomsTable) GetConstraintViolations(ctx context.Context) (types.Map, error) {
constraintViolationsRefVal, ok, err := t.tableStruct.MaybeGet(constraintViolationsKey)
if err != nil {
return types.EmptyMap, err
}
if !ok {
emptyMap, err := types.NewMap(ctx, t.vrw)
return emptyMap, err
}
constraintViolationsVal, err := constraintViolationsRefVal.(types.Ref).TargetValue(ctx, t.vrw)
if err != nil {
return types.EmptyMap, err
}
return constraintViolationsVal.(types.Map), nil
}
// SetConstraintViolations implements Table.
func (t nomsTable) SetConstraintViolations(ctx context.Context, violationsMap types.Map) (Table, error) {
// We can't just call violationsMap.Empty() as we can't guarantee that the caller passed in an instantiated map
if violationsMap == types.EmptyMap || violationsMap.Len() == 0 {
updatedStruct, err := t.tableStruct.Delete(constraintViolationsKey)
if err != nil {
return nil, err
}
return nomsTable{t.vrw, t.ns, updatedStruct}, nil
}
constraintViolationsRef, err := refFromNomsValue(ctx, t.vrw, violationsMap)
if err != nil {
return nil, err
}
updatedStruct, err := t.tableStruct.Set(constraintViolationsKey, constraintViolationsRef)
if err != nil {
return nil, err
}
return nomsTable{t.vrw, t.ns, updatedStruct}, nil
}
// GetAutoIncrement implements Table.
func (t nomsTable) GetAutoIncrement(ctx context.Context) (uint64, error) {
val, ok, err := t.tableStruct.MaybeGet(autoIncrementKey)
if err != nil {
return 0, err
}
if !ok {
return 1, nil
}
// older versions might have serialized auto-increment
// value as types.Int or types.Float.
switch t := val.(type) {
case types.Int:
return uint64(t), nil
case types.Uint:
return uint64(t), nil
case types.Float:
return uint64(t), nil
default:
return 0, ErrUnknownAutoIncrementValue
}
}
// SetAutoIncrement implements Table.
func (t nomsTable) SetAutoIncrement(ctx context.Context, val uint64) (Table, error) {
st, err := t.tableStruct.Set(autoIncrementKey, types.Uint(val))
if err != nil {
return nil, err
}
return nomsTable{t.vrw, t.ns, st}, nil
}
func (t nomsTable) DebugString(ctx context.Context, ns tree.NodeStore) string {
var buf bytes.Buffer
err := types.WriteEncodedValue(ctx, &buf, t.tableStruct)
if err != nil {
panic(err)
}
schemaRefVal, _, _ := t.tableStruct.MaybeGet(schemaRefKey)
schemaRef := schemaRefVal.(types.Ref)
schemaVal, err := schemaRef.TargetValue(ctx, t.vrw)
if err != nil {
panic(err)
}
buf.WriteString("\nschema: ")
err = types.WriteEncodedValue(ctx, &buf, schemaVal)
if err != nil {
panic(err)
}
iv, ok, err := t.tableStruct.MaybeGet(indexesKey)
if err != nil {
panic(err)
}
if ok {
buf.WriteString("\nindexes: ")
im, err := iv.(types.Ref).TargetValue(ctx, t.vrw)
if err != nil {
panic(err)
}
err = types.WriteEncodedValue(ctx, &buf, im)
if err != nil {
panic(err)
}
}
buf.WriteString("\ndata:\n")
data, err := t.GetTableRows(ctx)
if err != nil {
panic(err)
}
err = types.WriteEncodedValue(ctx, &buf, NomsMapFromIndex(data))
if err != nil {
panic(err)
}
return buf.String()
}
func refFromNomsValue(ctx context.Context, vrw types.ValueReadWriter, val types.Value) (types.Ref, error) {
return vrw.WriteValue(ctx, val)
}
func schemaFromRef(ctx context.Context, vrw types.ValueReadWriter, ref types.Ref) (schema.Schema, error) {
return schemaFromAddr(ctx, vrw, ref.TargetHash())
ddt := t.(doltDevTable)
return ddt.ns
}
func schemaFromAddr(ctx context.Context, vrw types.ValueReadWriter, addr hash.Hash) (schema.Schema, error) {
@@ -763,7 +204,7 @@ func newDoltDevTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.Nod
return nil, err
}
schemaRef, err := refFromNomsValue(ctx, vrw, schVal)
schemaRef, err := vrw.WriteValue(ctx, schVal)
if err != nil {
return nil, err
}
@@ -833,7 +274,7 @@ func (t doltDevTable) SetSchema(ctx context.Context, sch schema.Schema) (Table,
return nil, err
}
schRef, err := refFromNomsValue(ctx, t.vrw, newSchemaVal)
schRef, err := t.vrw.WriteValue(ctx, newSchemaVal)
if err != nil {
return nil, err
}
@@ -915,64 +356,6 @@ func (t doltDevTable) SetIndexes(ctx context.Context, indexes IndexSet) (Table,
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) GetConflicts(ctx context.Context) (conflict.ConflictSchema, ConflictIndex, error) {
conflicts, err := t.msg.TryConflicts(nil)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
ouraddr := hash.New(conflicts.OurSchemaBytes())
theiraddr := hash.New(conflicts.TheirSchemaBytes())
baseaddr := hash.New(conflicts.AncestorSchemaBytes())
if ouraddr.IsEmpty() {
sch, err := t.GetSchema(ctx)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
empty, err := NewEmptyConflictIndex(ctx, t.vrw, t.ns, sch, sch, sch)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
return conflict.ConflictSchema{}, empty, nil
}
ourschema, err := getSchemaAtAddr(ctx, t.vrw, ouraddr)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
theirschema, err := getSchemaAtAddr(ctx, t.vrw, theiraddr)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
baseschema, err := getSchemaAtAddr(ctx, t.vrw, baseaddr)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
conflictschema := conflict.ConflictSchema{
Base: baseschema,
Schema: ourschema,
MergeSchema: theirschema,
}
mapaddr := hash.New(conflicts.DataBytes())
var conflictIdx ConflictIndex
if mapaddr.IsEmpty() {
conflictIdx, err = NewEmptyConflictIndex(ctx, t.vrw, t.ns, ourschema, theirschema, baseschema)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
} else {
conflictIdx, err = conflictIndexFromAddr(ctx, t.vrw, t.ns, ourschema, theirschema, baseschema, mapaddr)
if err != nil {
return conflict.ConflictSchema{}, nil, err
}
}
return conflictschema, conflictIdx, nil
}
// GetArtifacts implements Table.
func (t doltDevTable) GetArtifacts(ctx context.Context) (ArtifactIndex, error) {
if t.Format() != types.Format_DOLT {
@@ -1017,90 +400,6 @@ func (t doltDevTable) SetArtifacts(ctx context.Context, artifacts ArtifactIndex)
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) HasConflicts(ctx context.Context) (bool, error) {
conflicts, err := t.msg.TryConflicts(nil)
if err != nil {
return false, err
}
addr := hash.New(conflicts.OurSchemaBytes())
return !addr.IsEmpty(), nil
}
func (t doltDevTable) SetConflicts(ctx context.Context, sch conflict.ConflictSchema, conflicts ConflictIndex) (Table, error) {
conflictsRef, err := RefFromConflictIndex(ctx, t.vrw, conflicts)
if err != nil {
return nil, err
}
conflictsAddr := conflictsRef.TargetHash()
baseaddr, err := getAddrForSchema(ctx, t.vrw, sch.Base)
if err != nil {
return nil, err
}
ouraddr, err := getAddrForSchema(ctx, t.vrw, sch.Schema)
if err != nil {
return nil, err
}
theiraddr, err := getAddrForSchema(ctx, t.vrw, sch.MergeSchema)
if err != nil {
return nil, err
}
msg := t.clone()
cmsg, err := msg.TryConflicts(nil)
if err != nil {
return nil, err
}
copy(cmsg.DataBytes(), conflictsAddr[:])
copy(cmsg.OurSchemaBytes(), ouraddr[:])
copy(cmsg.TheirSchemaBytes(), theiraddr[:])
copy(cmsg.AncestorSchemaBytes(), baseaddr[:])
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) ClearConflicts(ctx context.Context) (Table, error) {
msg := t.clone()
conflicts, err := msg.TryConflicts(nil)
if err != nil {
return nil, err
}
var emptyhash hash.Hash
copy(conflicts.DataBytes(), emptyhash[:])
copy(conflicts.OurSchemaBytes(), emptyhash[:])
copy(conflicts.TheirSchemaBytes(), emptyhash[:])
copy(conflicts.AncestorSchemaBytes(), emptyhash[:])
return doltDevTable{t.vrw, t.ns, msg}, nil
}
func (t doltDevTable) GetConstraintViolations(ctx context.Context) (types.Map, error) {
addr := hash.New(t.msg.ViolationsBytes())
if addr.IsEmpty() {
return types.NewMap(ctx, t.vrw)
}
v, err := t.vrw.MustReadValue(ctx, addr)
if err != nil {
return types.Map{}, err
}
return v.(types.Map), nil
}
func (t doltDevTable) SetConstraintViolations(ctx context.Context, violations types.Map) (Table, error) {
var addr hash.Hash
if violations != types.EmptyMap && violations.Len() != 0 {
ref, err := refFromNomsValue(ctx, t.vrw, violations)
if err != nil {
return nil, err
}
addr = ref.TargetHash()
}
msg := t.clone()
copy(msg.ViolationsBytes(), addr[:])
return doltDevTable{t.vrw, t.ns, msg}, nil
}
// GetAutoIncrement returns the next value to be used for the AUTO_INCREMENT column.
func (t doltDevTable) GetAutoIncrement(ctx context.Context) (uint64, error) {
res := t.msg.AutoIncrementValue()
@@ -1175,18 +474,7 @@ func (t doltDevTable) fields() (serialTableFields, error) {
}, nil
}
func getSchemaAtAddr(ctx context.Context, vrw types.ValueReadWriter, addr hash.Hash) (schema.Schema, error) {
return encoding.UnmarshalSchemaAtAddr(ctx, vrw, addr)
}
func getAddrForSchema(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema) (hash.Hash, error) {
st, err := encoding.MarshalSchema(ctx, vrw, sch)
if err != nil {
return hash.Hash{}, err
}
ref, err := vrw.WriteValue(ctx, st)
if err != nil {
return hash.Hash{}, err
}
return ref.TargetHash(), nil
func RefFromNomsTable(ctx context.Context, table Table) (types.Ref, error) {
ddt := table.(doltDevTable)
return ddt.vrw.WriteValue(ctx, ddt.nomsValue())
}
go/libraries/doltcore/doltdb/table.go
+6 -207
View File
@@ -20,7 +20,6 @@ import (
"fmt"
"unicode"
"github.com/dolthub/dolt/go/libraries/doltcore/conflict"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/hash"
@@ -60,17 +59,6 @@ type Table struct {
overriddenSchema schema.Schema
}
// NewNomsTable creates a noms Struct which stores row data, index data, and schema.
// Deprecated: use NewTable instead.
func NewNomsTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows types.Map, indexes durable.IndexSet, autoIncVal types.Value) (*Table, error) {
dt, err := durable.NewNomsTable(ctx, vrw, ns, sch, rows, indexes, autoIncVal)
if err != nil {
return nil, err
}
return &Table{table: dt}, nil
}
// NewTable creates a durable object which stores row data, index data, and schema.
func NewTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rows durable.Index, indexes durable.IndexSet, autoIncVal types.Value) (*Table, error) {
dt, err := durable.NewTable(ctx, vrw, ns, sch, rows, indexes, autoIncVal)
@@ -80,11 +68,6 @@ func NewTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore,
return &Table{table: dt}, nil
}
// NewTableFromDurable creates a table from the given durable object.
func NewTableFromDurable(table durable.Table) *Table {
return &Table{table: table}
}
func NewEmptyTable(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema) (*Table, error) {
rows, err := durable.NewEmptyPrimaryIndex(ctx, vrw, ns, sch)
if err != nil {
@@ -127,24 +110,6 @@ func (t *Table) GetOverriddenSchema() schema.Schema {
return t.overriddenSchema
}
// SetConflicts sets the merge conflicts for this table.
func (t *Table) SetConflicts(ctx context.Context, schemas conflict.ConflictSchema, conflictData durable.ConflictIndex) (*Table, error) {
table, err := t.table.SetConflicts(ctx, schemas, conflictData)
if err != nil {
return nil, err
}
return &Table{table: table}, nil
}
// GetConflicts returns a map built from ValueReadWriter when there are no conflicts in table.
func (t *Table) GetConflicts(ctx context.Context) (conflict.ConflictSchema, durable.ConflictIndex, error) {
if t.Format() == types.Format_DOLT {
panic("should use artifacts")
}
return t.table.GetConflicts(ctx)
}
// HasConflicts returns true if this table contains merge conflicts.
func (t *Table) HasConflicts(ctx context.Context) (bool, error) {
if t.Format() == types.Format_DOLT {
@@ -155,7 +120,8 @@ func (t *Table) HasConflicts(ctx context.Context) (bool, error) {
return art.HasConflicts(ctx)
}
return t.table.HasConflicts(ctx)
panic("Unsupported format: " + t.Format().VersionString())
}
// GetArtifacts returns the merge artifacts for this table.
@@ -182,20 +148,7 @@ func (t *Table) NumRowsInConflict(ctx context.Context) (uint64, error) {
return artIdx.ConflictCount(ctx)
}
ok, err := t.table.HasConflicts(ctx)
if err != nil {
return 0, err
}
if !ok {
return 0, nil
}
_, cons, err := t.table.GetConflicts(ctx)
if err != nil {
return 0, err
}
return cons.Count(), nil
panic("Unsupported format: " + t.Format().VersionString())
}
// NumConstraintViolations returns the number of constraint violations for this table.
@@ -208,12 +161,7 @@ func (t *Table) NumConstraintViolations(ctx context.Context) (uint64, error) {
return artIdx.ConstraintViolationCount(ctx)
}
cvs, err := t.table.GetConstraintViolations(ctx)
if err != nil {
return 0, err
}
return cvs.Len(), nil
panic("Unsupported format: " + t.Format().VersionString())
}
// ClearConflicts deletes all merge conflicts for this table.
@@ -222,7 +170,7 @@ func (t *Table) ClearConflicts(ctx context.Context) (*Table, error) {
return t.clearArtifactConflicts(ctx)
}
return t.clearConflicts(ctx)
panic("Unsupported format: " + t.Format().VersionString())
}
func (t *Table) clearArtifactConflicts(ctx context.Context) (*Table, error) {
@@ -241,21 +189,13 @@ func (t *Table) clearArtifactConflicts(ctx context.Context) (*Table, error) {
return &Table{table: table}, nil
}
func (t *Table) clearConflicts(ctx context.Context) (*Table, error) {
table, err := t.table.ClearConflicts(ctx)
if err != nil {
return nil, err
}
return &Table{table: table}, nil
}
// GetConflictSchemas returns the merge conflict schemas for this table.
func (t *Table) GetConflictSchemas(ctx context.Context, tblName TableName) (base, sch, mergeSch schema.Schema, err error) {
if t.Format() == types.Format_DOLT {
return t.getProllyConflictSchemas(ctx, tblName)
}
return t.getNomsConflictSchemas(ctx)
panic("Unsupported format: " + t.Format().VersionString())
}
// The conflict schema is implicitly determined based on the first conflict in the artifacts table.
@@ -337,36 +277,6 @@ func tableFromRootIsh(ctx context.Context, vrw types.ValueReadWriter, ns tree.No
return tbl, ok, nil
}
func (t *Table) getNomsConflictSchemas(ctx context.Context) (base, sch, mergeSch schema.Schema, err error) {
cs, _, err := t.table.GetConflicts(ctx)
if err != nil {
return nil, nil, nil, err
}
return cs.Base, cs.Schema, cs.MergeSchema, nil
}
// GetConstraintViolations returns a map of all constraint violations for this table, along with a bool indicating
// whether the table has any violations.
func (t *Table) GetConstraintViolations(ctx context.Context) (types.Map, error) {
if t.Format() == types.Format_DOLT {
panic("should use artifacts")
}
return t.table.GetConstraintViolations(ctx)
}
// SetConstraintViolations sets this table's violations to the given map. If the map is empty, then the constraint
// violations entry on the embedded struct is removed.
func (t *Table) SetConstraintViolations(ctx context.Context, violationsMap types.Map) (*Table, error) {
if t.Format() == types.Format_DOLT {
panic("should use artifacts")
}
table, err := t.table.SetConstraintViolations(ctx, violationsMap)
if err != nil {
return nil, err
}
return &Table{table: table}, nil
}
// GetSchema returns the schema.Schema for this Table.
func (t *Table) GetSchema(ctx context.Context) (schema.Schema, error) {
return t.table.GetSchema(ctx)
@@ -406,18 +316,6 @@ func (t *Table) HashOf() (hash.Hash, error) {
return t.table.HashOf()
}
// UpdateNomsRows replaces the current row data and returns and updated Table.
// Calls to UpdateNomsRows will not be written to the database. The root must
// be updated with the updated table, and the root must be committed or written.
// Deprecated: use Table.UpdateRows() instead.
func (t *Table) UpdateNomsRows(ctx context.Context, updatedRows types.Map) (*Table, error) {
table, err := t.table.SetTableRows(ctx, durable.IndexFromNomsMap(updatedRows, t.ValueReadWriter(), t.NodeStore()))
if err != nil {
return nil, err
}
return &Table{table: table}, nil
}
// UpdateRows replaces the current row data and returns and updated Table.
// Calls to UpdateRows will not be written to the database. The root must
// be updated with the updated table, and the root must be committed or written.
@@ -429,17 +327,6 @@ func (t *Table) UpdateRows(ctx context.Context, updatedRows durable.Index) (*Tab
return &Table{table: table}, nil
}
// GetNomsRowData retrieves the underlying map which is a map from a primary key to a list of field values.
// Deprecated: use Table.GetRowData() instead.
func (t *Table) GetNomsRowData(ctx context.Context) (types.Map, error) {
idx, err := t.table.GetTableRows(ctx)
if err != nil {
return types.Map{}, err
}
return durable.NomsMapFromIndex(idx), nil
}
// GetRowData retrieves the underlying map which is a map from a primary key to a list of field values.
func (t *Table) GetRowData(ctx context.Context) (durable.Index, error) {
return t.table.GetTableRows(ctx)
@@ -458,57 +345,6 @@ func (t *Table) GetRowDataHash(ctx context.Context) (hash.Hash, error) {
return idx.HashOf()
}
// ResolveConflicts resolves conflicts for this table.
func (t *Table) ResolveConflicts(ctx context.Context, pkTuples []types.Value) (invalid, notFound []types.Value, tbl *Table, err error) {
removed := 0
conflictSchema, confIdx, err := t.GetConflicts(ctx)
if err != nil {
return nil, nil, nil, err
}
if confIdx.Format() == types.Format_DOLT {
panic("resolve conflicts not implemented for new storage format")
}
confData := durable.NomsMapFromConflictIndex(confIdx)
confEdit := confData.Edit()
for _, pkTupleVal := range pkTuples {
if has, err := confData.Has(ctx, pkTupleVal); err != nil {
return nil, nil, nil, err
} else if has {
removed++
confEdit.Remove(pkTupleVal)
} else {
notFound = append(notFound, pkTupleVal)
}
}
if removed == 0 {
return invalid, notFound, tbl, ErrNoConflictsResolved
}
conflicts, err := confEdit.Map(ctx)
if err != nil {
return nil, nil, nil, err
}
if conflicts.Len() == 0 {
table, err := t.table.ClearConflicts(ctx)
if err != nil {
return nil, nil, nil, err
}
return invalid, notFound, &Table{table: table}, nil
}
table, err := t.table.SetConflicts(ctx, conflictSchema, durable.ConflictIndexFromNomsMap(conflicts, t.ValueReadWriter()))
if err != nil {
return nil, nil, nil, err
}
return invalid, notFound, &Table{table: table}, nil
}
// GetIndexSet returns the internal index map which goes from index name to a ref of the row data map.
func (t *Table) GetIndexSet(ctx context.Context) (durable.IndexSet, error) {
return t.table.GetIndexes(ctx)
@@ -523,27 +359,6 @@ func (t *Table) SetIndexSet(ctx context.Context, indexes durable.IndexSet) (*Tab
return &Table{table: table}, nil
}
// GetNomsIndexRowData retrieves the underlying map of an index, in which the primary key consists of all indexed columns.
// Deprecated: use Table.GetIndexRowData() instead.
func (t *Table) GetNomsIndexRowData(ctx context.Context, indexName string) (types.Map, error) {
sch, err := t.GetSchema(ctx)
if err != nil {
return types.EmptyMap, err
}
indexes, err := t.GetIndexSet(ctx)
if err != nil {
return types.EmptyMap, err
}
idx, err := indexes.GetIndex(ctx, sch, nil, indexName)
if err != nil {
return types.EmptyMap, err
}
return durable.NomsMapFromIndex(idx), nil
}
// GetIndexRowData retrieves the underlying map of an index, in which the primary key consists of all indexed columns.
func (t *Table) GetIndexRowData(ctx context.Context, indexName string) (durable.Index, error) {
sch, err := t.GetSchema(ctx)
@@ -574,22 +389,6 @@ func (t *Table) SetIndexRows(ctx context.Context, indexName string, idx durable.
return t.SetIndexSet(ctx, indexes)
}
// SetNomsIndexRows replaces the current row data for the given index and returns an updated Table.
// Deprecated: use Table.SetIndexRows() instead.
func (t *Table) SetNomsIndexRows(ctx context.Context, indexName string, idx types.Map) (*Table, error) {
indexes, err := t.GetIndexSet(ctx)
if err != nil {
return nil, err
}
indexes, err = indexes.PutNomsIndex(ctx, indexName, idx)
if err != nil {
return nil, err
}
return t.SetIndexSet(ctx, indexes)
}
// DeleteIndexRowData removes the underlying map of an index, along with its key entry. This should only be used
// when removing an index altogether. If the intent is to clear an index's data, then use SetNomsIndexRows with
// an empty map.
go/libraries/doltcore/merge/conflict_reader.go
-322
View File
@@ -1,322 +0,0 @@
// Copyright 2019 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package merge
import (
"context"
"errors"
"io"
"github.com/dolthub/dolt/go/libraries/doltcore/conflict"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/rowconv"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/types"
)
const (
oursStr = "our"
theirsStr = "their"
baseStr = "base"
)
const (
ConflictDiffTypeAdded = "added"
ConflictDiffTypeModified = "modified"
ConflictDiffTypeRemoved = "removed"
)
// ConflictReader is a class providing a NextConflict function which can be used in a pipeline as a pipeline.SourceFunc,
// or it can be used to read each conflict
type ConflictReader struct {
confItr types.MapIterator
joiner *rowconv.Joiner
sch schema.Schema
nbf *types.NomsBinFormat
keyless bool
}
// NewConflictReader returns a new conflict reader for a given table
func NewConflictReader(ctx context.Context, tbl *doltdb.Table, tblName doltdb.TableName) (*ConflictReader, error) {
base, sch, mergeSch, err := tbl.GetConflictSchemas(ctx, tblName) // tblName unused by old storage format
if err != nil {
return nil, err
}
if base == nil || sch == nil || mergeSch == nil {
base, err = tbl.GetSchema(ctx)
sch, mergeSch = base, base
}
if err != nil {
return nil, err
}
joiner, err := rowconv.NewJoiner(
[]rowconv.NamedSchema{
{Name: baseStr, Sch: base},
{Name: oursStr, Sch: sch},
{Name: theirsStr, Sch: mergeSch},
}, map[string]rowconv.ColNamingFunc{
baseStr: func(colName string) string { return baseStr + "_" + colName },
oursStr: func(colName string) string { return oursStr + "_" + colName },
theirsStr: func(colName string) string { return theirsStr + "_" + colName },
},
)
if err != nil {
return nil, err
}
readerSch := joiner.GetSchema()
readerSch, err = readerSch.AddColumn(schema.NewColumn("our_diff_type", schema.DoltConflictsOurDiffTypeTag, types.StringKind, false), nil)
if err != nil {
return nil, err
}
readerSch, err = readerSch.AddColumn(schema.NewColumn("their_diff_type", schema.DoltConflictsTheirDiffTypeTag, types.StringKind, false), nil)
if err != nil {
return nil, err
}
var keyless bool
if schema.IsKeyless(sch) {
keyless = true
readerSch, err = readerSch.AddColumn(
schema.NewColumn(
"base_cardinality",
schema.DoltConflictsBaseCardinalityTag,
types.UintKind,
false),
nil)
if err != nil {
return nil, err
}
readerSch, err = readerSch.AddColumn(
schema.NewColumn(
"our_cardinality",
schema.DoltConflictsOurCardinalityTag,
types.UintKind,
false),
nil)
if err != nil {
return nil, err
}
readerSch, err = readerSch.AddColumn(
schema.NewColumn(
"their_cardinality",
schema.DoltConflictsTheirCardinalityTag,
types.UintKind,
false),
nil)
if err != nil {
return nil, err
}
}
_, confIdx, err := tbl.GetConflicts(ctx)
if err != nil {
return nil, err
}
if confIdx.Format() == types.Format_DOLT {
panic("conflict reader not implemented for new storage format")
}
confData := durable.NomsMapFromConflictIndex(confIdx)
confItr, err := confData.Iterator(ctx)
if err != nil {
return nil, err
}
return &ConflictReader{
confItr: confItr,
joiner: joiner,
sch: readerSch,
nbf: tbl.Format(),
keyless: keyless,
}, nil
}
// GetSchema gets the schema of the rows that this reader will return
func (cr *ConflictReader) GetSchema() schema.Schema {
return cr.sch
}
// GetJoiner returns the joiner used to join a row with its base, and merge versions
func (cr *ConflictReader) GetJoiner() *rowconv.Joiner {
return cr.joiner
}
// NextConflict can be called successively to retrieve the conflicts in a table. Once all conflicts have been returned
// io.EOF will be returned in the error field. This can be used in a pipeline, or to iterate through all the conflicts
// in a table.
func (cr *ConflictReader) NextConflict(ctx context.Context) (row.Row, error) {
key, value, err := cr.confItr.Next(ctx)
if err != nil {
return nil, err
}
if key == nil {
return nil, io.EOF
}
keyTpl := key.(types.Tuple)
conflict, err := conflict.ConflictFromTuple(value.(types.Tuple))
if err != nil {
return nil, err
}
var joinedRow row.Row
if !cr.keyless {
joinedRow, err = cr.pkJoinedRow(keyTpl, conflict)
} else {
joinedRow, err = cr.keylessJoinedRow(keyTpl, conflict)
}
if err != nil {
return nil, err
}
ourDiffType := getDiffType(conflict.Base, conflict.Value)
theirDiffType := getDiffType(conflict.Base, conflict.MergeValue)
joinedRow, err = joinedRow.SetColVal(schema.DoltConflictsOurDiffTypeTag, types.String(ourDiffType), cr.sch)
if err != nil {
return nil, err
}
joinedRow, err = joinedRow.SetColVal(schema.DoltConflictsTheirDiffTypeTag, types.String(theirDiffType), cr.sch)
if err != nil {
return nil, err
}
return joinedRow, nil
}
func (cr *ConflictReader) pkJoinedRow(key types.Tuple, conflict conflict.Conflict) (row.Row, error) {
var err error
namedRows := make(map[string]row.Row)
if !types.IsNull(conflict.Base) {
namedRows[baseStr], err = row.FromNoms(cr.joiner.SchemaForName(baseStr), key, conflict.Base.(types.Tuple))
if err != nil {
return nil, err
}
}
if !types.IsNull(conflict.Value) {
namedRows[oursStr], err = row.FromNoms(cr.joiner.SchemaForName(oursStr), key, conflict.Value.(types.Tuple))
if err != nil {
return nil, err
}
}
if !types.IsNull(conflict.MergeValue) {
namedRows[theirsStr], err = row.FromNoms(cr.joiner.SchemaForName(theirsStr), key, conflict.MergeValue.(types.Tuple))
if err != nil {
return nil, err
}
}
joinedRow, err := cr.joiner.Join(namedRows)
if err != nil {
return nil, err
}
return joinedRow, nil
}
func (cr *ConflictReader) keylessJoinedRow(key types.Tuple, conflict conflict.Conflict) (row.Row, error) {
var err error
namedRows := make(map[string]row.Row)
var baseCard, ourCard, theirCard uint64
if !types.IsNull(conflict.Base) {
namedRows[baseStr], baseCard, err = row.KeylessRowsFromTuples(key, conflict.Base.(types.Tuple))
if err != nil {
return nil, err
}
}
if !types.IsNull(conflict.Value) {
namedRows[oursStr], ourCard, err = row.KeylessRowsFromTuples(key, conflict.Value.(types.Tuple))
if err != nil {
return nil, err
}
}
if !types.IsNull(conflict.MergeValue) {
namedRows[theirsStr], theirCard, err = row.KeylessRowsFromTuples(key, conflict.MergeValue.(types.Tuple))
if err != nil {
return nil, err
}
}
joinedRow, err := cr.joiner.Join(namedRows)
if err != nil {
return nil, err
}
joinedRow, err = setCardinalities(types.Uint(baseCard), types.Uint(ourCard), types.Uint(theirCard), joinedRow, cr.sch)
if err != nil {
return nil, err
}
return joinedRow, nil
}
func setCardinalities(base, ours, theirs types.Uint, joinedRow row.Row, sch schema.Schema) (row.Row, error) {
joinedRow, err := joinedRow.SetColVal(schema.DoltConflictsBaseCardinalityTag, base, sch)
if err != nil {
return nil, err
}
joinedRow, err = joinedRow.SetColVal(schema.DoltConflictsOurCardinalityTag, ours, sch)
if err != nil {
return nil, err
}
joinedRow, err = joinedRow.SetColVal(schema.DoltConflictsTheirCardinalityTag, theirs, sch)
if err != nil {
return nil, err
}
return joinedRow, nil
}
func getDiffType(base types.Value, other types.Value) string {
if types.IsNull(base) {
return ConflictDiffTypeAdded
} else if types.IsNull(other) {
return ConflictDiffTypeRemoved
}
return ConflictDiffTypeModified
}
// GetKeyForConflicts returns the pk for a conflict row
func (cr *ConflictReader) GetKeyForConflict(ctx context.Context, r row.Row) (types.Value, error) {
rows, err := cr.joiner.Split(r)
if err != nil {
return nil, err
}
for rowType, r := range rows {
key, err := r.NomsMapKey(cr.joiner.SchemaForName(rowType)).Value(ctx)
if err != nil {
return nil, err
}
return key, nil
}
return nil, errors.New("could not determine key")
}
// Close should release resources being held
func (cr *ConflictReader) Close() error {
return nil
}
go/libraries/doltcore/merge/keyless_integration_test.go
+4 -61
View File
@@ -325,11 +325,8 @@ func TestKeylessMergeConflicts(t *testing.T) {
}
func assertConflicts(t *testing.T, ctx context.Context, tbl *doltdb.Table, expected conflictEntries) {
if types.IsFormat_DOLT(tbl.Format()) {
assertProllyConflicts(t, ctx, tbl, expected)
return
}
assertNomsConflicts(t, ctx, tbl, expected)
types.AssertFormat_DOLT(tbl.Format())
assertProllyConflicts(t, ctx, tbl, expected)
}
func assertProllyConflicts(t *testing.T, ctx context.Context, tbl *doltdb.Table, expected conflictEntries) {
@@ -381,32 +378,6 @@ func assertProllyConflicts(t *testing.T, ctx context.Context, tbl *doltdb.Table,
}
func assertNomsConflicts(t *testing.T, ctx context.Context, tbl *doltdb.Table, expected conflictEntries) {
_, confIdx, err := tbl.GetConflicts(ctx)
require.NoError(t, err)
conflicts := durable.NomsMapFromConflictIndex(confIdx)
assert.True(t, conflicts.Len() > 0)
assert.Equal(t, int(conflicts.Len()), len(expected))
expectedSet := expected.toTupleSet()
actual, err := conflicts.Iterator(ctx)
require.NoError(t, err)
for {
_, act, err := actual.Next(ctx)
if act == nil {
return
}
assert.NoError(t, err)
h, err := act.Hash(types.Format_Default)
assert.NoError(t, err)
exp, ok := expectedSet[h]
assert.True(t, ok)
assert.True(t, exp.Equals(act))
}
}
func mustGetRowValueFromTable(t *testing.T, ctx context.Context, tbl *doltdb.Table, key val.Tuple) val.Tuple {
idx, err := tbl.GetRowData(ctx)
require.NoError(t, err)
@@ -434,12 +405,8 @@ func mustGetRowValueFromRootIsh(t *testing.T, ctx context.Context, vrw types.Val
// |expected| is a tupleSet to compensate for random storage order
func assertKeylessRows(t *testing.T, ctx context.Context, tbl *doltdb.Table, expected keylessEntries) {
if types.IsFormat_DOLT(tbl.Format()) {
assertKeylessProllyRows(t, ctx, tbl, expected)
return
}
assertKeylessNomsRows(t, ctx, tbl, expected)
types.AssertFormat_DOLT(tbl.Format())
assertKeylessProllyRows(t, ctx, tbl, expected)
}
func assertKeylessProllyRows(t *testing.T, ctx context.Context, tbl *doltdb.Table, expected []keylessEntry) {
@@ -470,30 +437,6 @@ func assertKeylessProllyRows(t *testing.T, ctx context.Context, tbl *doltdb.Tabl
require.Equal(t, len(expected), c)
}
func assertKeylessNomsRows(t *testing.T, ctx context.Context, tbl *doltdb.Table, expected keylessEntries) {
rowData, err := tbl.GetNomsRowData(ctx)
require.NoError(t, err)
assert.Equal(t, int(rowData.Len()), len(expected))
expectedSet := expected.toTupleSet()
actual, err := rowData.Iterator(ctx)
require.NoError(t, err)
for {
_, act, err := actual.Next(ctx)
if act == nil {
break
}
assert.NoError(t, err)
h, err := act.Hash(types.Format_Default)
assert.NoError(t, err)
exp, ok := expectedSet[h]
assert.True(t, ok)
assert.True(t, exp.Equals(act))
}
}
const tblName = "noKey"
var keylessSch = dtu.MustSchema(
go/libraries/doltcore/merge/merge.go
+10 -96
View File
@@ -48,6 +48,12 @@ var ErrMultipleViolationsForRow = errors.New("multiple violations for row not su
var ErrSameTblAddedTwice = goerrors.NewKind("table with same name '%s' added in 2 commits can't be merged")
const (
ConflictDiffTypeAdded = "added"
ConflictDiffTypeModified = "modified"
ConflictDiffTypeRemoved = "removed"
)
func MergeCommits(ctx *sql.Context, tableResolver doltdb.TableResolver, commit, mergeCommit *doltdb.Commit, opts editor.Options) (*Result, error) {
optCmt, err := doltdb.GetCommitAncestor(ctx, commit, mergeCommit)
if err != nil {
@@ -169,23 +175,12 @@ func MergeRoots(
mergeOpts MergeOpts,
) (*Result, error) {
var (
conflictStash *conflictStash
violationStash *violationStash
nbf *types.NomsBinFormat
err error
nbf *types.NomsBinFormat
err error
)
nbf = ourRoot.VRW().Format()
if !types.IsFormat_DOLT(nbf) {
ourRoot, conflictStash, err = stashConflicts(ctx, ourRoot)
if err != nil {
return nil, err
}
ancRoot, violationStash, err = stashViolations(ctx, ancRoot)
if err != nil {
return nil, err
}
}
types.AssertFormat_DOLT(nbf)
// merge collations
oColl, err := ourRoot.GetCollation(ctx)
@@ -284,12 +279,7 @@ func MergeRoots(
continue
}
if mergedTable.conflict.Count() > 0 {
if types.IsFormat_DOLT(nbf) {
schConflicts = append(schConflicts, mergedTable.conflict)
} else {
// return schema conflict as error
return nil, mergedTable.conflict
}
schConflicts = append(schConflicts, mergedTable.conflict)
}
if mergedTable.table != nil {
@@ -387,39 +377,11 @@ func MergeRoots(
return nil, err
}
if types.IsFormat_DOLT(ourRoot.VRW().Format()) {
err = getConstraintViolationStats(ctx, mergedRoot, tblToStats)
if err != nil {
return nil, err
}
return &Result{
Root: mergedRoot,
SchemaConflicts: schConflicts,
Stats: tblToStats,
}, nil
}
mergedRoot, err = mergeCVsWithStash(ctx, mergedRoot, violationStash)
if err != nil {
return nil, err
}
err = getConstraintViolationStats(ctx, mergedRoot, tblToStats)
if err != nil {
return nil, err
}
mergedHasTableConflicts := checkForTableConflicts(tblToStats)
if !conflictStash.Empty() && mergedHasTableConflicts {
return nil, ErrCantOverwriteConflicts
} else if !conflictStash.Empty() {
mergedRoot, err = applyConflictStash(ctx, conflictStash.Stash, mergedRoot)
if err != nil {
return nil, err
}
}
return &Result{
Root: mergedRoot,
SchemaConflicts: schConflicts,
@@ -427,54 +389,6 @@ func MergeRoots(
}, nil
}
// mergeCVsWithStash merges the table constraint violations in |stash| with |root|.
// Returns an updated root with all the merged CVs.
func mergeCVsWithStash(ctx context.Context, root doltdb.RootValue, stash *violationStash) (doltdb.RootValue, error) {
updatedRoot := root
for name, stashed := range stash.Stash {
tbl, ok, err := root.GetTable(ctx, doltdb.TableName{Name: name})
if err != nil {
return nil, err
}
if !ok {
// the table with the CVs was deleted
continue
}
curr, err := tbl.GetConstraintViolations(ctx)
if err != nil {
return nil, err
}
unioned, err := types.UnionMaps(ctx, curr, stashed, func(key types.Value, currV types.Value, stashV types.Value) (types.Value, error) {
if !currV.Equals(stashV) {
panic(fmt.Sprintf("encountered conflict when merging constraint violations, conflicted key: %v\ncurrent value: %v\nstashed value: %v\n", key, currV, stashV))
}
return currV, nil
})
if err != nil {
return nil, err
}
tbl, err = tbl.SetConstraintViolations(ctx, unioned)
if err != nil {
return nil, err
}
updatedRoot, err = root.PutTable(ctx, doltdb.TableName{Name: name}, tbl)
if err != nil {
return nil, err
}
}
return updatedRoot, nil
}
// Checks if a table conflict occurred during the merge
func checkForTableConflicts(tblToStats map[doltdb.TableName]*MergeStats) bool {
for _, stat := range tblToStats {
if stat.HasConflicts() && !stat.HasRootObjectConflicts() {
return true
}
}
return false
}
// populates tblToStats with violation statistics
func getConstraintViolationStats(ctx context.Context, root doltdb.RootValue, tblToStats map[doltdb.TableName]*MergeStats) error {
for tblName, stats := range tblToStats {
go/libraries/doltcore/merge/merge_rows.go
+3 -32
View File
@@ -20,7 +20,6 @@ import (
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/libraries/doltcore/conflict"
"github.com/dolthub/dolt/go/libraries/doltcore/diff"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
@@ -213,6 +212,8 @@ func (rm *RootMerger) MergeTable(
return nil, nil, err
}
types.AssertFormat_DOLT(tm.vrw.Format())
// short-circuit here if we can
finished, finishedRootObj, stats, err := rm.MaybeShortCircuit(ctx, tm, mergeOpts)
if finished != nil || finishedRootObj != nil || stats != nil || err != nil {
@@ -243,11 +244,7 @@ func (rm *RootMerger) MergeTable(
var tbl *doltdb.Table
var rootObj doltdb.RootObject
if !tm.InvolvesRootObjects() {
if types.IsFormat_DOLT(tm.vrw.Format()) {
tbl, stats, err = mergeProllyTable(ctx, tm, mergeSch, mergeInfo, diffInfo)
} else {
panic("data format not supported in this version of Dolt")
}
tbl, stats, err = mergeProllyTable(ctx, tm, mergeSch, mergeInfo, diffInfo)
if err != nil {
return nil, nil, err
}
@@ -501,32 +498,6 @@ var MergeRootObjects = func(ctx context.Context, mro MergeRootObject) (doltdb.Ro
return nil, nil, errors.New("Dolt does not operate on root objects")
}
func setConflicts(ctx context.Context, cons durable.ConflictIndex, tbl, mergeTbl, ancTbl, tableToUpdate *doltdb.Table) (*doltdb.Table, error) {
ancSch, err := ancTbl.GetSchema(ctx)
if err != nil {
return nil, err
}
sch, err := tbl.GetSchema(ctx)
if err != nil {
return nil, err
}
mergeSch, err := mergeTbl.GetSchema(ctx)
if err != nil {
return nil, err
}
cs := conflict.NewConflictSchema(ancSch, sch, mergeSch)
tableToUpdate, err = tableToUpdate.SetConflicts(ctx, cs, cons)
if err != nil {
return nil, err
}
return tableToUpdate, nil
}
func calcTableMergeStats(ctx context.Context, tbl *doltdb.Table, mergeTbl *doltdb.Table) (MergeStats, error) {
ms := MergeStats{Operation: TableModified}
go/libraries/doltcore/merge/merge_schema.go
+2 -2
View File
@@ -187,10 +187,10 @@ func SchemaMerge(
// TODO: We'll remove this once it's possible to get diff and merge on different primary key sets
// TODO: decide how to merge different orders of PKS
if !schema.ArePrimaryKeySetsDiffable(format, ourSch, theirSch) {
if !schema.ArePrimaryKeySetsDiffable(ourSch, theirSch) {
return nil, SchemaConflict{}, mergeInfo, diffInfo, ErrMergeWithDifferentPks.New(tblName)
}
if !schema.ArePrimaryKeySetsDiffable(format, ourSch, ancSch) {
if !schema.ArePrimaryKeySetsDiffable(ourSch, ancSch) {
return nil, SchemaConflict{}, mergeInfo, diffInfo, ErrMergeWithDifferentPksFromAncestor.New(tblName)
}
go/libraries/doltcore/merge/merge_test.go
-4
View File
@@ -297,10 +297,6 @@ var testRows = []testRow{
}
func TestMergeCommits(t *testing.T) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip()
}
ddb, vrw, ns, rightCommitHash, ancCommitHash, root, mergeRoot, ancRoot, expectedRows, expectedArtifacts := setupMergeTest(t)
defer ddb.Close()
merger, err := NewMerger(root, mergeRoot, ancRoot, rightCommitHash, ancCommitHash, vrw, ns)
go/libraries/doltcore/merge/schema_merge_test.go
+4 -11
View File
@@ -40,7 +40,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
type schemaMergeTest struct {
@@ -1654,7 +1653,7 @@ func testSchemaMergeHelper(t *testing.T, tests []schemaMergeTest, flipSides bool
eo = eo.WithDeaf(editor.NewInMemDeaf(a.VRW()))
// attempt merge before skipping to assert no panics
result, err := merge.MergeRoots(sql.NewContext(ctx), doltdb.SimpleTableResolver{}, l, r, a, rootish{r}, rootish{a}, eo, mo)
maybeSkip(t, a.VRW().Format(), test, flipSides)
maybeSkip(t, test, flipSides)
if test.conflict {
// TODO: Test the conflict error message more deeply
@@ -1829,15 +1828,9 @@ func verifyMerge(t *testing.T, ctx context.Context, m doltdb.RootValue, result *
}
}
func maybeSkip(t *testing.T, nbf *types.NomsBinFormat, test schemaMergeTest, flipSides bool) {
if types.IsFormat_DOLT(nbf) {
if test.skipNewFmt || flipSides && test.skipFlipOnNewFormat {
t.Skip()
}
} else {
if test.skipOldFmt || flipSides && test.skipFlipOnOldFormat {
t.Skip()
}
func maybeSkip(t *testing.T, test schemaMergeTest, flipSides bool) {
if test.skipNewFmt || flipSides && test.skipFlipOnNewFormat {
t.Skip()
}
}
go/libraries/doltcore/merge/stash.go
-182
View File
@@ -1,182 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package merge
import (
"context"
"github.com/dolthub/dolt/go/libraries/doltcore/conflict"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/store/types"
)
type conflictStash struct {
Stash map[string]*conflictData
}
type conflictData struct {
HasConflicts bool
Sch conflict.ConflictSchema
ConfIdx durable.ConflictIndex
}
// Empty returns false if any table has a conflict.
// True otherwise.
func (s *conflictStash) Empty() bool {
for _, data := range s.Stash {
if data.HasConflicts {
return false
}
}
return true
}
type violationStash struct {
// todo: durable
Stash map[string]types.Map
}
// Empty returns false if any table has constraint violations.
// True otherwise.
func (s *violationStash) Empty() bool {
for _, data := range s.Stash {
if data.Len() > 0 {
return false
}
}
return true
}
func stashConflicts(ctx context.Context, root doltdb.RootValue) (doltdb.RootValue, *conflictStash, error) {
names, err := root.GetTableNames(ctx, doltdb.DefaultSchemaName, true)
if err != nil {
return nil, nil, err
}
updatedRoot := root
stash := make(map[string]*conflictData, len(names))
for _, name := range names {
tbl, _, err := root.GetTable(ctx, doltdb.TableName{Name: name})
if err != nil {
return nil, nil, err
}
d, err := getConflictData(ctx, tbl)
if err != nil {
return nil, nil, err
}
stash[name] = d
tbl, err = tbl.ClearConflicts(ctx)
if err != nil {
return nil, nil, err
}
updatedRoot, err = updatedRoot.PutTable(ctx, doltdb.TableName{Name: name}, tbl)
if err != nil {
return nil, nil, err
}
}
return updatedRoot, &conflictStash{stash}, nil
}
func stashViolations(ctx context.Context, root doltdb.RootValue) (doltdb.RootValue, *violationStash, error) {
names, err := root.GetTableNames(ctx, doltdb.DefaultSchemaName, true)
if err != nil {
return nil, nil, err
}
updatedRoot := root
stash := make(map[string]types.Map, len(names))
for _, name := range names {
tbl, _, err := root.GetTable(ctx, doltdb.TableName{Name: name})
if err != nil {
return nil, nil, err
}
v, err := tbl.GetConstraintViolations(ctx)
stash[name] = v
tbl, err = tbl.SetConstraintViolations(ctx, types.EmptyMap)
if err != nil {
return nil, nil, err
}
updatedRoot, err = updatedRoot.PutTable(ctx, doltdb.TableName{Name: name}, tbl)
if err != nil {
return nil, nil, err
}
}
return updatedRoot, &violationStash{stash}, nil
}
// applyConflictStash applies the data in |stash| to the root value. Missing
// tables will be skipped. This function will override any previous conflict
// data.
func applyConflictStash(ctx context.Context, stash map[string]*conflictData, root doltdb.RootValue) (doltdb.RootValue, error) {
updatedRoot := root
for name, data := range stash {
tbl, ok, err := root.GetTable(ctx, doltdb.TableName{Name: name})
if err != nil {
return nil, err
}
if !ok {
continue
}
tbl, err = setConflictData(ctx, tbl, data)
if err != nil {
return nil, err
}
updatedRoot, err = updatedRoot.PutTable(ctx, doltdb.TableName{Name: name}, tbl)
if err != nil {
return nil, err
}
}
return updatedRoot, nil
}
func getConflictData(ctx context.Context, tbl *doltdb.Table) (*conflictData, error) {
var sch conflict.ConflictSchema
var confIdx durable.ConflictIndex
hasCnf, err := tbl.HasConflicts(ctx)
if err != nil {
return nil, err
}
if hasCnf {
sch, confIdx, err = tbl.GetConflicts(ctx)
if err != nil {
return nil, err
}
}
return &conflictData{
HasConflicts: hasCnf,
Sch: sch,
ConfIdx: confIdx,
}, nil
}
func setConflictData(ctx context.Context, tbl *doltdb.Table, data *conflictData) (*doltdb.Table, error) {
var err error
if !data.HasConflicts {
tbl, err = tbl.ClearConflicts(ctx)
} else {
tbl, err = tbl.SetConflicts(ctx, data.Sch, data.ConfIdx)
}
if err != nil {
return nil, err
}
return tbl, nil
}
go/libraries/doltcore/merge/violations_fk.go
+18 -313
View File
@@ -18,21 +18,14 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"time"
"github.com/dolthub/go-mysql-server/sql"
gmstypes "github.com/dolthub/go-mysql-server/sql/types"
"github.com/dolthub/dolt/go/libraries/doltcore/diff"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
json2 "github.com/dolthub/dolt/go/libraries/doltcore/sqle/json"
"github.com/dolthub/dolt/go/libraries/doltcore/table"
"github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms"
diff2 "github.com/dolthub/dolt/go/store/diff"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/types"
@@ -118,13 +111,13 @@ func RegisterForeignKeyViolations(
if err != nil {
return err
}
err = parentFkConstraintViolations(ctx, baseRoot.VRW(), foreignKey, postParent, postParent, postChild, emptyIdx, receiver)
err = parentFkConstraintViolations(ctx, foreignKey, postParent, postParent, postChild, emptyIdx, receiver)
if err != nil {
return err
}
} else {
// Parent exists in the ancestor
err = parentFkConstraintViolations(ctx, baseRoot.VRW(), foreignKey, preParent, postParent, postChild, preParent.RowData, receiver)
err = parentFkConstraintViolations(ctx, foreignKey, preParent, postParent, postChild, preParent.RowData, receiver)
if err != nil {
return err
}
@@ -266,54 +259,28 @@ func (f *foreignKeyViolationWriter) StartFK(ctx *sql.Context, fk doltdb.ForeignK
return err
}
if types.IsFormat_DOLT(tbl.Format()) {
arts, err := tbl.GetArtifacts(ctx)
if err != nil {
return err
}
artMap := durable.ProllyMapFromArtifactIndex(arts)
f.artEditor = artMap.Editor()
f.cInfoJsonData = jsonData
f.kd = sch.GetKeyDescriptor(tbl.NodeStore())
} else {
violMap, err := tbl.GetConstraintViolations(ctx)
if err != nil {
return err
}
f.violMapEditor = violMap.Edit()
f.nomsVInfo, err = jsonDataToNomsValue(ctx, tbl.ValueReadWriter(), jsonData)
if err != nil {
return err
}
types.AssertFormat_DOLT(tbl.Format())
arts, err := tbl.GetArtifacts(ctx)
if err != nil {
return err
}
artMap := durable.ProllyMapFromArtifactIndex(arts)
f.artEditor = artMap.Editor()
f.cInfoJsonData = jsonData
f.kd = sch.GetKeyDescriptor(tbl.NodeStore())
return nil
}
func (f *foreignKeyViolationWriter) EndCurrFK(ctx context.Context) error {
if types.IsFormat_DOLT(f.currTbl.Format()) {
artMap, err := f.artEditor.Flush(ctx)
if err != nil {
return err
}
artIdx := durable.ArtifactIndexFromProllyMap(artMap)
tbl, err := f.currTbl.SetArtifacts(ctx, artIdx)
if err != nil {
return err
}
f.rootValue, err = f.rootValue.PutTable(ctx, f.currFk.TableName, tbl)
if err != nil {
return err
}
return nil
}
types.AssertFormat_DOLT(f.currTbl.Format())
violMap, err := f.violMapEditor.Map(ctx)
artMap, err := f.artEditor.Flush(ctx)
if err != nil {
return err
}
tbl, err := f.currTbl.SetConstraintViolations(ctx, violMap)
artIdx := durable.ArtifactIndexFromProllyMap(artMap)
tbl, err := f.currTbl.SetArtifacts(ctx, artIdx)
if err != nil {
return err
}
@@ -325,21 +292,18 @@ func (f *foreignKeyViolationWriter) EndCurrFK(ctx context.Context) error {
}
func (f *foreignKeyViolationWriter) NomsFKViolationFound(ctx context.Context, rowKey, rowValue types.Tuple) error {
cvKey, cvVal, err := toConstraintViolationRow(ctx, CvType_ForeignKey, f.nomsVInfo, rowKey, rowValue)
if err != nil {
return err
}
f.violMapEditor.Set(cvKey, cvVal)
f.violatedTables.Add(f.currFk.TableName)
return nil
}
func (f *foreignKeyViolationWriter) ProllyFKViolationFound(ctx context.Context, rowKey, rowValue val.Tuple) error {
meta := prolly.ConstraintViolationMeta{VInfo: f.cInfoJsonData, Value: rowValue}
err := f.artEditor.ReplaceConstraintViolation(ctx, rowKey, f.theirRootIsh, prolly.ArtifactTypeForeignKeyViol, meta)
@@ -357,16 +321,15 @@ var _ FKViolationReceiver = (*foreignKeyViolationWriter)(nil)
// parentFkConstraintViolations processes foreign key constraint violations for the parent in a foreign key.
func parentFkConstraintViolations(
ctx context.Context,
vr types.ValueReader,
foreignKey doltdb.ForeignKey,
preParent, postParent, postChild *constraintViolationsLoadedTable,
preParentRowData durable.Index,
receiver FKViolationReceiver,
) error {
if preParentRowData.Format() != types.Format_DOLT {
m := durable.NomsMapFromIndex(preParentRowData)
return nomsParentFkConstraintViolations(ctx, vr, foreignKey, postParent, postChild, preParent.Schema, m, receiver)
panic("unsupported format: " + preParentRowData.Format().VersionString())
}
if preParent.IndexData == nil || postParent.Schema.GetPKCols().Size() == 0 || preParent.Schema.GetPKCols().Size() == 0 {
m, err := durable.ProllyMapFromIndex(preParentRowData)
if err != nil {
@@ -405,9 +368,9 @@ func childFkConstraintViolations(
receiver FKViolationReceiver,
) error {
if preChildRowData.Format() != types.Format_DOLT {
m := durable.NomsMapFromIndex(preChildRowData)
return nomsChildFkConstraintViolations(ctx, vr, foreignKey, postParent, postChild, preChild.Schema, m, receiver)
panic("unsupported format: " + preChildRowData.Format().VersionString())
}
if preChild.IndexData == nil || postChild.Schema.GetPKCols().Size() == 0 || preChild.Schema.GetPKCols().Size() == 0 {
m, err := durable.ProllyMapFromIndex(preChildRowData)
if err != nil {
@@ -436,264 +399,6 @@ func childFkConstraintViolations(
return prollyChildSecDiffFkConstraintViolations(ctx, foreignKey, postParent, postChild, m, receiver)
}
func nomsParentFkConstraintViolations(
ctx context.Context,
vr types.ValueReader,
foreignKey doltdb.ForeignKey,
postParent, postChild *constraintViolationsLoadedTable,
preParentSch schema.Schema,
preParentRowData types.Map,
receiver FKViolationReceiver) error {
postParentIndexTags := postParent.Index.IndexedColumnTags()
postChildIndexTags := postChild.Index.IndexedColumnTags()
differ := diff.NewRowDiffer(ctx, preParentRowData.Format(), preParentSch, postParent.Schema, 1024)
defer differ.Close()
differ.Start(ctx, preParentRowData, durable.NomsMapFromIndex(postParent.RowData))
for {
diffSlice, hasMore, err := differ.GetDiffs(1, 10*time.Second)
if err != nil {
return err
}
if len(diffSlice) != 1 {
if hasMore {
return fmt.Errorf("no diff returned but should have errored earlier")
}
break
}
rowDiff := diffSlice[0]
switch rowDiff.ChangeType {
case types.DiffChangeRemoved, types.DiffChangeModified:
postParentRow, err := row.FromNoms(postParent.Schema, rowDiff.KeyValue.(types.Tuple), rowDiff.OldValue.(types.Tuple))
if err != nil {
return err
}
hasNulls := false
for _, tag := range postParentIndexTags {
if postParentRowEntry, ok := postParentRow.GetColVal(tag); !ok || types.IsNull(postParentRowEntry) {
hasNulls = true
break
}
}
if hasNulls {
continue
}
postParentIndexPartialKey, err := row.ReduceToIndexPartialKey(foreignKey.TableColumns, postParent.Index, postParentRow)
if err != nil {
return err
}
shouldContinue, err := func() (bool, error) {
var mapIter table.ReadCloser = noms.NewNomsRangeReader(
vr,
postParent.IndexSchema,
durable.NomsMapFromIndex(postParent.IndexData),
[]*noms.ReadRange{{Start: postParentIndexPartialKey, Inclusive: true, Reverse: false, Check: noms.InRangeCheckPartial(postParentIndexPartialKey)}})
defer mapIter.Close(ctx)
if _, err := mapIter.ReadRow(ctx); err == nil {
// If the parent table has other rows that satisfy the partial key then we choose to do nothing
return true, nil
} else if err != io.EOF {
return false, err
}
return false, nil
}()
if err != nil {
return err
}
if shouldContinue {
continue
}
postParentIndexPartialKeySlice, err := postParentIndexPartialKey.AsSlice()
if err != nil {
return err
}
for i := 0; i < len(postChildIndexTags); i++ {
postParentIndexPartialKeySlice[2*i] = types.Uint(postChildIndexTags[i])
}
postChildIndexPartialKey, err := types.NewTuple(postChild.Table.Format(), postParentIndexPartialKeySlice...)
if err != nil {
return err
}
err = nomsParentFkConstraintViolationsProcess(ctx, vr, foreignKey, postChild, postChildIndexPartialKey, receiver)
if err != nil {
return err
}
case types.DiffChangeAdded:
// We don't do anything if a parent row was added
default:
return fmt.Errorf("unknown diff change type")
}
if !hasMore {
break
}
}
return nil
}
func nomsParentFkConstraintViolationsProcess(
ctx context.Context,
vr types.ValueReader,
foreignKey doltdb.ForeignKey,
postChild *constraintViolationsLoadedTable,
postChildIndexPartialKey types.Tuple,
receiver FKViolationReceiver,
) error {
indexData := durable.NomsMapFromIndex(postChild.IndexData)
rowData := durable.NomsMapFromIndex(postChild.RowData)
mapIter := noms.NewNomsRangeReader(
vr,
postChild.IndexSchema,
indexData,
[]*noms.ReadRange{{Start: postChildIndexPartialKey, Inclusive: true, Reverse: false, Check: noms.InRangeCheckPartial(postChildIndexPartialKey)}})
defer mapIter.Close(ctx)
var postChildIndexRow row.Row
var err error
for postChildIndexRow, err = mapIter.ReadRow(ctx); err == nil; postChildIndexRow, err = mapIter.ReadRow(ctx) {
postChildIndexKey, err := postChildIndexRow.NomsMapKey(postChild.IndexSchema).Value(ctx)
if err != nil {
return err
}
postChildRowKey, err := postChild.Index.ToTableTuple(ctx, postChildIndexKey.(types.Tuple), postChild.Table.Format())
if err != nil {
return err
}
postChildRowVal, ok, err := rowData.MaybeGetTuple(ctx, postChildRowKey)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("index %s on %s contains data that table does not", foreignKey.TableIndex, foreignKey.TableName)
}
err = receiver.NomsFKViolationFound(ctx, postChildRowKey, postChildRowVal)
if err != nil {
return err
}
}
if err != io.EOF {
return err
}
return nil
}
// nomsChildFkConstraintViolations processes foreign key constraint violations for the child in a foreign key.
func nomsChildFkConstraintViolations(
ctx context.Context,
vr types.ValueReader,
foreignKey doltdb.ForeignKey,
postParent, postChild *constraintViolationsLoadedTable,
preChildSch schema.Schema,
preChildRowData types.Map,
receiver FKViolationReceiver,
) error {
var postParentIndexTags, postChildIndexTags []uint64
if postParent.Index.Name() == "" {
postParentIndexTags = foreignKey.ReferencedTableColumns
postChildIndexTags = foreignKey.TableColumns
} else {
postParentIndexTags = postParent.Index.IndexedColumnTags()
postChildIndexTags = postChild.Index.IndexedColumnTags()
}
differ := diff.NewRowDiffer(ctx, preChildRowData.Format(), preChildSch, postChild.Schema, 1024)
defer differ.Close()
differ.Start(ctx, preChildRowData, durable.NomsMapFromIndex(postChild.RowData))
for {
diffSlice, hasMore, err := differ.GetDiffs(1, 10*time.Second)
if err != nil {
return err
}
if len(diffSlice) != 1 {
if hasMore {
return fmt.Errorf("no diff returned but should have errored earlier")
}
break
}
rowDiff := diffSlice[0]
switch rowDiff.ChangeType {
case types.DiffChangeAdded, types.DiffChangeModified:
postChildRow, err := row.FromNoms(postChild.Schema, rowDiff.KeyValue.(types.Tuple), rowDiff.NewValue.(types.Tuple))
if err != nil {
return err
}
hasNulls := false
for _, tag := range postChildIndexTags {
if postChildRowEntry, ok := postChildRow.GetColVal(tag); !ok || types.IsNull(postChildRowEntry) {
hasNulls = true
break
}
}
if hasNulls {
continue
}
postChildIndexPartialKey, err := row.ReduceToIndexPartialKey(postChildIndexTags, postChild.Index, postChildRow)
if err != nil {
return err
}
postChildIndexPartialKeySlice, err := postChildIndexPartialKey.AsSlice()
if err != nil {
return err
}
for i := 0; i < len(postParentIndexTags); i++ {
postChildIndexPartialKeySlice[2*i] = types.Uint(postParentIndexTags[i])
}
parentPartialKey, err := types.NewTuple(postChild.Table.Format(), postChildIndexPartialKeySlice...)
if err != nil {
return err
}
err = childFkConstraintViolationsProcess(ctx, vr, postParent, rowDiff, parentPartialKey, receiver)
if err != nil {
return err
}
case types.DiffChangeRemoved:
// We don't do anything if a child row was removed
default:
return fmt.Errorf("unknown diff change type")
}
if !hasMore {
break
}
}
return nil
}
// childFkConstraintViolationsProcess handles processing the constraint violations for the child of a foreign key.
func childFkConstraintViolationsProcess(
ctx context.Context,
vr types.ValueReader,
postParent *constraintViolationsLoadedTable,
rowDiff *diff2.Difference,
parentPartialKey types.Tuple,
receiver FKViolationReceiver,
) error {
var mapIter table.ReadCloser = noms.NewNomsRangeReader(
vr,
postParent.IndexSchema,
durable.NomsMapFromIndex(postParent.IndexData),
[]*noms.ReadRange{{Start: parentPartialKey, Inclusive: true, Reverse: false, Check: noms.InRangeCheckPartial(parentPartialKey)}})
defer mapIter.Close(ctx)
// If the row exists in the parent, then we don't need to do anything
if _, err := mapIter.ReadRow(ctx); err != nil {
if err != io.EOF {
return err
}
err = receiver.NomsFKViolationFound(ctx, rowDiff.KeyValue.(types.Tuple), rowDiff.NewValue.(types.Tuple))
if err != nil {
return err
}
return nil
}
return nil
}
// newConstraintViolationsLoadedTable returns a *constraintViolationsLoadedTable. Returns false if the table was loaded
// but the index could not be found. If the table could not be found, then an error is returned.
func newConstraintViolationsLoadedTable(
go/libraries/doltcore/migrate/environment.go
-265
View File
@@ -1,265 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package migrate
import (
"context"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/utils/earl"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/types"
)
const (
doltDir = dbfactory.DoltDir
nomsDir = dbfactory.DataDir
oldGenDir = "oldgen"
manifestFile = "manifest"
migrationRef = "migration"
)
var (
targetFormat = types.Format_DOLT
migrationMsg = fmt.Sprintf("migrating database to Noms Binary Format %s", targetFormat.VersionString())
)
// Environment is a migration environment.
type Environment struct {
Migration *env.DoltEnv
Existing *env.DoltEnv
DropConflicts bool
}
// NewEnvironment creates a migration Environment for |existing|.
func NewEnvironment(ctx context.Context, existing *env.DoltEnv) (Environment, error) {
mfs, err := getMigrateFS(existing.FS)
if err != nil {
return Environment{}, err
}
if err = initMigrationDB(ctx, existing, existing.FS, mfs); err != nil {
return Environment{}, err
}
mdb, err := doltdb.LoadDoltDB(ctx, targetFormat, doltdb.LocalDirDoltDB, mfs)
if err != nil {
return Environment{}, err
}
config, err := env.LoadDoltCliConfig(env.GetCurrentUserHomeDir, mfs)
if err != nil {
return Environment{}, err
}
migration := env.NewDoltEnv(
existing.Version,
config,
existing.RepoState,
mdb,
mfs,
)
return Environment{
Migration: migration,
Existing: existing,
}, nil
}
func initMigrationDB(ctx context.Context, existing *env.DoltEnv, src, dest filesys.Filesys) (err error) {
base, err := src.Abs(".")
if err != nil {
return err
}
ierr := src.Iter(doltDir, true, func(path string, size int64, isDir bool) (stop bool) {
path, err = filepath.Rel(base, path)
if err != nil {
stop = true
return
}
if isDir {
err = dest.MkDirs(path)
stop = err != nil
return
}
if strings.Contains(path, nomsDir) {
return
}
if err = filesys.CopyFile(path, path, src, dest); err != nil {
stop = true
return
}
return
})
if ierr != nil {
return ierr
}
if err != nil {
return err
}
absPath, err := dest.Abs(filepath.Join(doltDir, nomsDir))
if err != nil {
return err
}
if err = dest.MkDirs(absPath); err != nil {
return err
}
u, err := earl.Parse("file://" + filepath.ToSlash(absPath))
if err != nil {
return err
}
params := map[string]any{dbfactory.ChunkJournalParam: struct{}{}}
ddb, err := doltdb.LoadDoltDBWithParams(ctx, targetFormat, u.String(), dest, params)
vrw := ddb.ValueReadWriter()
ns := ddb.NodeStore()
db := doltdb.HackDatasDatabaseFromDoltDB(ddb)
// write init commit for migration
name, email, err := env.GetNameAndEmail(existing.Config)
if err != nil {
return err
}
meta, err := datas.NewCommitMeta(name, email, migrationMsg)
if err != nil {
return err
}
rv, err := doltdb.EmptyRootValue(ctx, vrw, ns)
if err != nil {
return err
}
ds, err := db.GetDataset(ctx, ref.NewInternalRef(migrationRef).String())
if err != nil {
return err
}
_, err = db.Commit(ctx, ds, rv.NomsValue(), datas.CommitOptions{Meta: meta})
return nil
}
// SwapChunkStores atomically swaps the ChunkStores of |menv.Migration| and |menv.Existing|.
func SwapChunkStores(ctx context.Context, menv Environment) error {
src, dest := menv.Migration.FS, menv.Existing.FS
absSrc, err := src.Abs(filepath.Join(doltDir, nomsDir))
if err != nil {
return err
}
absDest, err := dest.Abs(filepath.Join(doltDir, nomsDir))
if err != nil {
return err
}
var cpErr error
err = src.Iter(absSrc, true, func(p string, size int64, isDir bool) (stop bool) {
if strings.Contains(p, manifestFile) || isDir {
return
}
var relPath string
if relPath, cpErr = filepath.Rel(absSrc, p); cpErr != nil {
stop = true
return
}
srcPath := filepath.Join(absSrc, relPath)
destPath := filepath.Join(absDest, relPath)
if cpErr = filesys.CopyFile(srcPath, destPath, src, dest); cpErr != nil {
stop = true
}
return
})
if err != nil {
return err
}
if cpErr != nil {
return cpErr
}
return swapManifests(ctx, src, dest)
}
func swapManifests(ctx context.Context, src, dest filesys.Filesys) (err error) {
// backup the current manifest
manifest := filepath.Join(doltDir, nomsDir, manifestFile)
bak := filepath.Join(doltDir, nomsDir, manifestFile+".bak")
if err = filesys.CopyFile(manifest, bak, dest, dest); err != nil {
return err
}
// backup the current oldgen manifest, if one exists
gcManifest := filepath.Join(doltDir, nomsDir, oldGenDir, manifestFile)
oldGen, _ := dest.Exists(gcManifest)
if oldGen {
bak = filepath.Join(doltDir, nomsDir, oldGenDir, manifestFile+".bak")
if err = filesys.CopyFile(gcManifest, bak, dest, dest); err != nil {
return err
}
}
// copy manifest to |dest| under temporary name
tmp := filepath.Join(doltDir, nomsDir, "temp-manifest")
if err = filesys.CopyFile(manifest, tmp, src, dest); err != nil {
return err
}
// delete current oldgen manifest
if oldGen {
if err = dest.Delete(gcManifest, true); err != nil {
return err
}
}
// atomically swap the manifests
return dest.MoveFile(tmp, manifest)
// exit immediately!
}
func getMigrateFS(existing filesys.Filesys) (filesys.Filesys, error) {
uniq := fmt.Sprintf("dolt_migration_%d", time.Now().UnixNano())
tmpPath := filepath.Join(existing.TempDir(), uniq)
if err := existing.MkDirs(tmpPath); err != nil {
return nil, err
}
mfs, err := filesys.LocalFilesysWithWorkingDir(tmpPath)
if err != nil {
return nil, err
}
if err = mfs.MkDirs(doltDir); err != nil {
return nil, err
}
return mfs, nil
}
go/libraries/doltcore/migrate/integration_test.go
-263
View File
@@ -1,263 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package migrate_test
import (
"context"
"testing"
"github.com/dolthub/go-mysql-server/sql"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/dolthub/dolt/go/cmd/dolt/commands"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/dtestutils"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/migrate"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
type migrationTest struct {
name string
hook setupHook
setup []string
asserts []assertion
err string
}
// a setupHook performs special setup operations that cannot
// be performed via SQL statements (eg TEXT type PRIMARY KEY)
type setupHook func(context.Context, *env.DoltEnv) (*env.DoltEnv, error)
type assertion struct {
query string
expected []sql.Row
}
func TestMigration(t *testing.T) {
if types.Format_Default != types.Format_LD_1 {
t.Skip()
}
tests := []migrationTest{
{
name: "smoke test",
setup: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (1),(2),(3)",
"CALL dolt_add('.')",
"CALL dolt_commit('-am', 'new table')",
},
asserts: []assertion{
{
query: "SELECT * FROM test",
expected: []sql.Row{{int32(1)}, {int32(2)}, {int32(3)}},
},
{
query: "SELECT count(*) FROM dolt_log",
expected: []sql.Row{{int64(2)}},
},
{
query: "SELECT count(*) FROM `dolt/dolt_migrated_commits`.dolt_commit_mapping",
expected: []sql.Row{{int64(2)}},
},
},
},
{
name: "TEXT primary key, BLOB secondary key",
hook: SetupHookRefKeys,
setup: []string{
// from setup hook:
// CREATE TABLE test (
// pk TEXT PRIMARY KEY,
// c0 int,
// c1 BLOB,
// INDEX blob_idx(c1),
// );
"INSERT INTO test VALUES ('a', 2, 'a')",
"CALL dolt_add('.')",
"CALL dolt_commit('-am', 'new table')",
},
asserts: []assertion{
{
query: "SELECT * FROM test",
expected: []sql.Row{{"a", int32(2), []byte("a")}},
},
{
query: "DESCRIBE test",
expected: []sql.Row{
{"pk", "varchar(16383)", "NO", "PRI", "NULL", ""},
{"c0", "int", "YES", "", "NULL", ""},
{"c1", "varbinary(16383)", "YES", "MUL", "NULL", ""},
},
},
},
},
{
name: "create more commits",
setup: []string{
"CREATE TABLE test (pk int primary key)",
"INSERT INTO test VALUES (1),(2),(3)",
"CALL dolt_commit('-Am', 'new table')",
"INSERT INTO test VALUES (4)",
"CALL dolt_commit('-am', 'added row 4')",
"INSERT INTO test VALUES (5)",
"CALL dolt_commit('-am', 'added row 5')",
},
asserts: []assertion{
{
query: "SELECT count(*) FROM dolt_log",
expected: []sql.Row{{int64(4)}},
},
{
query: "SELECT count(*) FROM `dolt/dolt_migrated_commits`.dolt_commit_mapping",
expected: []sql.Row{{int64(4)}},
},
{
query: "SELECT count(*) FROM `dolt/dolt_migrated_commits`.dolt_commit_mapping WHERE new_commit_hash IN (SELECT commit_hash FROM dolt_log)",
expected: []sql.Row{{int64(4)}},
},
{
query: "SELECT count(*) FROM `dolt/dolt_migrated_commits`.dolt_commit_mapping WHERE new_commit_hash NOT IN (SELECT commit_hash FROM dolt_log)",
expected: []sql.Row{{int64(0)}},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.Background()
preEnv := setupMigrationTest(t, ctx, test)
postEnv := runMigration(t, ctx, preEnv)
root, err := postEnv.WorkingRoot(ctx)
require.NoError(t, err)
for _, a := range test.asserts {
actual, err := sqle.ExecuteSelect(ctx, postEnv, root, a.query)
assert.NoError(t, err)
assert.Equal(t, a.expected, actual)
}
})
}
}
func setupMigrationTest(t *testing.T, ctx context.Context, test migrationTest) *env.DoltEnv {
dEnv := dtestutils.CreateTestEnv()
// run setup hook before other setup queries
if test.hook != nil {
var err error
dEnv, err = test.hook(ctx, dEnv)
require.NoError(t, err)
}
cliCtx, err := commands.NewArgFreeCliContext(ctx, dEnv, dEnv.FS)
require.NoError(t, err)
defer cliCtx.Close()
cmd := commands.SqlCmd{}
for _, query := range test.setup {
code := cmd.Exec(ctx, cmd.Name(), []string{"-q", query}, dEnv, cliCtx)
require.Equal(t, 0, code)
}
return dEnv
}
func SetupHookRefKeys(ctx context.Context, dEnv *env.DoltEnv) (*env.DoltEnv, error) {
pk, _ := schema.NewColumnWithTypeInfo("pk", 1, typeinfo.TextType, true, "", false, "", schema.NotNullConstraint{})
c0, _ := schema.NewColumnWithTypeInfo("c0", 2, typeinfo.Int32Type, false, "", false, "")
c1, _ := schema.NewColumnWithTypeInfo("c1", 3, typeinfo.BlobType, false, "", false, "")
sch, err := schema.SchemaFromCols(schema.NewColCollection(pk, c0, c1))
if err != nil {
return nil, err
}
_, err = sch.Indexes().AddIndexByColNames("blob_idx", []string{"c1"}, nil, schema.IndexProperties{IsUserDefined: true})
if err != nil {
return nil, err
}
ws, err := dEnv.WorkingSet(ctx)
if err != nil {
return nil, err
}
root, err := doltdb.CreateEmptyTable(ctx, ws.WorkingRoot(), doltdb.TableName{Name: "test"}, sch)
if err != nil {
return nil, err
}
if err = dEnv.UpdateWorkingSet(ctx, ws.WithWorkingRoot(root)); err != nil {
return nil, err
}
return dEnv, nil
}
func runMigration(t *testing.T, ctx context.Context, preEnv *env.DoltEnv) (postEnv *env.DoltEnv) {
ddb, err := initTestMigrationDB(ctx)
require.NoError(t, err)
postEnv = env.NewDoltEnv(
preEnv.Version,
preEnv.Config,
preEnv.RepoState,
ddb,
preEnv.FS,
)
err = migrate.TraverseDAG(ctx, migrate.Environment{}, preEnv.DoltDB(ctx), postEnv.DoltDB(ctx))
assert.NoError(t, err)
return
}
func initTestMigrationDB(ctx context.Context) (*doltdb.DoltDB, error) {
var db datas.Database
storage := &chunks.MemoryStorage{}
cs := storage.NewViewWithFormat("__DOLT__")
vrw := types.NewValueStore(cs)
ns := tree.NewNodeStore(cs)
db = datas.NewTypesDatabase(vrw, ns)
name, email := "user", "user@fake.horse"
meta, err := datas.NewCommitMeta(name, email, "test migration")
if err != nil {
return nil, err
}
rv, err := doltdb.EmptyRootValue(ctx, vrw, ns)
if err != nil {
return nil, err
}
ds, err := db.GetDataset(ctx, ref.NewInternalRef("migration").String())
if err != nil {
return nil, err
}
_, err = db.Commit(ctx, ds, rv.NomsValue(), datas.CommitOptions{Meta: meta})
if err != nil {
return nil, err
}
ddb, err := doltdb.DoltDBFromCS(cs, "")
if err != nil {
return nil, err
}
return ddb, nil
}
go/libraries/doltcore/migrate/progress.go
-321
View File
@@ -1,321 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package migrate
import (
"context"
"fmt"
"io"
"time"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/cmd/dolt/cli"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/shim"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
const (
MigratedCommitsBranch = "dolt_migrated_commits"
MigratedCommitsTable = "dolt_commit_mapping"
)
var (
mappingSchema, _ = schema.SchemaFromCols(schema.NewColCollection(
schema.NewColumn("old_commit_hash", 0, types.StringKind, true),
schema.NewColumn("new_commit_hash", 1, types.StringKind, false),
))
desc = val.NewTupleDescriptor(val.Type{Enc: val.StringEnc, Nullable: false})
)
// progress maintains the state of migration.
type progress struct {
stack []*doltdb.Commit
// mapping tracks migrated commits
// it maps old commit hash to new hash
mapping *prolly.MutableMap
kb, vb *val.TupleBuilder
buffPool pool.BuffPool
vs *types.ValueStore
cs chunks.ChunkStore
}
func newProgress(ctx context.Context, cs chunks.ChunkStore) (*progress, error) {
kd := val.NewTupleDescriptor(val.Type{
Enc: val.ByteStringEnc,
Nullable: false,
})
vd := val.NewTupleDescriptor(val.Type{
Enc: val.ByteStringEnc,
Nullable: false,
})
ns := tree.NewNodeStore(cs)
vs := types.NewValueStore(cs)
mapping, err := prolly.NewMapFromTuples(ctx, ns, kd, vd)
if err != nil {
return nil, err
}
mut := mapping.Mutate()
kb := val.NewTupleBuilder(kd, ns)
vb := val.NewTupleBuilder(vd, ns)
return &progress{
stack: make([]*doltdb.Commit, 0, 128),
mapping: mut,
kb: kb,
vb: vb,
buffPool: ns.Pool(),
vs: vs,
cs: cs,
}, nil
}
func (p *progress) Has(ctx context.Context, addr hash.Hash) (ok bool, err error) {
p.kb.PutByteString(0, addr[:])
k, err := p.kb.Build(p.buffPool)
if err != nil {
return false, err
}
return p.mapping.Has(ctx, k)
}
func (p *progress) Get(ctx context.Context, old hash.Hash) (new hash.Hash, err error) {
p.kb.PutByteString(0, old[:])
k, err := p.kb.Build(p.buffPool)
if err != nil {
return new, err
}
err = p.mapping.Get(ctx, k, func(_, v val.Tuple) error {
if len(v) > 0 {
n, ok := p.vb.Desc.GetBytes(0, v)
if !ok {
return fmt.Errorf("failed to get string address from commit mapping value")
}
new = hash.New(n)
}
return nil
})
return new, err
}
func (p *progress) Put(ctx context.Context, old, new hash.Hash) (err error) {
p.kb.PutByteString(0, old[:])
k, err := p.kb.Build(p.buffPool)
if err != nil {
return err
}
p.vb.PutByteString(0, new[:])
v, err := p.vb.Build(p.buffPool)
if err != nil {
return err
}
err = p.mapping.Put(ctx, k, v)
return
}
func (p *progress) Push(ctx context.Context, cm *doltdb.Commit) (err error) {
p.stack = append(p.stack, cm)
return
}
func (p *progress) Pop(ctx context.Context) (cm *doltdb.Commit, err error) {
if len(p.stack) == 0 {
return nil, nil
}
top := len(p.stack) - 1
cm = p.stack[top]
p.stack = p.stack[:top]
return
}
func (p *progress) Log(ctx context.Context, format string, args ...any) {
cli.Println(time.Now().UTC().String() + " " + fmt.Sprintf(format, args...))
}
func (p *progress) Finalize(ctx context.Context) (prolly.Map, error) {
m, err := p.mapping.Map(ctx)
if err != nil {
return prolly.Map{}, err
}
v := shim.ValueFromMap(m)
ref, err := p.vs.WriteValue(ctx, v)
if err != nil {
return prolly.Map{}, err
}
last, err := p.vs.Root(ctx)
if err != nil {
return prolly.Map{}, err
}
ok, err := p.vs.Commit(ctx, last, last)
if err != nil {
return prolly.Map{}, err
} else if !ok {
return prolly.Map{}, fmt.Errorf("failed to commit, manifest swapped out beneath us")
}
p.Log(ctx, "Wrote commit mapping!! [commit_mapping_ref: %s]", ref.TargetHash().String())
p.Log(ctx, "Commit mapping allow mapping pre-migration commit hashes to post-migration commit hashes, "+
"it is available on branch '%s' in table '%s'", MigratedCommitsBranch, MigratedCommitsTable)
return m, nil
}
func persistMigratedCommitMapping(ctx context.Context, ddb *doltdb.DoltDB, mapping prolly.Map) error {
// create a new branch to persist the migrated commit mapping
init, err := ddb.ResolveCommitRef(ctx, ref.NewInternalRef(doltdb.CreationBranch))
if err != nil {
return err
}
br := ref.NewBranchRef(MigratedCommitsBranch)
err = ddb.NewBranchAtCommit(ctx, br, init, nil)
if err != nil {
return err
}
ns, vrw := ddb.NodeStore(), ddb.ValueReadWriter()
m, err := prolly.NewMapFromTuples(ctx, ns, desc, desc)
if err != nil {
return err
}
rows := m.Mutate()
bld := val.NewTupleBuilder(desc, ns)
// convert |mapping| values from hash.Hash to string
iter, err := mapping.IterAll(ctx)
if err != nil {
return err
}
var k, v val.Tuple
kd, vd := mapping.Descriptors()
for {
k, v, err = iter.Next(ctx)
if err == io.EOF {
break
} else if err != nil {
return err
}
o, _ := kd.GetBytes(0, k)
bld.PutString(0, hash.New(o).String())
key, err := bld.Build(ddb.NodeStore().Pool())
if err != nil {
return err
}
n, _ := vd.GetBytes(0, v)
bld.PutString(0, hash.New(n).String())
value, err := bld.Build(ddb.NodeStore().Pool())
if err != nil {
return err
}
if err = rows.Put(ctx, key, value); err != nil {
return err
}
}
m, err = rows.Map(ctx)
if err != nil {
return err
}
idx := durable.IndexFromProllyMap(m)
tbl, err := doltdb.NewTable(ctx, vrw, ns, mappingSchema, idx, nil, nil)
if err != nil {
return err
}
root, err := init.GetRootValue(ctx)
if err != nil {
return err
}
root, err = root.PutTable(ctx, doltdb.TableName{Name: MigratedCommitsTable}, tbl)
if err != nil {
return err
}
return commitRoot(ctx, ddb, br, root, init)
}
func commitRoot(
ctx context.Context,
ddb *doltdb.DoltDB,
br ref.BranchRef,
root doltdb.RootValue,
parent *doltdb.Commit,
) error {
roots := doltdb.Roots{
Head: root,
Working: root,
Staged: root,
}
parents := []*doltdb.Commit{parent}
meta, err := parent.GetCommitMeta(ctx)
if err != nil {
return err
}
meta, err = datas.NewCommitMeta(meta.Name, meta.Email, meta.Description)
if err != nil {
return err
}
pcm, err := ddb.NewPendingCommit(ctx, roots, parents, false, meta)
if err != nil {
return err
}
wsr, err := ref.WorkingSetRefForHead(br)
if err != nil {
return err
}
ws, err := ddb.ResolveWorkingSet(ctx, wsr)
if err != nil {
return err
}
prev, err := ws.HashOf()
if err != nil {
return err
}
ws = ws.WithWorkingRoot(root).WithStagedRoot(root)
_, err = ddb.CommitWithWorkingSet(ctx, br, wsr, pcm, ws, prev, &datas.WorkingSetMeta{
Name: meta.Name,
Email: meta.Email,
Timestamp: uint64(time.Now().Unix()),
}, nil)
return err
}
go/libraries/doltcore/migrate/transform.go
-725
View File
@@ -1,725 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package migrate
import (
"context"
"fmt"
"github.com/dolthub/go-mysql-server/sql"
gmstypes "github.com/dolthub/go-mysql-server/sql/types"
"github.com/dolthub/vitess/go/vt/proto/query"
"golang.org/x/sync/errgroup"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
"github.com/dolthub/dolt/go/libraries/utils/set"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
var (
flushRef = ref.NewInternalRef("migration-flush")
)
func migrateWorkingSet(ctx context.Context, menv Environment, brRef ref.BranchRef, wsRef ref.WorkingSetRef, old, new *doltdb.DoltDB) error {
oldHead, err := old.ResolveCommitRef(ctx, brRef)
if err != nil {
return err
}
oldHeadRoot, err := oldHead.GetRootValue(ctx)
if err != nil {
return err
}
oldWs, err := old.ResolveWorkingSet(ctx, wsRef)
if err == doltdb.ErrWorkingSetNotFound {
// If a branch was created prior to dolt version 0.26.10, no working set will exist for it.
// In this case, we will pretend it exists with the same root as the head commit.
oldWs = doltdb.EmptyWorkingSet(wsRef)
oldWs = oldWs.WithWorkingRoot(oldHeadRoot).WithStagedRoot(oldHeadRoot)
} else if err != nil {
return err
}
newHead, err := new.ResolveCommitRef(ctx, brRef)
if err != nil {
return err
}
newHeadRoot, err := newHead.GetRootValue(ctx)
if err != nil {
return err
}
wr, err := migrateRoot(ctx, menv, oldHeadRoot, oldWs.WorkingRoot(), newHeadRoot)
if err != nil {
return err
}
sr, err := migrateRoot(ctx, menv, oldHeadRoot, oldWs.StagedRoot(), newHeadRoot)
if err != nil {
return err
}
err = validateRootValue(ctx, oldHeadRoot, oldWs.WorkingRoot(), wr)
if err != nil {
return err
}
err = validateRootValue(ctx, oldHeadRoot, oldWs.StagedRoot(), sr)
if err != nil {
return err
}
newWs := doltdb.EmptyWorkingSet(wsRef).WithWorkingRoot(wr).WithStagedRoot(sr)
return new.UpdateWorkingSet(ctx, wsRef, newWs, hash.Hash{}, oldWs.Meta(), nil)
}
func migrateCommit(ctx context.Context, menv Environment, oldCm *doltdb.Commit, new *doltdb.DoltDB, prog *progress) error {
oldHash, err := oldCm.HashOf()
if err != nil {
return err
}
ok, err := prog.Has(ctx, oldHash)
if err != nil {
return err
} else if ok {
return nil
}
if oldCm.NumParents() == 0 {
return migrateInitCommit(ctx, oldCm, new, prog)
}
hs := oldHash.String()
prog.Log(ctx, "migrating commit %s", hs)
oldRoot, err := oldCm.GetRootValue(ctx)
if err != nil {
return err
}
optCmt, err := oldCm.GetParent(ctx, 0)
if err != nil {
return err
}
oldParentCm, ok := optCmt.ToCommit()
if !ok {
return doltdb.ErrGhostCommitEncountered
}
oldParentRoot, err := oldParentCm.GetRootValue(ctx)
if err != nil {
return err
}
oph, err := oldParentCm.HashOf()
if err != nil {
return err
}
ok, err = prog.Has(ctx, oph)
if err != nil {
return err
} else if !ok {
return fmt.Errorf("cannot find commit mapping for Commit (%s)", oph.String())
}
newParentAddr, err := prog.Get(ctx, oph)
if err != nil {
return err
}
optCmt, err = new.ReadCommit(ctx, newParentAddr)
if err != nil {
return err
}
newParentCm, ok := optCmt.ToCommit()
if !ok {
return doltdb.ErrGhostCommitEncountered
}
newParentRoot, err := newParentCm.GetRootValue(ctx)
if err != nil {
return err
}
mRoot, err := migrateRoot(ctx, menv, oldParentRoot, oldRoot, newParentRoot)
if err != nil {
return err
}
_, addr, err := new.WriteRootValue(ctx, mRoot)
if err != nil {
return err
}
value, err := new.ValueReadWriter().ReadValue(ctx, addr)
if err != nil {
return err
}
opts, err := migrateCommitOptions(ctx, oldCm, prog)
if err != nil {
return err
}
migratedCm, err := new.CommitDangling(ctx, value, opts)
if err != nil {
return err
}
// update progress
newHash, err := migratedCm.HashOf()
if err != nil {
return err
}
if err = prog.Put(ctx, oldHash, newHash); err != nil {
return err
}
// flush ChunkStore
if err = new.SetHead(ctx, flushRef, newHash); err != nil {
return err
}
err = new.ShallowGC(ctx)
if err != nil && err != chunks.ErrUnsupportedOperation {
return err
}
// validate root after we flush the ChunkStore to facilitate
// investigating failed migrations
if err = validateRootValue(ctx, oldParentRoot, oldRoot, mRoot); err != nil {
return err
}
return nil
}
func migrateInitCommit(ctx context.Context, cm *doltdb.Commit, new *doltdb.DoltDB, prog *progress) error {
oldHash, err := cm.HashOf()
if err != nil {
return err
}
rv, err := doltdb.EmptyRootValue(ctx, new.ValueReadWriter(), new.NodeStore())
if err != nil {
return err
}
meta, err := cm.GetCommitMeta(ctx)
if err != nil {
return err
}
datasDB := doltdb.HackDatasDatabaseFromDoltDB(new)
creation := ref.NewInternalRef(doltdb.CreationBranch)
ds, err := datasDB.GetDataset(ctx, creation.String())
if err != nil {
return err
}
ds, err = datasDB.Commit(ctx, ds, rv.NomsValue(), datas.CommitOptions{Meta: meta})
if err != nil {
return err
}
newCm, err := new.ResolveCommitRef(ctx, creation)
if err != nil {
return err
}
newHash, err := newCm.HashOf()
if err != nil {
return err
}
return prog.Put(ctx, oldHash, newHash)
}
func migrateCommitOptions(ctx context.Context, oldCm *doltdb.Commit, prog *progress) (datas.CommitOptions, error) {
parents, err := oldCm.ParentHashes(ctx)
if err != nil {
return datas.CommitOptions{}, err
}
if len(parents) == 0 {
panic("expected non-zero parents list")
}
for i := range parents {
migrated, err := prog.Get(ctx, parents[i])
if err != nil {
return datas.CommitOptions{}, err
}
parents[i] = migrated
}
meta, err := oldCm.GetCommitMeta(ctx)
if err != nil {
return datas.CommitOptions{}, err
}
return datas.CommitOptions{
Parents: parents,
Meta: meta,
}, nil
}
func migrateRoot(ctx context.Context, menv Environment, oldParent, oldRoot, newParent doltdb.RootValue) (doltdb.RootValue, error) {
migrated := newParent
fkc, err := oldRoot.GetForeignKeyCollection(ctx)
if err != nil {
return nil, err
}
migrated, err = migrated.PutForeignKeyCollection(ctx, fkc)
if err != nil {
return nil, err
}
removedTables, err := getRemovedTableNames(ctx, oldParent, oldRoot)
if err != nil {
return nil, err
}
migrated, err = migrated.RemoveTables(ctx, true, false, doltdb.ToTableNames(removedTables, doltdb.DefaultSchemaName)...)
if err != nil {
return nil, err
}
err = oldRoot.IterTables(ctx, func(name doltdb.TableName, oldTbl *doltdb.Table, sch schema.Schema) (bool, error) {
ok, err := oldTbl.HasConflicts(ctx)
if err != nil {
return true, err
} else if ok && !menv.DropConflicts {
return true, fmt.Errorf("cannot migrate table with conflicts (%s)", name)
}
// TODO: schema names
newSch, err := migrateSchema(ctx, name.Name, sch)
if err != nil {
return true, err
}
if err = validateSchema(newSch); err != nil {
return true, err
}
// if there was a schema change in this commit,
// diff against an empty table and rewrite everything
var parentSch schema.Schema
oldParentTbl, ok, err := oldParent.GetTable(ctx, name)
if err != nil {
return true, err
}
if ok {
parentSch, err = oldParentTbl.GetSchema(ctx)
if err != nil {
return true, err
}
}
if !ok || !schema.SchemasAreEqual(sch, parentSch) {
// provide empty table to diff against
oldParentTbl, err = doltdb.NewEmptyTable(ctx, oldParent.VRW(), oldParent.NodeStore(), sch)
if err != nil {
return true, err
}
}
newParentTbl, ok, err := newParent.GetTable(ctx, name)
if err != nil {
return true, err
}
if !ok || !schema.SchemasAreEqual(sch, parentSch) {
// provide empty table to diff against
newParentTbl, err = doltdb.NewEmptyTable(ctx, newParent.VRW(), newParent.NodeStore(), newSch)
if err != nil {
return true, err
}
}
mtbl, err := migrateTable(ctx, newSch, oldParentTbl, oldTbl, newParentTbl)
if err != nil {
return true, err
}
migrated, err = migrated.PutTable(ctx, name, mtbl)
if err != nil {
return true, err
}
return false, nil
})
if err != nil {
return nil, err
}
return migrated, nil
}
// renames also get returned here
func getRemovedTableNames(ctx context.Context, prev, curr doltdb.RootValue) ([]string, error) {
prevNames, err := prev.GetTableNames(ctx, doltdb.DefaultSchemaName, true)
if err != nil {
return nil, err
}
tblNameSet := set.NewStrSet(prevNames)
currNames, err := curr.GetTableNames(ctx, doltdb.DefaultSchemaName, true)
if err != nil {
return nil, err
}
tblNameSet.Remove(currNames...)
return tblNameSet.AsSlice(), nil
}
func migrateTable(ctx context.Context, newSch schema.Schema, oldParentTbl, oldTbl, newParentTbl *doltdb.Table) (*doltdb.Table, error) {
idx, err := oldParentTbl.GetRowData(ctx)
if err != nil {
return nil, err
}
oldParentRows := durable.NomsMapFromIndex(idx)
idx, err = oldTbl.GetRowData(ctx)
if err != nil {
return nil, err
}
oldRows := durable.NomsMapFromIndex(idx)
idx, err = newParentTbl.GetRowData(ctx)
if err != nil {
return nil, err
}
newParentRows, err := durable.ProllyMapFromIndex(idx)
if err != nil {
return nil, err
}
oldParentSet, err := oldParentTbl.GetIndexSet(ctx)
if err != nil {
return nil, err
}
oldSet, err := oldTbl.GetIndexSet(ctx)
if err != nil {
return nil, err
}
newParentSet, err := newParentTbl.GetIndexSet(ctx)
if err != nil {
return nil, err
}
var newRows durable.Index
var newSet durable.IndexSet
originalCtx := ctx
eg, ctx := errgroup.WithContext(ctx)
eg.Go(func() error {
var merr error
newRows, merr = migrateIndex(ctx, newSch, oldParentRows, oldRows, newParentRows, newParentTbl.NodeStore())
return merr
})
vrw, ns := newParentTbl.ValueReadWriter(), newParentTbl.NodeStore()
eg.Go(func() error {
var merr error
newSet, merr = migrateIndexSet(ctx, newSch, oldParentSet, oldSet, newParentSet, vrw, ns)
return merr
})
if err = eg.Wait(); err != nil {
return nil, err
}
ai, err := oldTbl.GetAutoIncrementValue(originalCtx)
if err != nil {
return nil, err
}
autoInc := types.Uint(ai)
return doltdb.NewTable(originalCtx, vrw, ns, newSch, newRows, newSet, autoInc)
}
func migrateSchema(ctx context.Context, tableName string, existing schema.Schema) (schema.Schema, error) {
// dolt_schemas and dolt_docs previously included columns with
// SQL type TEXT, but NomsKind of StringKind
if doltdb.HasDoltPrefix(tableName) {
var patched bool
cols := existing.GetAllCols().GetColumns()
for i, c := range cols {
qt := c.TypeInfo.ToSqlType().Type()
if qt == query.Type_TEXT && c.Kind == types.StringKind {
// NewColumn picks SQL type from NomsKind, converting this TEXT column to VARCHAR
cols[i] = schema.NewColumn(c.Name, c.Tag, c.Kind, c.IsPartOfPK, c.Constraints...)
patched = true
}
}
if patched {
allCols := schema.NewColCollection(cols...)
schema.NewIndexCollection(allCols, existing.GetPKCols())
return schema.NewSchema(
allCols,
existing.GetPkOrdinals(),
existing.GetCollation(),
existing.Indexes(),
existing.Checks(),
)
}
return existing, nil
}
// Blob types cannot be index keys in the new format:
// substitute VARCHAR(max) for TEXT, VARBINARY(max) for BLOB
// TODO: print warning to users
var patched bool
tags := schema.GetKeyColumnTags(existing)
cols := existing.GetAllCols().GetColumns()
for i, c := range cols {
if tags.Contains(c.Tag) {
var err error
switch c.TypeInfo.ToSqlType().Type() {
case query.Type_TEXT:
patched = true
info := typeinfo.StringDefaultType
cols[i], err = schema.NewColumnWithTypeInfo(c.Name, c.Tag, info, c.IsPartOfPK, c.Default, c.AutoIncrement, c.Comment, c.Constraints...)
case query.Type_BLOB:
patched = true
info := typeinfo.VarbinaryDefaultType
cols[i], err = schema.NewColumnWithTypeInfo(c.Name, c.Tag, info, c.IsPartOfPK, c.Default, c.AutoIncrement, c.Comment, c.Constraints...)
}
if err != nil {
return nil, err
}
}
}
// String types are sorted using a binary collation in __LD_1__
// force-set collation to utf8mb4_0900_bin to match the order
for i, c := range cols {
st, ok := c.TypeInfo.ToSqlType().(sql.StringType)
if !ok {
continue
}
patched = true
var err error
switch st.Type() {
case query.Type_CHAR, query.Type_VARCHAR, query.Type_TEXT:
st, err = gmstypes.CreateString(st.Type(), st.Length(), sql.Collation_utf8mb4_0900_bin)
case query.Type_BINARY, query.Type_VARBINARY, query.Type_BLOB:
st, err = gmstypes.CreateString(st.Type(), st.Length(), sql.Collation_binary)
}
if err != nil {
return nil, err
}
info, err := typeinfo.FromSqlType(st)
if err != nil {
return nil, err
}
cols[i], err = schema.NewColumnWithTypeInfo(c.Name, c.Tag, info, c.IsPartOfPK, c.Default, c.AutoIncrement, c.Comment, c.Constraints...)
if err != nil {
return nil, err
}
}
if !patched {
return existing, nil
}
sch, err := schema.NewSchema(
schema.NewColCollection(cols...),
existing.GetPkOrdinals(),
existing.GetCollation(),
existing.Indexes(),
existing.Checks(),
)
if err != nil {
return nil, err
}
return sch, nil
}
func migrateIndexSet(
ctx context.Context,
sch schema.Schema,
oldParentSet, oldSet, newParentSet durable.IndexSet,
vrw types.ValueReadWriter, ns tree.NodeStore,
) (durable.IndexSet, error) {
newSet, err := durable.NewIndexSet(ctx, vrw, ns)
if err != nil {
return nil, err
}
for _, def := range sch.Indexes().AllIndexes() {
idx, err := oldParentSet.GetIndex(ctx, sch, nil, def.Name())
if err != nil {
return nil, err
}
oldParent := durable.NomsMapFromIndex(idx)
idx, err = oldSet.GetIndex(ctx, sch, nil, def.Name())
if err != nil {
return nil, err
}
old := durable.NomsMapFromIndex(idx)
idx, err = newParentSet.GetIndex(ctx, sch, nil, def.Name())
if err != nil {
return nil, err
}
newParent, err := durable.ProllyMapFromIndex(idx)
if err != nil {
return nil, err
}
newIdx, err := migrateIndex(ctx, def.Schema(), oldParent, old, newParent, ns)
if err != nil {
return nil, err
}
newSet, err = newSet.PutIndex(ctx, def.Name(), newIdx)
if err != nil {
return nil, err
}
}
return newSet, nil
}
func migrateIndex(
ctx context.Context,
sch schema.Schema,
oldParent, oldMap types.Map,
newParent prolly.Map,
ns tree.NodeStore,
) (durable.Index, error) {
eg, ctx := errgroup.WithContext(ctx)
differ := make(chan types.ValueChanged, 256)
writer := make(chan val.Tuple, 256)
kt, vt := tupleTranslatorsFromSchema(sch, ns)
// read old noms map
eg.Go(func() error {
defer close(differ)
return oldMap.Diff(ctx, oldParent, differ)
})
// translate noms tuples to prolly tuples
eg.Go(func() error {
defer close(writer)
return translateTuples(ctx, kt, vt, differ, writer)
})
var newMap prolly.Map
// write tuples in new prolly map
eg.Go(func() (err error) {
newMap, err = writeProllyMap(ctx, newParent, writer)
return
})
if err := eg.Wait(); err != nil {
return nil, err
}
return durable.IndexFromProllyMap(newMap), nil
}
func translateTuples(ctx context.Context, kt, vt translator, differ <-chan types.ValueChanged, writer chan<- val.Tuple) error {
for {
var (
diff types.ValueChanged
newKey val.Tuple
newVal val.Tuple
ok bool
err error
)
select {
case diff, ok = <-differ:
if !ok {
return nil // done
}
case _ = <-ctx.Done():
return ctx.Err()
}
switch diff.ChangeType {
case types.DiffChangeAdded:
fallthrough
case types.DiffChangeModified:
newVal, err = vt.TranslateTuple(ctx, diff.NewValue.(types.Tuple))
if err != nil {
return err
}
fallthrough
case types.DiffChangeRemoved:
newKey, err = kt.TranslateTuple(ctx, diff.Key.(types.Tuple))
if err != nil {
return err
}
}
select {
case writer <- newKey:
case _ = <-ctx.Done():
return ctx.Err()
}
select {
case writer <- newVal:
case _ = <-ctx.Done():
return ctx.Err()
}
}
}
func writeProllyMap(ctx context.Context, prev prolly.Map, writer <-chan val.Tuple) (m prolly.Map, err error) {
var (
k, v val.Tuple
ok bool
)
mut := prev.Mutate()
for {
select {
case k, ok = <-writer:
if !ok {
m, err = mut.Map(ctx)
return // done
}
case <-ctx.Done():
return
}
select {
case v, ok = <-writer:
assertTrue(ok)
case <-ctx.Done():
return
}
if err = mut.Put(ctx, k, v); err != nil {
return
}
}
}
go/libraries/doltcore/migrate/traverse.go
-213
View File
@@ -1,213 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package migrate
import (
"context"
"fmt"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
)
// TraverseDAG traverses |old|, migrating values to |new|.
func TraverseDAG(ctx context.Context, menv Environment, old, new *doltdb.DoltDB) (err error) {
var heads []ref.DoltRef
var prog *progress
heads, err = old.GetHeadRefs(ctx)
if err != nil {
return err
}
datasdb := doltdb.HackDatasDatabaseFromDoltDB(new)
cs := datas.ChunkStoreFromDatabase(datasdb)
prog, err = newProgress(ctx, cs)
if err != nil {
return err
}
for i := range heads {
if err = traverseRefHistory(ctx, menv, heads[i], old, new, prog); err != nil {
return err
}
}
if err = validateBranchMapping(ctx, old, new); err != nil {
return err
}
// write the migrated commit mapping to a special branch
m, err := prog.Finalize(ctx)
if err != nil {
return err
}
if err = persistMigratedCommitMapping(ctx, new, m); err != nil {
return err
}
if err = old.Close(); err != nil {
return err
}
if err = new.Close(); err != nil {
return err
}
return
}
func traverseRefHistory(ctx context.Context, menv Environment, r ref.DoltRef, old, new *doltdb.DoltDB, prog *progress) error {
switch r.GetType() {
case ref.BranchRefType:
if err := traverseBranchHistory(ctx, menv, r, old, new, prog); err != nil {
return err
}
wsRef, err := ref.WorkingSetRefForHead(r)
if err != nil {
return err
}
return migrateWorkingSet(ctx, menv, r.(ref.BranchRef), wsRef, old, new)
case ref.TagRefType:
return traverseTagHistory(ctx, menv, r.(ref.TagRef), old, new, prog)
case ref.RemoteRefType:
return traverseBranchHistory(ctx, menv, r, old, new, prog)
case ref.WorkspaceRefType, ref.InternalRefType:
return nil
default:
panic(fmt.Sprintf("unknown ref type %s", r.String()))
}
}
func traverseBranchHistory(ctx context.Context, menv Environment, r ref.DoltRef, old, new *doltdb.DoltDB, prog *progress) error {
cm, err := old.ResolveCommitRef(ctx, r)
if err != nil {
return err
}
if err = traverseCommitHistory(ctx, menv, cm, new, prog); err != nil {
return err
}
oldHash, err := cm.HashOf()
if err != nil {
return err
}
newHash, err := prog.Get(ctx, oldHash)
if err != nil {
return err
}
return new.SetHead(ctx, r, newHash)
}
func traverseTagHistory(ctx context.Context, menv Environment, r ref.TagRef, old, new *doltdb.DoltDB, prog *progress) error {
t, err := old.ResolveTag(ctx, r)
if err != nil {
return err
}
if err = traverseCommitHistory(ctx, menv, t.Commit, new, prog); err != nil {
return err
}
oldHash, err := t.Commit.HashOf()
if err != nil {
return err
}
newHash, err := prog.Get(ctx, oldHash)
if err != nil {
return err
}
optCmt, err := new.ReadCommit(ctx, newHash)
if err != nil {
return err
}
cm, ok := optCmt.ToCommit()
if !ok {
return doltdb.ErrGhostCommitEncountered
}
return new.NewTagAtCommit(ctx, r, cm, t.Meta)
}
func traverseCommitHistory(ctx context.Context, menv Environment, cm *doltdb.Commit, new *doltdb.DoltDB, prog *progress) error {
ch, err := cm.HashOf()
if err != nil {
return err
}
ok, err := prog.Has(ctx, ch)
if err != nil || ok {
return err
}
for {
ph, err := cm.ParentHashes(ctx)
if err != nil {
return err
}
idx, err := firstAbsent(ctx, prog, ph)
if err != nil {
return err
}
if idx < 0 {
// parents for |cm| are done, migrate |cm|
if err = migrateCommit(ctx, menv, cm, new, prog); err != nil {
return err
}
// pop the stack, traverse upwards
cm, err = prog.Pop(ctx)
if err != nil {
return err
}
if cm == nil {
return nil // done
}
continue
}
// push the stack, traverse downwards
if err = prog.Push(ctx, cm); err != nil {
return err
}
optCmt, err := cm.GetParent(ctx, idx)
if err != nil {
return err
}
cm, ok = optCmt.ToCommit()
if !ok {
return doltdb.ErrGhostCommitEncountered
}
}
}
func firstAbsent(ctx context.Context, p *progress, addrs []hash.Hash) (int, error) {
for i := range addrs {
ok, err := p.Has(ctx, addrs[i])
if err != nil {
return -1, err
}
if !ok {
return i, nil
}
}
return -1, nil
}
go/libraries/doltcore/migrate/tuples.go
-402
View File
@@ -1,402 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package migrate
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/shopspring/decimal"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/json"
"github.com/dolthub/dolt/go/store/pool"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
const (
maxInlineValue = 16383
)
var ErrCannotMigrateText = errors.New("could not migrate TEXT value to VARCHAR, TEXT value exceeds 16383 size limit")
var ErrCannotMigrateBlob = errors.New("could not migrate BLOB value to VARBINARY, BLOB value exceeds 16383 size limit")
type translator struct {
builder *val.TupleBuilder
// maps columns tags to ordinal position
mapping map[uint64]int
ns tree.NodeStore
pool pool.BuffPool
}
func tupleTranslatorsFromSchema(sch schema.Schema, ns tree.NodeStore) (kt, vt translator) {
kd := sch.GetKeyDescriptor(ns)
vd := sch.GetValueDescriptor(ns)
keyMap := sch.GetPKCols().TagToIdx
valMap := sch.GetNonPKCols().TagToIdx
if !schema.IsKeyless(sch) {
kt = newTupleTranslator(ns, keyMap, kd)
vt = newTupleTranslator(ns, valMap, vd)
return
}
// for keyless tables, we must account for the id and cardinality columns
keyMap2 := map[uint64]int{schema.KeylessRowIdTag: 0}
valMap2 := map[uint64]int{schema.KeylessRowCardinalityTag: 0}
// shift positions for other columns
for tag, pos := range valMap {
valMap2[tag] = pos + 1
}
// assert previous keyMap was empty
assertTrue(len(keyMap) == 0)
kt = newTupleTranslator(ns, keyMap2, kd)
vt = newTupleTranslator(ns, valMap2, vd)
return
}
func newTupleTranslator(ns tree.NodeStore, mapping map[uint64]int, desc *val.TupleDesc) translator {
return translator{
builder: val.NewTupleBuilder(desc, ns),
mapping: mapping,
ns: ns,
pool: pool.NewBuffPool(),
}
}
// TranslateTuple translates a types.Tuple into a val.Tuple.
func (t translator) TranslateTuple(ctx context.Context, tup types.Tuple) (val.Tuple, error) {
if !isEven(tup.Len()) {
return nil, fmt.Errorf("expected even-legnth tuple (len %d)", tup.Len())
}
var tag uint64
err := tup.IterFields(func(i uint64, value types.Value) (stop bool, err error) {
// even fields are column tags, odd fields are column values
if isEven(i) {
tag = uint64(value.(types.Uint))
} else {
// |tag| set in previous iteration
pos, ok := t.mapping[tag]
if ok {
err = translateNomsField(ctx, t.ns, value, pos, t.builder)
stop = err != nil
} // else tombstone column
}
return
})
if err != nil {
return nil, err
}
defer func() {
if r := recover(); r != nil {
panic(tup.String())
}
}()
return t.builder.Build(t.pool)
}
func translateNomsField(ctx context.Context, ns tree.NodeStore, value types.Value, idx int, b *val.TupleBuilder) error {
nk := value.Kind()
switch nk {
case types.NullKind:
return nil // todo(andy): log warning?
case types.UintKind:
translateUintField(value.(types.Uint), idx, b)
case types.IntKind:
translateIntField(value.(types.Int), idx, b)
case types.FloatKind:
translateFloatField(value.(types.Float), idx, b)
case types.TimestampKind:
translateTimestampField(value.(types.Timestamp), idx, b)
case types.BoolKind:
b.PutBool(idx, bool(value.(types.Bool)))
case types.StringKind:
return translateStringField(ctx, ns, value.(types.String), idx, b)
case types.UUIDKind:
uuid := value.(types.UUID)
b.PutHash128(idx, uuid[:])
case types.InlineBlobKind:
b.PutByteString(idx, value.(types.InlineBlob))
case types.DecimalKind:
b.PutDecimal(idx, decimal.Decimal(value.(types.Decimal)))
case types.GeometryKind:
v := value.(types.Geometry).Inner
translateGeometryField(v, idx, b)
case types.PointKind,
types.LineStringKind,
types.PolygonKind,
types.MultiPointKind,
types.MultiLineStringKind,
types.MultiPolygonKind,
types.GeometryCollectionKind:
translateGeometryField(value, idx, b)
case types.JSONKind:
return translateJSONField(ctx, ns, value.(types.JSON), idx, b)
case types.BlobKind:
return translateBlobField(ctx, ns, value.(types.Blob), idx, b)
case types.ExtendedKind:
return fmt.Errorf("extended types are invalid during migration")
default:
return fmt.Errorf("encountered unexpected NomsKind %s",
types.KindToString[nk])
}
return nil
}
func translateUintField(value types.Uint, idx int, b *val.TupleBuilder) {
typ := b.Desc.Types[idx]
switch typ.Enc {
case val.Uint8Enc:
b.PutUint8(idx, uint8(value))
case val.Uint16Enc:
b.PutUint16(idx, uint16(value))
case val.Uint32Enc:
b.PutUint32(idx, uint32(value))
case val.Uint64Enc:
b.PutUint64(idx, uint64(value))
case val.EnumEnc:
b.PutEnum(idx, uint16(value))
case val.SetEnc:
b.PutSet(idx, uint64(value))
default:
panic(fmt.Sprintf("unexpected encoding for uint (%d)", typ.Enc))
}
}
func translateIntField(value types.Int, idx int, b *val.TupleBuilder) {
typ := b.Desc.Types[idx]
switch typ.Enc {
case val.Int8Enc:
b.PutInt8(idx, int8(value))
case val.Int16Enc:
b.PutInt16(idx, int16(value))
case val.Int32Enc:
b.PutInt32(idx, int32(value))
case val.Int64Enc:
b.PutInt64(idx, int64(value))
case val.YearEnc:
b.PutYear(idx, int16(value))
case val.TimeEnc:
b.PutSqlTime(idx, int64(value))
default:
panic(fmt.Sprintf("unexpected encoding for int (%d)", typ.Enc))
}
}
func translateFloatField(value types.Float, idx int, b *val.TupleBuilder) {
typ := b.Desc.Types[idx]
switch typ.Enc {
case val.Float32Enc:
b.PutFloat32(idx, float32(value))
case val.Float64Enc:
b.PutFloat64(idx, float64(value))
default:
panic(fmt.Sprintf("unexpected encoding for float (%d)", typ.Enc))
}
}
func translateStringField(ctx context.Context, ns tree.NodeStore, value types.String, idx int, b *val.TupleBuilder) error {
typ := b.Desc.Types[idx]
switch typ.Enc {
case val.StringEnc:
b.PutString(idx, string(value))
case val.StringAddrEnc:
// note: previously, TEXT fields were serialized as types.String
rd := strings.NewReader(string(value))
bb := ns.BlobBuilder()
defer ns.PutBlobBuilder(bb)
bb.Init(len(value))
_, addr, err := bb.Chunk(ctx, rd)
if err != nil {
return err
}
b.PutStringAddr(idx, addr)
default:
panic(fmt.Sprintf("unexpected encoding for string (%d)", typ.Enc))
}
return nil
}
func translateTimestampField(value types.Timestamp, idx int, b *val.TupleBuilder) {
typ := b.Desc.Types[idx]
switch typ.Enc {
case val.DateEnc:
b.PutDate(idx, time.Time(value))
case val.DatetimeEnc:
b.PutDatetime(idx, time.Time(value))
default:
panic(fmt.Sprintf("unexpected encoding for timestamp (%d)", typ.Enc))
}
}
func translateGeometryField(value types.Value, idx int, b *val.TupleBuilder) {
nk := value.Kind()
switch nk {
case types.PointKind:
p := types.ConvertTypesPointToSQLPoint(value.(types.Point))
b.PutGeometry(idx, p.Serialize())
case types.LineStringKind:
l := types.ConvertTypesLineStringToSQLLineString(value.(types.LineString))
b.PutGeometry(idx, l.Serialize())
case types.PolygonKind:
p := types.ConvertTypesPolygonToSQLPolygon(value.(types.Polygon))
b.PutGeometry(idx, p.Serialize())
case types.MultiPointKind:
p := types.ConvertTypesMultiPointToSQLMultiPoint(value.(types.MultiPoint))
b.PutGeometry(idx, p.Serialize())
case types.MultiLineStringKind:
l := types.ConvertTypesMultiLineStringToSQLMultiLineString(value.(types.MultiLineString))
b.PutGeometry(idx, l.Serialize())
case types.MultiPolygonKind:
p := types.ConvertTypesMultiPolygonToSQLMultiPolygon(value.(types.MultiPolygon))
b.PutGeometry(idx, p.Serialize())
case types.GeometryCollectionKind:
p := types.ConvertTypesGeomCollToSQLGeomColl(value.(types.GeomColl))
b.PutGeometry(idx, p.Serialize())
default:
panic(fmt.Sprintf("unexpected NomsKind for geometry (%d)", nk))
}
}
func translateJSONField(ctx context.Context, ns tree.NodeStore, value types.JSON, idx int, b *val.TupleBuilder) error {
s, err := json.NomsJSONToString(ctx, json.NomsJSON(value))
if err != nil {
return err
}
buf := bytes.NewBuffer([]byte(s))
bb := ns.BlobBuilder()
defer ns.PutBlobBuilder(bb)
bb.Init(len(s))
_, addr, err := bb.Chunk(ctx, buf)
if err != nil {
return err
}
b.PutJSONAddr(idx, addr)
return nil
}
func translateBlobField(ctx context.Context, ns tree.NodeStore, value types.Blob, idx int, b *val.TupleBuilder) error {
switch b.Desc.Types[idx].Enc {
// maybe convert from TEXT/BLOB to VARBINARY/VARCHAR
// if this column is a primary/secondary index key
case val.StringEnc, val.ByteStringEnc:
return translateBlobValueToInlineField(ctx, value, idx, b)
case val.StringAddrEnc, val.BytesAddrEnc:
// common case
default:
return fmt.Errorf("unexpected encoding for blob (%d)", b.Desc.Types[idx].Enc)
}
buf := make([]byte, value.Len())
_, err := value.ReadAt(ctx, buf, 0)
if err == io.EOF {
err = nil
} else if err != nil {
return err
}
bb := ns.BlobBuilder()
defer ns.PutBlobBuilder(bb)
bb.Init(int(value.Len()))
_, addr, err := bb.Chunk(ctx, bytes.NewReader(buf))
if err != nil {
return err
}
typ := b.Desc.Types[idx]
switch typ.Enc {
case val.BytesAddrEnc:
b.PutBytesAddr(idx, addr)
case val.StringAddrEnc:
b.PutStringAddr(idx, addr)
}
return nil
}
func translateBlobValueToInlineField(ctx context.Context, value types.Blob, idx int, b *val.TupleBuilder) error {
if value.Len() >= maxInlineValue {
if b.Desc.Types[idx].Enc == val.StringEnc {
return ErrCannotMigrateText
} else {
return ErrCannotMigrateBlob
}
}
buf := make([]byte, value.Len())
_, err := value.ReadAt(ctx, buf, 0)
if err == io.EOF {
err = nil
} else if err != nil {
return err
}
typ := b.Desc.Types[idx]
switch typ.Enc {
case val.ByteStringEnc:
b.PutByteString(idx, buf)
case val.StringEnc:
b.PutString(idx, string(buf))
default:
panic(fmt.Sprintf("unexpected encoding for blob (%d)", typ.Enc))
}
return nil
}
func isEven(n uint64) bool {
return n%2 == 0
}
go/libraries/doltcore/migrate/validation.go
-345
View File
@@ -1,345 +0,0 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package migrate
import (
"context"
"fmt"
"io"
"runtime"
"strings"
"time"
"unicode"
"github.com/dolthub/go-mysql-server/sql"
gmstypes "github.com/dolthub/go-mysql-server/sql/types"
"github.com/dolthub/vitess/go/vt/proto/query"
"golang.org/x/sync/errgroup"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle"
"github.com/dolthub/dolt/go/store/types"
)
func validateBranchMapping(ctx context.Context, old, new *doltdb.DoltDB) error {
branches, err := old.GetBranches(ctx)
if err != nil {
return err
}
var ok bool
for _, bref := range branches {
_, ok, err = new.HasBranch(ctx, bref.GetPath())
if err != nil {
return err
}
if !ok {
return fmt.Errorf("failed to map branch %s", bref.GetPath())
}
}
return nil
}
func validateRootValue(ctx context.Context, oldParent, old, new doltdb.RootValue) error {
names, err := old.GetTableNames(ctx, doltdb.DefaultSchemaName, true)
if err != nil {
return err
}
for _, name := range names {
o, ok, err := old.GetTable(ctx, doltdb.TableName{Name: name})
if err != nil {
return err
}
if !ok {
h, _ := old.HashOf()
return fmt.Errorf("expected to find table %s in root value (%s)", name, h.String())
}
// Skip tables that haven't changed
op, ok, err := oldParent.GetTable(ctx, doltdb.TableName{Name: name})
if err != nil {
return err
}
if ok {
oldHash, err := o.HashOf()
if err != nil {
return err
}
oldParentHash, err := op.HashOf()
if err != nil {
return err
}
if oldHash.Equal(oldParentHash) {
continue
}
}
n, ok, err := new.GetTable(ctx, doltdb.TableName{Name: name})
if err != nil {
return err
}
if !ok {
h, _ := new.HashOf()
return fmt.Errorf("expected to find table %s in root value (%s)", name, h.String())
}
if err = validateTableData(ctx, name, o, n); err != nil {
return err
}
}
return nil
}
func validateTableData(ctx context.Context, name string, old, new *doltdb.Table) error {
parts, err := partitionTable(ctx, old)
if err != nil {
return err
} else if len(parts) == 0 {
return nil
}
eg, ctx := errgroup.WithContext(ctx)
for i := range parts {
start, end := parts[i][0], parts[i][1]
eg.Go(func() error {
return validateTableDataPartition(ctx, name, old, new, start, end)
})
}
return eg.Wait()
}
func validateTableDataPartition(ctx context.Context, name string, old, new *doltdb.Table, start, end uint64) error {
sctx := sql.NewContext(ctx)
_, oldIter, err := sqle.DoltTablePartitionToRowIter(sctx, name, old, start, end)
if err != nil {
return err
}
newSch, newIter, err := sqle.DoltTablePartitionToRowIter(sctx, name, new, start, end)
if err != nil {
return err
}
var o, n sql.Row
for {
o, err = oldIter.Next(sctx)
if err == io.EOF {
break
} else if err != nil {
return err
}
n, err = newIter.Next(sctx)
if err != nil {
return err
}
ok, err := equalRows(ctx, o, n, newSch)
if err != nil {
return err
} else if !ok {
return fmt.Errorf("differing rows for table %s (%s != %s)",
name, sql.FormatRow(o), sql.FormatRow(n))
}
}
// validated that newIter is also exhausted
_, err = newIter.Next(sctx)
if err != io.EOF {
return fmt.Errorf("differing number of rows for table %s", name)
}
return nil
}
func equalRows(ctx context.Context, old, new sql.Row, sch sql.Schema) (bool, error) {
if len(new) != len(old) || len(new) != len(sch) {
return false, nil
}
var cmp int
for i := range new {
// special case string comparisons
s, ok, err := sql.Unwrap[string](ctx, old[i])
if err != nil {
return false, err
}
if ok {
old[i] = strings.TrimRightFunc(s, unicode.IsSpace)
}
s, ok, err = sql.Unwrap[string](ctx, new[i])
if err != nil {
return false, err
}
if ok {
new[i] = strings.TrimRightFunc(s, unicode.IsSpace)
}
// special case time comparison to account
// for precision changes between formats
if _, ok := old[i].(time.Time); ok {
var o, n interface{}
if o, _, err = gmstypes.Int64.Convert(ctx, old[i]); err != nil {
return false, err
}
if n, _, err = gmstypes.Int64.Convert(ctx, new[i]); err != nil {
return false, err
}
if cmp, err = gmstypes.Int64.Compare(ctx, o, n); err != nil {
return false, err
}
} else {
if cmp, err = sch[i].Type.Compare(ctx, old[i], new[i]); err != nil {
return false, err
}
}
if cmp != 0 {
return false, nil
}
}
return true, nil
}
func validateSchema(existing schema.Schema) error {
for _, c := range existing.GetAllCols().GetColumns() {
qt := c.TypeInfo.ToSqlType().Type()
err := assertNomsKind(c.Kind, nomsKindsFromQueryTypes(qt)...)
if err != nil {
return err
}
}
return nil
}
func nomsKindsFromQueryTypes(qt query.Type) []types.NomsKind {
switch qt {
case query.Type_UINT8:
return []types.NomsKind{types.UintKind, types.BoolKind}
case query.Type_UINT16, query.Type_UINT24,
query.Type_UINT32, query.Type_UINT64:
return []types.NomsKind{types.UintKind}
case query.Type_INT8:
return []types.NomsKind{types.IntKind, types.BoolKind}
case query.Type_INT16, query.Type_INT24,
query.Type_INT32, query.Type_INT64:
return []types.NomsKind{types.IntKind}
case query.Type_YEAR, query.Type_TIME:
return []types.NomsKind{types.IntKind}
case query.Type_FLOAT32, query.Type_FLOAT64:
return []types.NomsKind{types.FloatKind}
case query.Type_TIMESTAMP, query.Type_DATE, query.Type_DATETIME:
return []types.NomsKind{types.TimestampKind}
case query.Type_DECIMAL:
return []types.NomsKind{types.DecimalKind}
case query.Type_TEXT, query.Type_BLOB:
return []types.NomsKind{
types.BlobKind,
types.StringKind,
}
case query.Type_VARCHAR, query.Type_CHAR:
return []types.NomsKind{types.StringKind}
case query.Type_VARBINARY, query.Type_BINARY:
return []types.NomsKind{types.InlineBlobKind}
case query.Type_BIT, query.Type_ENUM, query.Type_SET:
return []types.NomsKind{types.UintKind}
case query.Type_GEOMETRY:
return []types.NomsKind{
types.GeometryKind,
types.PointKind,
types.LineStringKind,
types.PolygonKind,
types.MultiPointKind,
types.MultiLineStringKind,
types.MultiPolygonKind,
types.GeometryCollectionKind,
}
case query.Type_JSON:
return []types.NomsKind{types.JSONKind}
default:
panic(fmt.Sprintf("unexpect query.Type %s", qt.String()))
}
}
func assertNomsKind(kind types.NomsKind, candidates ...types.NomsKind) error {
for _, c := range candidates {
if kind == c {
return nil
}
}
cs := make([]string, len(candidates))
for i, c := range candidates {
cs[i] = types.KindToString[c]
}
return fmt.Errorf("expected NomsKind to be one of (%s), got NomsKind (%s)",
strings.Join(cs, ", "), types.KindToString[kind])
}
func partitionTable(ctx context.Context, tbl *doltdb.Table) ([][2]uint64, error) {
idx, err := tbl.GetRowData(ctx)
if err != nil {
return nil, err
}
c, err := idx.Count()
if err != nil {
return nil, err
}
if c == 0 {
return nil, nil
}
n := runtime.NumCPU() * 2
szc, err := idx.Count()
if err != nil {
return nil, err
}
sz := int(szc) / n
parts := make([][2]uint64, n)
parts[0][0] = 0
parts[n-1][1], err = idx.Count()
if err != nil {
return nil, err
}
for i := 1; i < len(parts); i++ {
parts[i-1][1] = uint64(i * sz)
parts[i][0] = uint64(i * sz)
}
return parts, nil
}
func assertTrue(b bool) {
if !b {
panic("expected true")
}
}
go/libraries/doltcore/rowconv/joiner.go
-181
View File
@@ -1,181 +0,0 @@
// Copyright 2019 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rowconv
import (
"errors"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/types"
)
// ColNamingFunc defines a function signature which takes the name of a column, and returns the name that should be used
// for the column in the joined dataset.
type ColNamingFunc func(colName string) string
type stringUint64Tuple struct {
str string
u64 uint64
}
// NamedSchema is an object that associates a schema with a string
type NamedSchema struct {
// Name the name given to the schema
Name string
// Sch is the schema
Sch schema.Schema
}
// Joiner is an object that can be used to join multiple rows together into a single row (See Join), and also to reverse
// this operation by taking a joined row and getting back a map of rows (See Split).
type Joiner struct {
srcSchemas map[string]schema.Schema
tagMaps map[string]map[uint64]uint64
revTagMap map[uint64]stringUint64Tuple
joined schema.Schema
}
// NewJoiner creates a joiner from a slice of NamedSchemas and a map of ColNamingFuncs. A new schema for joined rows will
// be created, and the columns for joined schemas will be named according to the ColNamingFunc associated with each schema
// name.
func NewJoiner(namedSchemas []NamedSchema, namers map[string]ColNamingFunc) (*Joiner, error) {
tags := make(map[string][]uint64)
revTagMap := make(map[uint64]stringUint64Tuple)
tagMaps := make(map[string]map[uint64]uint64, len(namedSchemas))
srcSchemas := make(map[string]schema.Schema)
for _, namedSch := range namedSchemas {
tagMaps[namedSch.Name] = make(map[uint64]uint64)
srcSchemas[namedSch.Name] = namedSch.Sch
}
var cols []schema.Column
var destTag uint64
var err error
for _, namedSch := range namedSchemas {
sch := namedSch.Sch
name := namedSch.Name
allCols := sch.GetAllCols()
namer := namers[name]
allCols.IterInSortedOrder(func(srcTag uint64, col schema.Column) (stop bool) {
newColName := namer(col.Name)
var newCol schema.Column
newCol, err = schema.NewColumnWithTypeInfo(newColName, destTag, col.TypeInfo, false, col.Default, false, col.Comment)
if err != nil {
return true
}
cols = append(cols, newCol)
tagMaps[name][srcTag] = destTag
revTagMap[destTag] = stringUint64Tuple{str: name, u64: srcTag}
tags[name] = append(tags[name], destTag)
destTag++
return false
})
}
if err != nil {
return nil, err
}
colColl := schema.NewColCollection(cols...)
joined := schema.UnkeyedSchemaFromCols(colColl)
return &Joiner{srcSchemas, tagMaps, revTagMap, joined}, nil
}
// Join takes a map from schema name to row which has that schema, and returns a single joined row containing all the
// data
func (j *Joiner) Join(namedRows map[string]row.Row) (row.Row, error) {
var nbf *types.NomsBinFormat
colVals := make(row.TaggedValues)
for name, r := range namedRows {
if r == nil {
continue
}
if nbf == nil {
nbf = r.Format()
} else if nbf.VersionString() != r.Format().VersionString() {
return nil, errors.New("not all rows have the same format")
}
_, err := r.IterCols(func(tag uint64, val types.Value) (stop bool, err error) {
destTag := j.tagMaps[name][tag]
colVals[destTag] = val
return false, nil
})
if err != nil {
return nil, err
}
}
return row.New(nbf, j.joined, colVals)
}
// Split takes a row which has the created joined schema, and splits it into a map of rows where the key of the map is
// the name of the schema for the associated row.
func (j *Joiner) Split(r row.Row) (map[string]row.Row, error) {
colVals := make(map[string]row.TaggedValues, len(j.srcSchemas))
for name := range j.srcSchemas {
colVals[name] = make(row.TaggedValues)
}
_, err := r.IterCols(func(tag uint64, val types.Value) (stop bool, err error) {
schemaNameAndTag, ok := j.revTagMap[tag]
if !ok {
return false, nil
}
colVals[schemaNameAndTag.str][schemaNameAndTag.u64] = val
return false, nil
})
if err != nil {
return nil, err
}
rows := make(map[string]row.Row, len(colVals))
for name, sch := range j.srcSchemas {
var err error
currColVals := colVals[name]
if len(currColVals) == 0 {
continue
}
rows[name], err = row.New(r.Format(), sch, currColVals)
if err != nil {
return nil, err
}
}
return rows, nil
}
// GetSchema returns the schema which all joined rows will have, and any row passed into split should have.
func (j *Joiner) GetSchema() schema.Schema {
return j.joined
}
// SchemaForName retrieves the original schema which has the given name.
func (j *Joiner) SchemaForName(name string) schema.Schema {
return j.srcSchemas[name]
}
go/libraries/doltcore/rowconv/joiner_test.go
-187
View File
@@ -1,187 +0,0 @@
// Copyright 2019 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rowconv
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/types"
)
const (
firstTag uint64 = iota
lastTag
ageTag
cityTag
)
var peopleCols = schema.NewColCollection(
schema.NewColumn("last", lastTag, types.StringKind, true),
schema.NewColumn("first", firstTag, types.StringKind, true),
schema.NewColumn("age", ageTag, types.IntKind, false),
schema.NewColumn("city", cityTag, types.StringKind, false),
)
var peopleSch = schema.MustSchemaFromCols(peopleCols)
type toJoinAndExpectedResult struct {
toJoinVals map[string]row.TaggedValues
expected map[string]types.Value
}
func TestJoiner(t *testing.T) {
tests := []struct {
name string
namedSchemas []NamedSchema
namers map[string]ColNamingFunc
toJoin []toJoinAndExpectedResult
}{
{
name: "join diff versions of row",
namedSchemas: []NamedSchema{{"to", peopleSch}, {"from", peopleSch}},
namers: map[string]ColNamingFunc{"to": toNamer, "from": fromNamer},
toJoin: []toJoinAndExpectedResult{
{
toJoinVals: map[string]row.TaggedValues{
"from": {
lastTag: types.String("Richardson"),
firstTag: types.String("Richard"),
ageTag: types.Int(42),
cityTag: types.String("San Francisco"),
},
"to": {
lastTag: types.String("Richardson"),
firstTag: types.String("Richard"),
ageTag: types.Int(43),
cityTag: types.String("Los Angeles"),
},
},
expected: map[string]types.Value{
"from_last": types.String("Richardson"),
"from_first": types.String("Richard"),
"from_city": types.String("San Francisco"),
"from_age": types.Int(42),
"to_last": types.String("Richardson"),
"to_first": types.String("Richard"),
"to_city": types.String("Los Angeles"),
"to_age": types.Int(43),
},
},
{
toJoinVals: map[string]row.TaggedValues{
"from": {
lastTag: types.String("Richardson"),
firstTag: types.String("Richard"),
ageTag: types.Int(42),
cityTag: types.String("San Francisco"),
},
},
expected: map[string]types.Value{
"from_last": types.String("Richardson"),
"from_first": types.String("Richard"),
"from_city": types.String("San Francisco"),
"from_age": types.Int(42),
},
},
{
toJoinVals: map[string]row.TaggedValues{
"to": {
lastTag: types.String("Richardson"),
firstTag: types.String("Richard"),
ageTag: types.Int(43),
cityTag: types.String("Los Angeles"),
},
},
expected: map[string]types.Value{
"to_last": types.String("Richardson"),
"to_first": types.String("Richard"),
"to_city": types.String("Los Angeles"),
"to_age": types.Int(43),
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
j, err := NewJoiner(test.namedSchemas, test.namers)
require.NoError(t, err)
for _, tj := range test.toJoin {
rows := map[string]row.Row{}
for _, namedSch := range test.namedSchemas {
r, err := row.New(types.Format_Default, namedSch.Sch, tj.toJoinVals[namedSch.Name])
require.NoError(t, err)
rows[namedSch.Name] = r
}
joinedRow, err := j.Join(rows)
require.NoError(t, err)
joinedSch := j.GetSchema()
_, err = joinedRow.IterCols(func(tag uint64, val types.Value) (stop bool, err error) {
col, ok := joinedSch.GetAllCols().GetByTag(tag)
assert.True(t, ok)
expectedVal := tj.expected[col.Name]
assert.Equal(t, val, expectedVal)
return false, nil
})
require.NoError(t, err)
splitRows, err := j.Split(joinedRow)
require.NoError(t, err)
assert.Equal(t, len(tj.toJoinVals), len(splitRows))
for _, namedSch := range test.namedSchemas {
name := namedSch.Name
sch := namedSch.Sch
actual := splitRows[name]
expectedVals := tj.toJoinVals[name]
if actual == nil && expectedVals == nil {
continue
}
assert.False(t, actual == nil || expectedVals == nil)
expected, err := row.New(types.Format_Default, sch, expectedVals)
require.NoError(t, err)
assert.True(t, row.AreEqual(actual, expected, sch))
}
}
})
}
}
func fromNamer(name string) string {
return "from_" + name
}
func toNamer(name string) string {
return "to_" + name
}
go/libraries/doltcore/schema/schema.go
+2 -2
View File
@@ -265,7 +265,7 @@ func GetSharedCols(schema Schema, cmpNames []string, cmpKinds []types.NomsKind)
// ArePrimaryKeySetsDiffable checks if two schemas are diffable. Assumes the
// passed in schema are from the same table between commits. If __DOLT__, then
// it also checks if the underlying SQL types of the columns are equal.
func ArePrimaryKeySetsDiffable(format *types.NomsBinFormat, fromSch, toSch Schema) bool {
func ArePrimaryKeySetsDiffable(fromSch, toSch Schema) bool {
if fromSch == nil && toSch == nil {
return false
// Empty case
@@ -292,7 +292,7 @@ func ArePrimaryKeySetsDiffable(format *types.NomsBinFormat, fromSch, toSch Schem
if (c1.Tag != c2.Tag) || (c1.IsPartOfPK != c2.IsPartOfPK) {
return false
}
if types.IsFormat_DOLT(format) && !c1.TypeInfo.ToSqlType().Equals(c2.TypeInfo.ToSqlType()) {
if !c1.TypeInfo.ToSqlType().Equals(c2.TypeInfo.ToSqlType()) {
return false
}
}
go/libraries/doltcore/schema/schema_test.go
+2 -13
View File
@@ -339,7 +339,7 @@ func TestArePrimaryKeySetsDiffable(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
d := ArePrimaryKeySetsDiffable(types.Format_Default, test.From, test.To)
d := ArePrimaryKeySetsDiffable(test.From, test.To)
require.Equal(t, test.Diffable, d)
// If they are diffable then we should be able to map their schemas from one to another.
@@ -359,7 +359,6 @@ func TestArePrimaryKeySetsDiffableTypeChanges(t *testing.T) {
From Schema
To Schema
Diffable bool
Format *types.NomsBinFormat
}{
{
Name: "Int -> String (New Format)",
@@ -368,22 +367,12 @@ func TestArePrimaryKeySetsDiffableTypeChanges(t *testing.T) {
To: MustSchemaFromCols(NewColCollection(
NewColumn("pk", 0, types.StringKind, true))),
Diffable: false,
Format: types.Format_DOLT,
},
{
Name: "Int -> String (Old Format)",
From: MustSchemaFromCols(NewColCollection(
NewColumn("pk", 0, types.IntKind, true))),
To: MustSchemaFromCols(NewColCollection(
NewColumn("pk", 0, types.StringKind, true))),
Diffable: true,
Format: types.Format_LD_1,
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
d := ArePrimaryKeySetsDiffable(test.Format, test.From, test.To)
d := ArePrimaryKeySetsDiffable(test.From, test.To)
require.Equal(t, test.Diffable, d)
})
}
go/libraries/doltcore/sqle/commit_hooks_test.go
+2 -25
View File
@@ -31,7 +31,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/libraries/utils/test"
@@ -381,30 +380,8 @@ func createTestRowData(t *testing.T, vrw types.ValueReadWriter, ns tree.NodeStor
return idx
}
vals := []row.TaggedValues{
{idTag: types.UUID(id0), firstTag: types.String("bill"), lastTag: types.String("billerson"), ageTag: types.Uint(53)},
{idTag: types.UUID(id1), firstTag: types.String("eric"), lastTag: types.String("ericson"), isMarriedTag: types.Bool(true), ageTag: types.Uint(21)},
{idTag: types.UUID(id2), firstTag: types.String("john"), lastTag: types.String("johnson"), isMarriedTag: types.Bool(false), ageTag: types.Uint(53)},
{idTag: types.UUID(id3), firstTag: types.String("robert"), lastTag: types.String("robertson"), ageTag: types.Uint(36)},
}
var err error
rows := make([]row.Row, len(vals))
m, err := types.NewMap(context.Background(), vrw)
assert.NoError(t, err)
ed := m.Edit()
for i, val := range vals {
r, err := row.New(vrw.Format(), sch, val)
require.NoError(t, err)
rows[i] = r
ed = ed.Set(r.NomsMapKey(sch), r.NomsMapValue(sch))
}
m, err = ed.Map(context.Background())
assert.NoError(t, err)
return durable.IndexFromNomsMap(m, vrw, ns)
t.Fatal("unsupported format")
return nil
}
func createHooksTestTable(vrw types.ValueReadWriter, ns tree.NodeStore, tSchema schema.Schema, rowData durable.Index) (*doltdb.Table, error) {
go/libraries/doltcore/sqle/dsess/autoincrement_tracker.go
+28 -34
View File
@@ -35,7 +35,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess/mutexmap"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/globalstate"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
type LockMode int64
@@ -374,41 +373,36 @@ func (a *AutoIncrementTracker) deepSet(ctx *sql.Context, tableName string, table
}
func getMaxIndexValue(ctx *sql.Context, indexData durable.Index) (uint64, error) {
if types.IsFormat_DOLT(indexData.Format()) {
idx, err := durable.ProllyMapFromIndex(indexData)
if err != nil {
return 0, err
}
iter, err := idx.IterAllReverse(ctx)
if err != nil {
return 0, err
}
kd, _ := idx.Descriptors()
k, _, err := iter.Next(ctx)
if err == io.EOF {
return 0, nil
} else if err != nil {
return 0, err
}
// TODO: is the auto-inc column always the first column in the index?
field, err := tree.GetField(ctx, kd, 0, k, idx.NodeStore())
if err != nil {
return 0, err
}
maxVal, err := CoerceAutoIncrementValue(ctx, field)
if err != nil {
return 0, err
}
return maxVal, nil
idx, err := durable.ProllyMapFromIndex(indexData)
if err != nil {
return 0, err
}
// For an LD format table, this operation won't succeed
return math.MaxUint64, nil
iter, err := idx.IterAllReverse(ctx)
if err != nil {
return 0, err
}
kd, _ := idx.Descriptors()
k, _, err := iter.Next(ctx)
if err == io.EOF {
return 0, nil
} else if err != nil {
return 0, err
}
// TODO: is the auto-inc column always the first column in the index?
field, err := tree.GetField(ctx, kd, 0, k, idx.NodeStore())
if err != nil {
return 0, err
}
maxVal, err := CoerceAutoIncrementValue(ctx, field)
if err != nil {
return 0, err
}
return maxVal, nil
}
// AddNewTable initializes a new table with an auto increment column to the tracker, as necessary
go/libraries/doltcore/sqle/dtablefunctions/dolt_diff.go
+2 -5
View File
@@ -29,7 +29,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/merge"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/rowconv"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables"
@@ -57,7 +56,6 @@ type DiffTableFunction struct {
database sql.Database
overriddenSchema schema.Schema
ctx *sql.Context
joiner *rowconv.Joiner
fromDate *types.Timestamp
toDate *types.Timestamp
includeCols map[string]struct{}
@@ -233,7 +231,7 @@ func (dtf *DiffTableFunction) RowIter(ctx *sql.Context, _ sql.Row) (sql.RowIter,
dp := dtables.NewDiffPartition(dtf.tableDelta.ToTable, dtf.tableDelta.FromTable, toCommitStr, fromCommitStr, dtf.toDate, dtf.fromDate, toSchForPartition, fromSchForPartition, nil)
return dtables.NewDiffPartitionRowIter(dp, ddb, dtf.joiner), nil
return dtables.NewDiffPartitionRowIter(dp, ddb), nil
}
// findMatchingDelta returns the best matching table delta for the table name
@@ -678,11 +676,10 @@ func (dtf *DiffTableFunction) generateSchema(ctx *sql.Context, fromCommitVal, to
toSchForJoiner = dtf.overriddenSchema
}
diffTableSch, j, err := dtables.GetDiffTableSchemaAndJoiner(format, fromSchForJoiner, toSchForJoiner)
diffTableSch, err := dtables.GetDiffTableSchemaAndJoiner(format, fromSchForJoiner, toSchForJoiner)
if err != nil {
return err
}
dtf.joiner = j
// TODO: sql.Columns include a Source that indicates the table it came from, but we don't have a real table
// when the column comes from a table function, so we omit the table name when we create these columns.
go/libraries/doltcore/sqle/dtablefunctions/dolt_diff_summary.go
+3 -3
View File
@@ -352,7 +352,7 @@ func getSummaryForDelta(ctx *sql.Context, delta diff.TableDelta, sqledb dsess.Sq
return summ, nil
}
if !schema.ArePrimaryKeySetsDiffable(delta.Format(), delta.FromSch, delta.ToSch) {
if !schema.ArePrimaryKeySetsDiffable(delta.FromSch, delta.ToSch) {
if shouldErrorOnPKChange {
return nil, fmt.Errorf("failed to compute diff summary for table %s: %w", delta.CurName(), diff.ErrPrimaryKeySetChanged)
}
@@ -408,9 +408,9 @@ func (ds *DiffSummaryTableFunction) evaluateArguments() (interface{}, interface{
return fromCommitVal, toCommitVal, nil, tableName, nil
}
//------------------------------------
// ------------------------------------
// diffSummaryTableFunctionRowIter
//------------------------------------
// ------------------------------------
var _ sql.RowIter = &diffSummaryTableFunctionRowIter{}
go/libraries/doltcore/sqle/dtablefunctions/dolt_patch.go
+7 -7
View File
@@ -547,7 +547,7 @@ func canGetDataDiff(ctx *sql.Context, td diff.TableDelta) bool {
}
// not diffable
if !schema.ArePrimaryKeySetsDiffable(td.Format(), td.FromSch, td.ToSch) {
if !schema.ArePrimaryKeySetsDiffable(td.FromSch, td.ToSch) {
ctx.Session.Warn(&sql.Warning{
Level: "Warning",
Code: mysql.ERNotSupportedYet,
@@ -625,7 +625,7 @@ func getDataSqlPatchResults(ctx *sql.Context, diffQuerySch, targetSch sql.Schema
// fmt.Sprintf("select %s, %s from dolt_diff('%s', '%s', '%s')", columnsWithDiff, "diff_type", fromRef, toRef, tableName)
// on sql engine, which returns the schema and rowIter of the final data diff result.
func getDiffQuery(ctx *sql.Context, dbData env.DbData[*sql.Context], td diff.TableDelta, fromRefDetails, toRefDetails *refDetails) (sql.Schema, []sql.Expression, sql.RowIter, error) {
diffTableSchema, j, err := dtables.GetDiffTableSchemaAndJoiner(td.ToTable.Format(), td.FromSch, td.ToSch)
diffTableSchema, err := dtables.GetDiffTableSchemaAndJoiner(td.ToTable.Format(), td.FromSch, td.ToSch)
if err != nil {
return nil, nil, nil, err
}
@@ -638,7 +638,7 @@ func getDiffQuery(ctx *sql.Context, dbData env.DbData[*sql.Context], td diff.Tab
diffQuerySqlSch, projections := getDiffQuerySqlSchemaAndProjections(diffPKSch.Schema, columnsWithDiff)
dp := dtables.NewDiffPartition(td.ToTable, td.FromTable, toRefDetails.hashStr, fromRefDetails.hashStr, toRefDetails.commitTime, fromRefDetails.commitTime, td.ToSch, td.FromSch, nil)
ri := dtables.NewDiffPartitionRowIter(dp, dbData.Ddb, j)
ri := dtables.NewDiffPartitionRowIter(dp, dbData.Ddb)
return diffQuerySqlSch, projections, ri, nil
}
@@ -693,9 +693,9 @@ func getDiffQuerySqlSchemaAndProjections(diffTableSch sql.Schema, columns []stri
return cols, getFieldCols
}
//------------------------------------
// ------------------------------------
// patchTableFunctionRowIter
//------------------------------------
// ------------------------------------
var _ sql.RowIter = (*patchTableFunctionRowIter)(nil)
@@ -761,9 +761,9 @@ func (itr *patchTableFunctionRowIter) Close(_ *sql.Context) error {
return nil
}
//------------------------------------
// ------------------------------------
// patchStatementsRowIter
//------------------------------------
// ------------------------------------
var _ sql.RowIter = (*patchStatementsRowIter)(nil)
go/libraries/doltcore/sqle/dtablefunctions/dolt_preview_merge_conflicts.go
+2 -2
View File
@@ -574,9 +574,9 @@ func (pm *PreviewMergeConflictsTableFunction) evaluateArguments() (interface{},
return leftBranchVal, rightBranchVal, tableName, nil
}
//--------------------------------------------------
// --------------------------------------------------
// previewMergeConflictsTableFunctionRowIter
//--------------------------------------------------
// --------------------------------------------------
var _ sql.RowIter = &previewMergeConflictsTableFunctionRowIter{}
go/libraries/doltcore/sqle/dtables/column_diff_table.go
+2 -2
View File
@@ -519,7 +519,7 @@ func processTableColDelta(ctx *sql.Context, ddb *doltdb.DoltDB, delta diff.Table
// cells to compile a list of modified columns
func calculateColDelta(ctx *sql.Context, ddb *doltdb.DoltDB, delta *diff.TableDelta, colSchDiff *colSchemaDiff) ([]string, []string, error) {
// initialize row iterator
diffTableSchema, j, err := GetDiffTableSchemaAndJoiner(delta.ToTable.Format(), delta.FromSch, delta.ToSch)
diffTableSchema, err := GetDiffTableSchemaAndJoiner(delta.ToTable.Format(), delta.FromSch, delta.ToSch)
if err != nil {
return nil, nil, err
}
@@ -528,7 +528,7 @@ func calculateColDelta(ctx *sql.Context, ddb *doltdb.DoltDB, delta *diff.TableDe
now := time.Now() // accurate commit time returned elsewhere
// TODO: schema name?
dp := NewDiffPartition(delta.ToTable, delta.FromTable, delta.ToName.Name, delta.FromName.Name, (*dtypes.Timestamp)(&now), (*dtypes.Timestamp)(&now), delta.ToSch, delta.FromSch, nil)
ri := NewDiffPartitionRowIter(dp, ddb, j)
ri := NewDiffPartitionRowIter(dp, ddb)
var resultColNames []string
var resultDiffTypes []string
go/libraries/doltcore/sqle/dtables/commit_diff_table.go
+2 -5
View File
@@ -25,7 +25,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/rowconv"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
@@ -45,7 +44,6 @@ type CommitDiffTable struct {
requiredFilterErr error
ddb *doltdb.DoltDB
table *doltdb.Table
joiner *rowconv.Joiner
tableName doltdb.TableName
targetSchema schema.Schema
@@ -76,7 +74,7 @@ func NewCommitDiffTable(ctx *sql.Context, dbName string, tblName doltdb.TableNam
return nil, err
}
diffTableSchema, j, err := GetDiffTableSchemaAndJoiner(ddb.Format(), sch, sch)
diffTableSchema, err := GetDiffTableSchemaAndJoiner(ddb.Format(), sch, sch)
if err != nil {
return nil, err
}
@@ -94,7 +92,6 @@ func NewCommitDiffTable(ctx *sql.Context, dbName string, tblName doltdb.TableNam
workingRoot: wRoot,
stagedRoot: sRoot,
headRef: headRef,
joiner: j,
sqlSch: sqlSch,
targetSchema: sch,
}, nil
@@ -320,5 +317,5 @@ func (dt *CommitDiffTable) rootValForHash(ctx *sql.Context, hashStr string) (dol
func (dt *CommitDiffTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
dp := part.(DiffPartition)
return dp.GetRowIter(ctx, dt.ddb, dt.joiner)
return dp.GetRowIter(ctx)
}
go/libraries/doltcore/sqle/dtables/conflicts_tables.go
+5 -177
View File
@@ -15,15 +15,11 @@ package dtables
// limitations under the License.
import (
"errors"
"fmt"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/merge"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/store/types"
)
@@ -36,178 +32,10 @@ func NewConflictsTable(ctx *sql.Context, tblName doltdb.TableName, srcTable sql.
return nil, err
}
if types.IsFormat_DOLT(tbl.Format()) {
upd, ok := srcTable.(sql.UpdatableTable)
if !ok {
return nil, fmt.Errorf("%s can not have conflicts because it is not updateable", tblName)
}
return newProllyConflictsTable(ctx, tbl, upd, tblName, root, rs)
types.AssertFormat_DOLT(tbl.Format())
upd, ok := srcTable.(sql.UpdatableTable)
if !ok {
return nil, fmt.Errorf("%s can not have conflicts because it is not updateable", tblName)
}
return newNomsConflictsTable(ctx, tbl, tblName, root, rs)
}
func newNomsConflictsTable(ctx *sql.Context, tbl *doltdb.Table, tblName doltdb.TableName, root doltdb.RootValue, rs RootSetter) (sql.Table, error) {
rd, err := merge.NewConflictReader(ctx, tbl, tblName)
if err != nil {
return nil, err
}
confSch := rd.GetSchema()
sqlSch, err := sqlutil.FromDoltSchema("", doltdb.DoltConfTablePrefix+tblName.Name, confSch)
if err != nil {
return nil, err
}
return ConflictsTable{
tblName: tblName,
sqlSch: sqlSch,
root: root,
tbl: tbl,
rd: rd,
rs: rs,
}, nil
}
var _ sql.Table = ConflictsTable{}
var _ sql.DeletableTable = ConflictsTable{}
// ConflictsTable is a sql.Table implementation that provides access to the conflicts that exist for a user table
type ConflictsTable struct {
root doltdb.RootValue
rs RootSetter
tbl *doltdb.Table
rd *merge.ConflictReader
tblName doltdb.TableName
sqlSch sql.PrimaryKeySchema
}
type RootSetter interface {
SetRoot(ctx *sql.Context, root doltdb.RootValue) error
}
// Name returns the name of the table
func (ct ConflictsTable) Name() string {
return doltdb.DoltConfTablePrefix + ct.tblName.Name
}
// String returns a string identifying the table
func (ct ConflictsTable) String() string {
return doltdb.DoltConfTablePrefix + ct.tblName.Name
}
// Schema returns the sql.Schema of the table
func (ct ConflictsTable) Schema() sql.Schema {
return ct.sqlSch.Schema
}
// Collation implements the sql.Table interface.
func (ct ConflictsTable) Collation() sql.CollationID {
return sql.Collation_Default
}
// Partitions returns a PartitionIter which can be used to get all the data partitions
func (ct ConflictsTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
return index.SinglePartitionIterFromNomsMap(nil), nil
}
// PartitionRows returns a RowIter for the given partition
func (ct ConflictsTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
// conflict reader must be reset each time partitionRows is called.
rd, err := merge.NewConflictReader(ctx, ct.tbl, ct.tblName)
if err != nil {
return nil, err
}
return conflictRowIter{rd}, nil
}
// Deleter returns a RowDeleter for this table. The RowDeleter will get one call to Delete for each row to be deleted,
// and will end with a call to Close() to finalize the delete operation.
func (ct ConflictsTable) Deleter(ctx *sql.Context) sql.RowDeleter {
return &conflictDeleter{ct: ct, rs: ct.rs}
}
type conflictRowIter struct {
rd *merge.ConflictReader
}
// Next retrieves the next row. It will return io.EOF if it's the last row.
// After retrieving the last row, Close will be automatically closed.
func (itr conflictRowIter) Next(ctx *sql.Context) (sql.Row, error) {
cnf, err := itr.rd.NextConflict(ctx)
if err != nil {
return nil, err
}
return sqlutil.DoltRowToSqlRow(cnf, itr.rd.GetSchema())
}
// Close the iterator.
func (itr conflictRowIter) Close(*sql.Context) error {
return itr.rd.Close()
}
type conflictDeleter struct {
ct ConflictsTable
rs RootSetter
pks []types.Value
}
var _ sql.RowDeleter = &conflictDeleter{}
// Delete deletes the given row. Returns ErrDeleteRowNotFound if the row was not found. Delete will be called once for
// each row to process for the delete operation, which may involve many rows. After all rows have been processed,
// Close is called.
func (cd *conflictDeleter) Delete(ctx *sql.Context, r sql.Row) error {
cnfSch := cd.ct.rd.GetSchema()
// We could use a test VRW, but as any values which use VRWs will already exist, we can potentially save on memory usage
cnfRow, err := sqlutil.SqlRowToDoltRow(ctx, cd.ct.tbl.ValueReadWriter(), r, cnfSch)
if err != nil {
return err
}
pkVal, err := cd.ct.rd.GetKeyForConflict(ctx, cnfRow)
if err != nil {
return err
}
cd.pks = append(cd.pks, pkVal)
return nil
}
// StatementBegin implements the interface sql.TableEditor. Currently a no-op.
func (cd *conflictDeleter) StatementBegin(ctx *sql.Context) {}
// DiscardChanges implements the interface sql.TableEditor. Currently a no-op.
func (cd *conflictDeleter) DiscardChanges(ctx *sql.Context, errorEncountered error) error {
return nil
}
// StatementComplete implements the interface sql.TableEditor. Currently a no-op.
func (cd *conflictDeleter) StatementComplete(ctx *sql.Context) error {
return nil
}
// Close finalizes the delete operation, persisting the result.
func (cd *conflictDeleter) Close(ctx *sql.Context) error {
_, _, updatedTbl, err := cd.ct.tbl.ResolveConflicts(ctx, cd.pks)
if err != nil {
if errors.Is(err, doltdb.ErrNoConflictsResolved) {
return nil
}
return err
}
updatedRoot, err := cd.ct.root.PutTable(ctx, cd.ct.tblName, updatedTbl)
if err != nil {
return err
}
return cd.rs.SetRoot(ctx, updatedRoot)
return newProllyConflictsTable(ctx, tbl, upd, tblName, root, rs)
}
go/libraries/doltcore/sqle/dtables/constraint_violations.go
+1 -171
View File
@@ -17,12 +17,7 @@ package dtables
import (
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/store/types"
)
@@ -32,170 +27,5 @@ func NewConstraintViolationsTable(ctx *sql.Context, tblName doltdb.TableName, ro
return newProllyCVTable(ctx, tblName, root, rs)
}
return newNomsCVTable(ctx, tblName, root, rs)
}
func newNomsCVTable(ctx *sql.Context, tblName doltdb.TableName, root doltdb.RootValue, rs RootSetter) (sql.Table, error) {
var tbl *doltdb.Table
var err error
tbl, tblName, err = getTableInsensitiveOrError(ctx, root, tblName)
if err != nil {
return nil, err
}
cvSch, err := getConstraintViolationsSchema(ctx, tbl, tblName, root)
if err != nil {
return nil, err
}
sqlSch, err := sqlutil.FromDoltSchema("", doltdb.DoltConstViolTablePrefix+tblName.Name, cvSch)
if err != nil {
return nil, err
}
return &constraintViolationsTable{
tblName: tblName,
root: root,
cvSch: cvSch,
sqlSch: sqlSch,
tbl: tbl,
rs: rs,
}, nil
}
// constraintViolationsTable is a sql.Table implementation that provides access to the constraint violations that exist
// for a user table for the old format.
type constraintViolationsTable struct {
rs RootSetter
root doltdb.RootValue
cvSch schema.Schema
tbl *doltdb.Table
tblName doltdb.TableName
sqlSch sql.PrimaryKeySchema
}
var _ sql.Table = (*constraintViolationsTable)(nil)
var _ sql.DeletableTable = (*constraintViolationsTable)(nil)
// Name implements the interface sql.Table.
func (cvt *constraintViolationsTable) Name() string {
return doltdb.DoltConstViolTablePrefix + cvt.tblName.Name
}
// String implements the interface sql.Table.
func (cvt *constraintViolationsTable) String() string {
return doltdb.DoltConstViolTablePrefix + cvt.tblName.Name
}
// Schema implements the interface sql.Table.
func (cvt *constraintViolationsTable) Schema() sql.Schema {
return cvt.sqlSch.Schema
}
// Collation implements the interface sql.Table.
func (cvt *constraintViolationsTable) Collation() sql.CollationID {
return sql.Collation_Default
}
// Partitions implements the interface sql.Table.
func (cvt *constraintViolationsTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
return index.SinglePartitionIterFromNomsMap(nil), nil
}
// PartitionRows implements the interface sql.Table.
func (cvt *constraintViolationsTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
cvMap, err := cvt.tbl.GetConstraintViolations(ctx)
if err != nil {
return nil, err
}
iter, err := cvMap.Iterator(ctx)
if err != nil {
return nil, err
}
return &constraintViolationsIter{cvt.cvSch, iter}, nil
}
// Deleter implements the interface sql.DeletableTable.
func (cvt *constraintViolationsTable) Deleter(ctx *sql.Context) sql.RowDeleter {
cvMap, err := cvt.tbl.GetConstraintViolations(ctx)
if err != nil {
panic(err)
}
return &constraintViolationsDeleter{cvt, cvMap.Edit()}
}
// constraintViolationsIter is the iterator for constraintViolationsTable.
type constraintViolationsIter struct {
dSch schema.Schema
iter types.MapIterator
}
var _ sql.RowIter = (*constraintViolationsIter)(nil)
// Next implements the interface sql.RowIter.
func (cvi *constraintViolationsIter) Next(ctx *sql.Context) (sql.Row, error) {
k, v, err := cvi.iter.NextTuple(ctx)
if err != nil {
return nil, err
}
dRow, err := row.FromNoms(cvi.dSch, k, v)
if err != nil {
return nil, err
}
return sqlutil.DoltRowToSqlRow(dRow, cvi.dSch)
}
// Close implements the interface sql.RowIter.
func (cvi *constraintViolationsIter) Close(*sql.Context) error {
return nil
}
// constraintViolationsDeleter handles deletions on the generated constraintViolationsTable.
type constraintViolationsDeleter struct {
cvt *constraintViolationsTable
editor *types.MapEditor
}
var _ sql.RowDeleter = (*constraintViolationsDeleter)(nil)
// Delete implements the interface sql.RowDeleter.
func (cvd *constraintViolationsDeleter) Delete(ctx *sql.Context, r sql.Row) error {
dRow, err := sqlutil.SqlRowToDoltRow(ctx, cvd.cvt.tbl.ValueReadWriter(), r, cvd.cvt.cvSch)
if err != nil {
return err
}
key, err := dRow.NomsMapKey(cvd.cvt.cvSch).Value(ctx)
if err != nil {
return err
}
cvd.editor.Remove(key)
return nil
}
// StatementBegin implements the interface sql.TableEditor. Currently a no-op.
func (cvd *constraintViolationsDeleter) StatementBegin(ctx *sql.Context) {}
// DiscardChanges implements the interface sql.TableEditor. Currently a no-op.
func (cvd *constraintViolationsDeleter) DiscardChanges(ctx *sql.Context, errorEncountered error) error {
return nil
}
// StatementComplete implements the interface sql.TableEditor. Currently a no-op.
func (cvd *constraintViolationsDeleter) StatementComplete(ctx *sql.Context) error {
return nil
}
// Close implements the interface sql.RowDeleter.
func (cvd *constraintViolationsDeleter) Close(ctx *sql.Context) error {
updatedMap, err := cvd.editor.Map(ctx)
if err != nil {
return err
}
updatedTbl, err := cvd.cvt.tbl.SetConstraintViolations(ctx, updatedMap)
if err != nil {
return err
}
updatedRoot, err := cvd.cvt.root.PutTable(ctx, cvd.cvt.tblName, updatedTbl)
if err != nil {
return err
}
return cvd.cvt.rs.SetRoot(ctx, updatedRoot)
panic("Unsupported format: " + root.VRW().Format().VersionString())
}
go/libraries/doltcore/sqle/dtables/diff_iter.go
+4 -164
View File
@@ -21,32 +21,15 @@ import (
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/libraries/doltcore/diff"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/rowconv"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
// ldDiffRowItr is a sql.RowIter implementation which iterates over an LD formated DB in order to generate the
// dolt_diff_{table} results. This is legacy code at this point, as the DOLT format is what we'll support going forward.
type ldDiffRowItr struct {
ad diff.RowDiffer
diffSrc *diff.RowDiffSource
joiner *rowconv.Joiner
sch schema.Schema
fromCommitInfo commitInfo
toCommitInfo commitInfo
}
var _ sql.RowIter = &ldDiffRowItr{}
type commitInfo struct {
date *types.Timestamp
name types.String
@@ -54,147 +37,6 @@ type commitInfo struct {
dateTag uint64
}
func newLdDiffIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joiner, dp DiffPartition, lookup sql.IndexLookup) (*ldDiffRowItr, error) {
fromData, fromSch, err := tableData(ctx, dp.from, ddb)
if err != nil {
return nil, err
}
toData, toSch, err := tableData(ctx, dp.to, ddb)
if err != nil {
return nil, err
}
fromConv, err := dp.rowConvForSchema(ctx, ddb.ValueReadWriter(), dp.fromSch, fromSch)
if err != nil {
return nil, err
}
toConv, err := dp.rowConvForSchema(ctx, ddb.ValueReadWriter(), dp.toSch, toSch)
if err != nil {
return nil, err
}
sch := joiner.GetSchema()
toCol, _ := sch.GetAllCols().GetByName(toCommit)
fromCol, _ := sch.GetAllCols().GetByName(fromCommit)
toDateCol, _ := sch.GetAllCols().GetByName(toCommitDate)
fromDateCol, _ := sch.GetAllCols().GetByName(fromCommitDate)
fromCmInfo := commitInfo{name: types.String(dp.fromName), date: dp.fromDate, nameTag: fromCol.Tag, dateTag: fromDateCol.Tag}
toCmInfo := commitInfo{name: types.String(dp.toName), date: dp.toDate, nameTag: toCol.Tag, dateTag: toDateCol.Tag}
rd := diff.NewRowDiffer(ctx, ddb.Format(), fromSch, toSch, 1024)
// TODO (dhruv) don't cast to noms map
// Use index lookup if it exists
if lookup.IsEmpty() {
rd.Start(ctx, durable.NomsMapFromIndex(fromData), durable.NomsMapFromIndex(toData))
} else {
ranges, err := index.NomsRangesFromIndexLookup(ctx, lookup) // TODO: this is a testing method
if err != nil {
return nil, err
}
// TODO: maybe just use Check
rangeFunc := func(ctx context.Context, val types.Value) (bool, bool, error) {
v, ok := val.(types.Tuple)
if !ok {
return false, false, nil
}
return ranges[0].Check.Check(ctx, ddb.ValueReadWriter(), v)
}
rd.StartWithRange(ctx, durable.NomsMapFromIndex(fromData), durable.NomsMapFromIndex(toData), ranges[0].Start, rangeFunc)
}
src := diff.NewRowDiffSource(rd, joiner, ctx.Warn)
src.AddInputRowConversion(fromConv, toConv)
return &ldDiffRowItr{
ad: rd,
diffSrc: src,
joiner: joiner,
sch: joiner.GetSchema(),
fromCommitInfo: fromCmInfo,
toCommitInfo: toCmInfo,
}, nil
}
// Next returns the next row
func (itr *ldDiffRowItr) Next(ctx *sql.Context) (sql.Row, error) {
r, err := itr.diffSrc.NextDiff()
if err != nil {
return nil, err
}
toAndFromRows, err := itr.joiner.Split(r)
if err != nil {
return nil, err
}
_, hasTo := toAndFromRows[diff.To]
_, hasFrom := toAndFromRows[diff.From]
r, err = r.SetColVal(itr.toCommitInfo.nameTag, types.String(itr.toCommitInfo.name), itr.sch)
if err != nil {
return nil, err
}
r, err = r.SetColVal(itr.fromCommitInfo.nameTag, types.String(itr.fromCommitInfo.name), itr.sch)
if err != nil {
return nil, err
}
if itr.toCommitInfo.date != nil {
r, err = r.SetColVal(itr.toCommitInfo.dateTag, *itr.toCommitInfo.date, itr.sch)
if err != nil {
return nil, err
}
}
if itr.fromCommitInfo.date != nil {
r, err = r.SetColVal(itr.fromCommitInfo.dateTag, *itr.fromCommitInfo.date, itr.sch)
if err != nil {
return nil, err
}
}
sqlRow, err := sqlutil.DoltRowToSqlRow(r, itr.sch)
if err != nil {
return nil, err
}
if hasTo && hasFrom {
sqlRow = append(sqlRow, diffTypeModified)
} else if hasTo && !hasFrom {
sqlRow = append(sqlRow, diffTypeAdded)
} else {
sqlRow = append(sqlRow, diffTypeRemoved)
}
return sqlRow, nil
}
// Close closes the iterator
func (itr *ldDiffRowItr) Close(*sql.Context) (err error) {
defer itr.ad.Close()
defer func() {
closeErr := itr.diffSrc.Close()
if err == nil {
err = closeErr
}
}()
return nil
}
type commitInfo2 struct {
ts *time.Time
name string
@@ -543,24 +385,22 @@ func maybeTime(t *time.Time) interface{} {
return nil
}
//------------------------------------
// ------------------------------------
// diffPartitionRowIter
//------------------------------------
// ------------------------------------
var _ sql.RowIter = (*diffPartitionRowIter)(nil)
type diffPartitionRowIter struct {
ddb *doltdb.DoltDB
joiner *rowconv.Joiner
currentPartition *DiffPartition
currentRowIter *sql.RowIter
}
func NewDiffPartitionRowIter(partition *DiffPartition, ddb *doltdb.DoltDB, joiner *rowconv.Joiner) *diffPartitionRowIter {
func NewDiffPartitionRowIter(partition *DiffPartition, ddb *doltdb.DoltDB) *diffPartitionRowIter {
return &diffPartitionRowIter{
currentPartition: partition,
ddb: ddb,
joiner: joiner,
}
}
@@ -570,7 +410,7 @@ func (itr *diffPartitionRowIter) Next(ctx *sql.Context) (sql.Row, error) {
return nil, io.EOF
}
if itr.currentRowIter == nil {
rowIter, err := itr.currentPartition.GetRowIter(ctx, itr.ddb, itr.joiner)
rowIter, err := itr.currentPartition.GetRowIter(ctx)
if err != nil {
return nil, err
}
go/libraries/doltcore/sqle/dtables/diff_table.go
+7 -34
View File
@@ -67,7 +67,6 @@ type DiffTable struct {
head *doltdb.Commit
headCommitClosure *prolly.CommitClosure
table *doltdb.Table
joiner *rowconv.Joiner
tableName doltdb.TableName
sqlSch sql.PrimaryKeySchema
partitionFilters []sql.Expression
@@ -105,7 +104,7 @@ func NewDiffTable(ctx *sql.Context, dbName string, tblName doltdb.TableName, ddb
return nil, err
}
diffTableSchema, j, err := GetDiffTableSchemaAndJoiner(ddb.Format(), sch, sch)
diffTableSchema, err := GetDiffTableSchemaAndJoiner(ddb.Format(), sch, sch)
if err != nil {
return nil, err
}
@@ -125,7 +124,6 @@ func NewDiffTable(ctx *sql.Context, dbName string, tblName doltdb.TableName, ddb
sqlSch: sqlSch,
partitionFilters: nil,
table: table,
joiner: j,
}, nil
}
@@ -250,7 +248,7 @@ func (dt *DiffTable) HeadHash() (hash.Hash, error) {
func (dt *DiffTable) PartitionRows(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
dp := part.(DiffPartition)
return dp.GetRowIter(ctx, dt.ddb, dt.joiner)
return dp.GetRowIter(ctx)
}
func (dt *DiffTable) LookupPartitions(ctx *sql.Context, lookup sql.IndexLookup) (sql.PartitionIter, error) {
@@ -677,7 +675,7 @@ func (dp DiffPartition) Key() []byte {
return []byte(dp.toName + dp.fromName)
}
func (dp DiffPartition) GetRowIter(ctx *sql.Context, ddb *doltdb.DoltDB, joiner *rowconv.Joiner) (sql.RowIter, error) {
func (dp DiffPartition) GetRowIter(ctx *sql.Context) (sql.RowIter, error) {
return newProllyDiffIter(ctx, dp, dp.fromSch, dp.toSch, dp.ranges)
}
@@ -709,7 +707,7 @@ func (dp *DiffPartition) isDiffablePartition(ctx *sql.Context) (simpleDiff bool,
return false, false, err
}
easyDiff := schema.ArePrimaryKeySetsDiffable(dp.from.Format(), fromSch, toSch)
easyDiff := schema.ArePrimaryKeySetsDiffable(fromSch, toSch)
if easyDiff {
return true, false, nil
}
@@ -910,37 +908,12 @@ func (dp DiffPartition) rowConvForSchema(ctx context.Context, vrw types.ValueRea
// GetDiffTableSchemaAndJoiner returns the schema for the diff table given a
// target schema for a row |sch|. In the old storage format, it also returns the
// associated joiner.
func GetDiffTableSchemaAndJoiner(format *types.NomsBinFormat, fromSch, toSch schema.Schema) (diffTableSchema schema.Schema, j *rowconv.Joiner, err error) {
func GetDiffTableSchemaAndJoiner(format *types.NomsBinFormat, fromSch, toSch schema.Schema) (diffTableSchema schema.Schema, err error) {
if format == types.Format_DOLT {
diffTableSchema, err = CalculateDiffSchema(fromSch, toSch)
if err != nil {
return nil, nil, err
}
return CalculateDiffSchema(fromSch, toSch)
} else {
fromSch, toSch, err = expandFromToSchemas(fromSch, toSch)
if err != nil {
return nil, nil, err
}
j, err = rowconv.NewJoiner(
[]rowconv.NamedSchema{{Name: diff.To, Sch: toSch}, {Name: diff.From, Sch: fromSch}},
map[string]rowconv.ColNamingFunc{
diff.To: diff.ToColNamer,
diff.From: diff.FromColNamer,
})
if err != nil {
return nil, nil, err
}
diffTableSchema = j.GetSchema()
fullDiffCols := diffTableSchema.GetAllCols()
fullDiffCols = fullDiffCols.Append(
schema.NewColumn(diffTypeColName, schema.DiffTypeTag, types.StringKind, false),
)
diffTableSchema = schema.MustSchemaFromCols(fullDiffCols)
panic("Unsupported format for diff table schema calculation: " + format.VersionString())
}
return
}
// expandFromToSchemas converts input schemas to schemas appropriate for diffs. One argument must be
go/libraries/doltcore/sqle/dtables/versioned_table.go
+5
View File
@@ -25,3 +25,8 @@ type VersionableTable interface {
sql.Table
LockedToRoot(ctx *sql.Context, root doltdb.RootValue) (sql.IndexAddressableTable, error)
}
// RootSetter is an interface that can be used to set the root of a working set.
type RootSetter interface {
SetRoot(ctx *sql.Context, root doltdb.RootValue) error
}
go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go
-47
View File
@@ -490,25 +490,16 @@ func TestInsertIgnoreInto(t *testing.T) {
// TODO: merge this into the above test when we remove old format
func TestInsertDuplicateKeyKeyless(t *testing.T) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip()
}
enginetest.TestInsertDuplicateKeyKeyless(t, newDoltHarness(t))
}
// TODO: merge this into the above test when we remove old format
func TestInsertDuplicateKeyKeylessPrepared(t *testing.T) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip()
}
enginetest.TestInsertDuplicateKeyKeylessPrepared(t, newDoltHarness(t))
}
// TODO: merge this into the above test when we remove old format
func TestIgnoreIntoWithDuplicateUniqueKeyKeyless(t *testing.T) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip()
}
h := newDoltHarness(t)
defer h.Close()
enginetest.TestIgnoreIntoWithDuplicateUniqueKeyKeyless(t, h)
@@ -516,9 +507,6 @@ func TestIgnoreIntoWithDuplicateUniqueKeyKeyless(t *testing.T) {
// TODO: merge this into the above test when we remove old format
func TestIgnoreIntoWithDuplicateUniqueKeyKeylessPrepared(t *testing.T) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip()
}
enginetest.TestIgnoreIntoWithDuplicateUniqueKeyKeylessPrepared(t, newDoltHarness(t))
}
@@ -601,17 +589,14 @@ func TestSpatialScriptsPrepared(t *testing.T) {
}
func TestSpatialIndexScripts(t *testing.T) {
skipOldFormat(t)
enginetest.TestSpatialIndexScripts(t, newDoltHarness(t))
}
func TestSpatialIndexScriptsPrepared(t *testing.T) {
skipOldFormat(t)
enginetest.TestSpatialIndexScriptsPrepared(t, newDoltHarness(t))
}
func TestSpatialIndexPlans(t *testing.T) {
skipOldFormat(t)
enginetest.TestSpatialIndexPlans(t, newDoltHarness(t))
}
@@ -622,18 +607,12 @@ func TestTruncate(t *testing.T) {
}
func TestConvert(t *testing.T) {
if types.IsFormat_LD(types.Format_Default) {
t.Skip("noms format has outdated type enforcement")
}
h := newDoltHarness(t)
defer h.Close()
enginetest.TestConvertPrepared(t, h)
}
func TestConvertPrepared(t *testing.T) {
if types.IsFormat_LD(types.Format_Default) {
t.Skip("noms format has outdated type enforcement")
}
h := newDoltHarness(t)
defer h.Close()
enginetest.TestConvertPrepared(t, h)
@@ -868,7 +847,6 @@ func TestCreateDatabase(t *testing.T) {
}
func TestBlobs(t *testing.T) {
skipOldFormat(t)
h := newDoltHarness(t)
defer h.Close()
enginetest.TestBlobs(t, h)
@@ -899,14 +877,11 @@ func TestVectorType(t *testing.T) {
}
func TestIndexPrefix(t *testing.T) {
skipOldFormat(t)
harness := newDoltHarness(t)
RunIndexPrefixTest(t, harness)
}
func TestBigBlobs(t *testing.T) {
skipOldFormat(t)
h := newDoltHarness(t)
RunBigBlobsTest(t, h)
}
@@ -914,7 +889,6 @@ func TestBigBlobs(t *testing.T) {
func TestAdaptiveEncoding(t *testing.T) {
defer func() { schema.UseAdaptiveEncoding = false }()
schema.UseAdaptiveEncoding = true
skipOldFormat(t)
RunTestAdaptiveEncoding(t, newDoltHarness(t), AdaptiveEncodingTestType_Blob, AdaptiveEncodingTestPurpose_Representation)
RunTestAdaptiveEncoding(t, newDoltHarness(t), AdaptiveEncodingTestType_Blob, AdaptiveEncodingTestPurpose_Correctness)
@@ -956,9 +930,6 @@ func TestForeignKeyBranchesPrepared(t *testing.T) {
}
func TestFulltextIndexes(t *testing.T) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("FULLTEXT is not supported on the old format")
}
if runtime.GOOS == "windows" && os.Getenv("CI") != "" {
t.Skip("For some reason, this is flaky only on Windows CI.")
}
@@ -1341,21 +1312,6 @@ func TestDoltPreviewMergeConflictsPrepared(t *testing.T) {
RunDoltPreviewMergeConflictsPreparedTests(t, h)
}
// these tests are temporary while there is a difference between the old format
// and new format merge behaviors.
func TestOldFormatMergeConflictsAndCVs(t *testing.T) {
if types.IsFormat_DOLT(types.Format_Default) {
t.Skip()
}
for _, script := range OldFormatMergeConflictsAndCVsScripts {
func() {
h := newDoltHarness(t)
defer h.Close()
enginetest.TestScript(t, h, script)
}()
}
}
func TestDoltReset(t *testing.T) {
h := newDoltEnginetestHarness(t)
RunDoltResetTest(t, h)
@@ -1939,21 +1895,18 @@ func TestPreparedStatements(t *testing.T) {
}
func TestCharsetCollationEngine(t *testing.T) {
skipOldFormat(t)
h := newDoltHarness(t)
defer h.Close()
enginetest.TestCharsetCollationEngine(t, h)
}
func TestCharsetCollationWire(t *testing.T) {
skipOldFormat(t)
harness := newDoltHarness(t)
defer harness.Close()
enginetest.TestCharsetCollationWire(t, harness, newSessionBuilder(harness))
}
func TestDatabaseCollationWire(t *testing.T) {
skipOldFormat(t)
harness := newDoltHarness(t)
defer harness.Close()
enginetest.TestDatabaseCollationWire(t, harness, newSessionBuilder(harness))
go/libraries/doltcore/sqle/enginetest/dolt_engine_tests.go
+22 -77
View File
@@ -43,7 +43,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dtables"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/types"
)
const skipPreparedFlag = "DOLT_SKIP_PREPARED_ENGINETESTS"
@@ -802,9 +801,7 @@ func RunDoltDdlScripts(t *testing.T, harness DoltEnginetestHarness) {
require.NoError(t, err)
enginetest.TestScriptWithEngine(t, e, harness, script)
}
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("not fixing unique index on keyless tables for old format")
}
for _, script := range AddIndexScripts {
e, err := harness.NewEngine(t)
require.NoError(t, err)
@@ -952,21 +949,16 @@ func RunDoltConflictsTableNameTableTests(t *testing.T, h DoltEnginetestHarness)
}()
}
if types.IsFormat_DOLT(types.Format_Default) {
for _, script := range Dolt1ConflictTableNameTableTests {
func() {
h := h.NewHarness(t)
defer h.Close()
enginetest.TestScript(t, h, script)
}()
}
for _, script := range Dolt1ConflictTableNameTableTests {
func() {
h := h.NewHarness(t)
defer h.Close()
enginetest.TestScript(t, h, script)
}()
}
}
func RunKeylessDoltMergeCVsAndConflictsTests(t *testing.T, h DoltEnginetestHarness) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip()
}
for _, script := range KeylessMergeCVsAndConflictsScripts {
func() {
h := h.NewHarness(t)
@@ -1174,9 +1166,6 @@ func RunUnscopedDiffSystemTableTestsPrepared(t *testing.T, h DoltEnginetestHarne
}
func RunColumnDiffSystemTableTests(t *testing.T, h DoltEnginetestHarness) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("correct behavior of dolt_column_diff only guaranteed on new format")
}
for _, test := range ColumnDiffSystemTableScriptTests {
t.Run(test.Name, func(t *testing.T) {
enginetest.TestScript(t, h.NewHarness(t), test)
@@ -1185,9 +1174,6 @@ func RunColumnDiffSystemTableTests(t *testing.T, h DoltEnginetestHarness) {
}
func RunColumnDiffSystemTableTestsPrepared(t *testing.T, h DoltEnginetestHarness) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("correct behavior of dolt_column_diff only guaranteed on new format")
}
for _, test := range ColumnDiffSystemTableScriptTests {
t.Run(test.Name, func(t *testing.T) {
enginetest.TestScriptPrepared(t, h.NewHarness(t), test)
@@ -1393,10 +1379,6 @@ func RunCommitDiffSystemTableTestsPrepared(t *testing.T, harness DoltEnginetestH
}
func RunDoltDiffSystemTableTests(t *testing.T, h DoltEnginetestHarness) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("only new format support system table indexing")
}
for _, test := range DiffSystemTableScriptTests {
t.Run(test.Name, func(t *testing.T) {
h = h.NewHarness(t)
@@ -1406,23 +1388,17 @@ func RunDoltDiffSystemTableTests(t *testing.T, h DoltEnginetestHarness) {
})
}
if types.IsFormat_DOLT(types.Format_Default) {
for _, test := range Dolt1DiffSystemTableScripts {
func() {
h = h.NewHarness(t)
defer h.Close()
h.Setup(setup.MydbData)
enginetest.TestScript(t, h, test)
}()
}
for _, test := range Dolt1DiffSystemTableScripts {
func() {
h = h.NewHarness(t)
defer h.Close()
h.Setup(setup.MydbData)
enginetest.TestScript(t, h, test)
}()
}
}
func RunDoltDiffSystemTableTestsPrepared(t *testing.T, h DoltEnginetestHarness) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("only new format support system table indexing")
}
for _, test := range DiffSystemTableScriptTests {
t.Run(test.Name, func(t *testing.T) {
h = h.NewHarness(t)
@@ -1432,23 +1408,17 @@ func RunDoltDiffSystemTableTestsPrepared(t *testing.T, h DoltEnginetestHarness)
})
}
if types.IsFormat_DOLT(types.Format_Default) {
for _, test := range Dolt1DiffSystemTableScripts {
func() {
h = h.NewHarness(t)
defer h.Close()
h.Setup(setup.MydbData)
enginetest.TestScriptPrepared(t, h, test)
}()
}
for _, test := range Dolt1DiffSystemTableScripts {
func() {
h = h.NewHarness(t)
defer h.Close()
h.Setup(setup.MydbData)
enginetest.TestScriptPrepared(t, h, test)
}()
}
}
func RunNonlocalTableTests(t *testing.T, h DoltEnginetestHarness) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("only new format support system table indexing")
}
for _, test := range NonlocalScripts {
t.Run(test.Name, func(t *testing.T) {
h = h.NewHarness(t)
@@ -1460,10 +1430,6 @@ func RunNonlocalTableTests(t *testing.T, h DoltEnginetestHarness) {
}
func RunNonlocalTableTestsPrepared(t *testing.T, h DoltEnginetestHarness) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("only new format support system table indexing")
}
for _, test := range NonlocalScripts {
t.Run(test.Name, func(t *testing.T) {
h = h.NewHarness(t)
@@ -1519,10 +1485,6 @@ func RunQueryDiffTests(t *testing.T, harness DoltEnginetestHarness) {
}
func RunSystemTableIndexesTests(t *testing.T, harness DoltEnginetestHarness) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("only new format support system table indexing")
}
for _, stt := range SystemTableIndexTests {
harness = harness.NewHarness(t).WithParallelism(1)
defer harness.Close()
@@ -1572,10 +1534,6 @@ var biasedCosters = []memo.Coster{
}
func RunSystemTableIndexesTestsPrepared(t *testing.T, harness DoltEnginetestHarness) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip("only new format support system table indexing")
}
for _, stt := range SystemTableIndexTests {
harness = harness.NewHarness(t).WithParallelism(2)
defer harness.Close()
@@ -1919,12 +1877,7 @@ func RunDoltVerifyConstraintsTests(t *testing.T, harness DoltEnginetestHarness)
}
func RunDoltStorageFormatTests(t *testing.T, h DoltEnginetestHarness) {
var expectedFormatString string
if types.IsFormat_DOLT(types.Format_Default) {
expectedFormatString = "NEW ( __DOLT__ )"
} else {
expectedFormatString = fmt.Sprintf("OLD ( %s )", types.Format_Default.VersionString())
}
var expectedFormatString = "NEW ( __DOLT__ )"
script := queries.ScriptTest{
Name: "dolt storage format function works",
Assertions: []queries.ScriptTestAssertion{
@@ -1939,7 +1892,6 @@ func RunDoltStorageFormatTests(t *testing.T, h DoltEnginetestHarness) {
}
func RunThreeWayMergeWithSchemaChangeScripts(t *testing.T, h DoltEnginetestHarness) {
skipOldFormat(t)
runMergeScriptTestsInBothDirections(t, SchemaChangeTestsBasicCases, "basic cases", false)
runMergeScriptTestsInBothDirections(t, SchemaChangeTestsForDataConflicts, "data conflicts", false)
runMergeScriptTestsInBothDirections(t, SchemaChangeTestsCollations, "collation changes", false)
@@ -1962,7 +1914,6 @@ func RunThreeWayMergeWithSchemaChangeScripts(t *testing.T, h DoltEnginetestHarne
}
func RunThreeWayMergeWithSchemaChangeScriptsPrepared(t *testing.T, h DoltEnginetestHarness) {
skipOldFormat(t)
runMergeScriptTestsInBothDirections(t, SchemaChangeTestsBasicCases, "basic cases", false)
runMergeScriptTestsInBothDirections(t, SchemaChangeTestsForDataConflicts, "data conflicts", false)
runMergeScriptTestsInBothDirections(t, SchemaChangeTestsCollations, "collation changes", false)
@@ -2032,12 +1983,6 @@ var newFormatSkippedScripts = []string{
"Multiple indexes on the same columns in a different order",
}
func skipOldFormat(t *testing.T) {
if !types.IsFormat_DOLT(types.Format_Default) {
t.Skip()
}
}
func skipPreparedTests(t *testing.T) {
if skipPrepared {
t.Skip("skip prepared")
go/libraries/doltcore/sqle/enginetest/dolt_queries_merge.go
-356
View File
@@ -4594,362 +4594,6 @@ var SchemaConflictScripts = []queries.ScriptTest{
},
}
// OldFormatMergeConflictsAndCVsScripts tests old format merge behavior
// where violations are appended and merges are aborted if there are existing
// violations and/or conflicts.
var OldFormatMergeConflictsAndCVsScripts = []queries.ScriptTest{
{
Name: "merging branches into a constraint violated head. Any new violations are appended",
SetUpScript: []string{
"CREATE table parent (pk int PRIMARY KEY, col1 int);",
"CREATE table child (pk int PRIMARY KEY, parent_fk int, FOREIGN KEY (parent_fk) REFERENCES parent(pk));",
"CREATE table other (pk int);",
"CALL DOLT_ADD('.')",
"INSERT INTO parent VALUES (1, 1), (2, 2);",
"CALL DOLT_COMMIT('-am', 'setup');",
"CALL DOLT_BRANCH('branch1');",
"CALL DOLT_BRANCH('branch2');",
},
Assertions: []queries.ScriptTestAssertion{
{
// we need dolt_force_transaction_commit because we want to
// transaction commit constraint violations that occur as a
// result of a merge.
Query: "set autocommit = off, dolt_force_transaction_commit = on",
Expected: []sql.Row{{types.NewOkResult(0)}},
},
{
Query: "DELETE FROM parent where pk = 1;",
Expected: []sql.Row{{types.NewOkResult(1)}},
},
{
Query: "CALL DOLT_COMMIT('-am', 'delete parent 1');",
Expected: []sql.Row{{doltCommit}},
},
{
Query: "CALL DOLT_CHECKOUT('branch1');",
Expected: []sql.Row{{0, "Switched to branch 'branch1'"}},
},
{
Query: "INSERT INTO CHILD VALUES (1, 1);",
Expected: []sql.Row{{types.NewOkResult(1)}},
},
{
Query: "CALL DOLT_COMMIT('-am', 'insert child of parent 1');",
Expected: []sql.Row{{doltCommit}},
},
{
Query: "CALL DOLT_CHECKOUT('main');",
Expected: []sql.Row{{0, "Switched to branch 'main'"}},
},
{
Query: "CALL DOLT_MERGE('branch1');",
Expected: []sql.Row{{"", 0, 1, "conflicts found"}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
Expected: []sql.Row{{uint16(1), 1, 1}},
},
{
Query: "COMMIT;",
Expected: []sql.Row{},
},
{
Query: "CALL DOLT_COMMIT('-am', 'commit constraint violations');",
ExpectedErrStr: "error: the table(s) child have constraint violations",
},
{
Query: "CALL DOLT_COMMIT('-afm', 'commit constraint violations');",
Expected: []sql.Row{{doltCommit}},
},
{
Query: "CALL DOLT_BRANCH('branch3');",
Expected: []sql.Row{{0}},
},
{
Query: "DELETE FROM parent where pk = 2;",
Expected: []sql.Row{{types.NewOkResult(1)}},
},
{
Query: "CALL DOLT_COMMIT('-afm', 'remove parent 2');",
Expected: []sql.Row{{doltCommit}},
},
{
Query: "CALL DOLT_CHECKOUT('branch2');",
Expected: []sql.Row{{0, "Switched to branch 'branch2'"}},
},
{
Query: "INSERT INTO OTHER VALUES (1);",
Expected: []sql.Row{{types.NewOkResult(1)}},
},
{
Query: "CALL DOLT_COMMIT('-am', 'non-fk insert');",
Expected: []sql.Row{{doltCommit}},
},
{
Query: "CALL DOLT_CHECKOUT('main');",
Expected: []sql.Row{{0, "Switched to branch 'main'"}},
},
{
Query: "CALL DOLT_MERGE('branch2');",
Expected: []sql.Row{{"", 0, 1, "conflicts found"}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
Expected: []sql.Row{{uint16(1), 1, 1}},
},
{
Query: "COMMIT;",
Expected: []sql.Row{},
},
{
Query: "CALL DOLT_COMMIT('-am', 'commit non-conflicting merge');",
ExpectedErrStr: "error: the table(s) child have constraint violations",
},
{
Query: "CALL DOLT_COMMIT('-afm', 'commit non-conflicting merge');",
Expected: []sql.Row{{doltCommit}},
},
{
Query: "CALL DOLT_CHECKOUT('branch3');",
Expected: []sql.Row{{0, "Switched to branch 'branch3'"}},
},
{
Query: "INSERT INTO CHILD VALUES (2, 2);",
Expected: []sql.Row{{types.NewOkResult(1)}},
},
{
Query: "CALL DOLT_COMMIT('-afm', 'add child of parent 2');",
Expected: []sql.Row{{doltCommit}},
},
{
Query: "CALL DOLT_CHECKOUT('main');",
Expected: []sql.Row{{0, "Switched to branch 'main'"}},
},
{
Query: "CALL DOLT_MERGE('branch3');",
Expected: []sql.Row{{"", 0, 1, "conflicts found"}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
Expected: []sql.Row{{uint16(1), 1, 1}, {uint16(1), 2, 2}},
},
},
},
{
Name: "conflicting merge aborts when conflicts and violations already exist",
SetUpScript: []string{
"CREATE table parent (pk int PRIMARY KEY, col1 int);",
"CREATE table child (pk int PRIMARY KEY, parent_fk int, FOREIGN KEY (parent_fk) REFERENCES parent(pk));",
"CALL DOLT_ADD('.')",
"INSERT INTO parent VALUES (1, 1), (2, 1);",
"CALL DOLT_COMMIT('-am', 'create table with data');",
"CALL DOLT_BRANCH('other');",
"CALL DOLT_BRANCH('other2');",
"UPDATE parent SET col1 = 2 where pk = 1;",
"DELETE FROM parent where pk = 2;",
"CALL DOLT_COMMIT('-am', 'updating col1 to 2 and remove pk = 2');",
"CALL DOLT_CHECKOUT('other');",
"UPDATE parent SET col1 = 3 where pk = 1;",
"INSERT into child VALUEs (1, 2);",
"CALL DOLT_COMMIT('-am', 'updating col1 to 3 and adding child of pk 2');",
"CALL DOLT_CHECKOUT('other2')",
"UPDATE parent SET col1 = 4 where pk = 1",
"CALL DOLT_COMMIT('-am', 'updating col1 to 4');",
"CALL DOLT_CHECKOUT('main');",
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SET dolt_force_transaction_commit = 1",
Expected: []sql.Row{{types.NewOkResult(0)}},
},
{
Query: "CALL DOLT_MERGE('other');",
Expected: []sql.Row{{"", 0, 1, "conflicts found"}},
},
{
Query: "SELECT * from parent;",
Expected: []sql.Row{{1, 2}},
},
{
Query: "SELECT * from child;",
Expected: []sql.Row{{1, 2}},
},
{
Query: "SELECT base_col1, base_pk, our_col1, our_pk, their_col1, their_pk from dolt_conflicts_parent;",
Expected: []sql.Row{{1, 1, 2, 1, 3, 1}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
Expected: []sql.Row{{uint16(1), 1, 2}},
},
// commit so we can merge again
{
Query: "CALL DOLT_COMMIT('-afm', 'committing merge conflicts');",
Expected: []sql.Row{{doltCommit}},
},
{
Query: "CALL DOLT_MERGE('other2');",
ExpectedErrStr: "existing unresolved conflicts would be overridden by new conflicts produced by merge. Please resolve them and try again",
},
{
Query: "SELECT * from parent;",
Expected: []sql.Row{{1, 2}},
},
{
Query: "SELECT * from child;",
Expected: []sql.Row{{1, 2}},
},
{
Query: "SELECT base_col1, base_pk, our_col1, our_pk, their_col1, their_pk from dolt_conflicts_parent;",
Expected: []sql.Row{{1, 1, 2, 1, 3, 1}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
Expected: []sql.Row{{uint16(1), 1, 2}},
},
},
},
{
Name: "non-conflicting / non-violating merge succeeds when conflicts and violations already exist",
SetUpScript: []string{
"CREATE table parent (pk int PRIMARY KEY, col1 int);",
"CREATE table child (pk int PRIMARY KEY, parent_fk int, FOREIGN KEY (parent_fk) REFERENCES parent(pk));",
"CALL DOLT_ADD('.')",
"INSERT INTO parent VALUES (1, 1), (2, 1);",
"CALL DOLT_COMMIT('-am', 'create table with data');",
"CALL DOLT_BRANCH('other');",
"CALL DOLT_BRANCH('other2');",
"UPDATE parent SET col1 = 2 where pk = 1;",
"DELETE FROM parent where pk = 2;",
"CALL DOLT_COMMIT('-am', 'updating col1 to 2 and remove pk = 2');",
"CALL DOLT_CHECKOUT('other');",
"UPDATE parent SET col1 = 3 where pk = 1;",
"INSERT into child VALUES (1, 2);",
"CALL DOLT_COMMIT('-am', 'updating col1 to 3 and adding child of pk 2');",
"CALL DOLT_CHECKOUT('other2')",
"INSERT INTO parent values (3, 1);",
"CALL DOLT_COMMIT('-am', 'insert parent with pk 3');",
"CALL DOLT_CHECKOUT('main');",
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "SET dolt_force_transaction_commit = 1;",
Expected: []sql.Row{{types.NewOkResult(0)}},
},
{
Query: "CALL DOLT_MERGE('other');",
Expected: []sql.Row{{"", 0, 1, "conflicts found"}},
},
{
Query: "SELECT * from parent;",
Expected: []sql.Row{{1, 2}},
},
{
Query: "SELECT * from child;",
Expected: []sql.Row{{1, 2}},
},
{
Query: "SELECT base_col1, base_pk, our_col1, our_pk, their_col1, their_pk from dolt_conflicts_parent;",
Expected: []sql.Row{{1, 1, 2, 1, 3, 1}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
Expected: []sql.Row{{uint16(1), 1, 2}},
},
// commit so we can merge again
{
Query: "CALL DOLT_COMMIT('-afm', 'committing merge conflicts');",
Expected: []sql.Row{{doltCommit}},
},
{
Query: "CALL DOLT_MERGE('other2');",
Expected: []sql.Row{{"", 0, 1, "conflicts found"}},
},
{
Query: "SELECT * from parent;",
Expected: []sql.Row{{1, 2}, {3, 1}},
},
{
Query: "SELECT * from child;",
Expected: []sql.Row{{1, 2}},
},
{
Query: "SELECT base_col1, base_pk, our_col1, our_pk, their_col1, their_pk from dolt_conflicts_parent;",
Expected: []sql.Row{{1, 1, 2, 1, 3, 1}},
},
{
Query: "SELECT violation_type, pk, parent_fk from dolt_constraint_violations_child;",
Expected: []sql.Row{{uint16(1), 1, 2}},
},
},
},
// Unique key violations
{
Name: "unique key violations that already exist in the left abort the merge with an error",
SetUpScript: []string{
"SET dolt_force_transaction_commit = on;",
"CREATE TABLE t (pk int PRIMARY KEY, col1 int);",
"CALL DOLT_ADD('.')",
"CALL DOLT_COMMIT('-am', 'table');",
"CALL DOLT_BRANCH('right');",
"INSERT INTO t VALUES (1, 1), (2, 1);",
"CALL DOLT_COMMIT('-am', 'data');",
"CALL DOLT_CHECKOUT('right');",
"ALTER TABLE t ADD UNIQUE col1_uniq (col1);",
"CALL DOLT_COMMIT('-am', 'unqiue constraint');",
"CALL DOLT_CHECKOUT('main');",
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "CALL DOLT_MERGE('right');",
ExpectedErrStr: "duplicate unique key given: [1]",
},
{
Query: "SELECT * from t",
Expected: []sql.Row{{1, 1}, {2, 1}},
},
{
Query: "show create table t",
Expected: []sql.Row{{"t",
"CREATE TABLE `t` (\n" +
" `pk` int NOT NULL,\n" +
" `col1` int,\n" +
" PRIMARY KEY (`pk`)\n" +
") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"}},
},
},
},
// not a base case, but helpful to understand...
{
Name: "right adds a unique key constraint and fixes existing violations. On merge, because left still has the violation, merge is aborted.",
SetUpScript: []string{
"SET dolt_force_transaction_commit = on;",
"CREATE TABLE t (pk int PRIMARY KEY, col1 int);",
"CALL DOLT_ADD('.');",
"INSERT INTO t VALUES (1, 1), (2, 1);",
"CALL DOLT_COMMIT('-am', 'table and data');",
"CALL DOLT_CHECKOUT('-b', 'right');",
"UPDATE t SET col1 = 2 where pk = 2;",
"ALTER TABLE t ADD UNIQUE col1_uniq (col1);",
"CALL DOLT_COMMIT('-am', 'right adds a unique index');",
"CALL DOLT_CHECKOUT('main');",
"INSERT INTO t VALUES (3, 3);",
"CALL DOLT_COMMIT('-am', 'left edit');",
},
Assertions: []queries.ScriptTestAssertion{
{
Query: "CALL DOLT_MERGE('right');",
ExpectedErrStr: "duplicate unique key given: [1]",
},
},
},
}
var GeneratedColumnMergeTestScripts = []queries.ScriptTest{
{
Name: "merge a generated stored column",
go/libraries/doltcore/sqle/enginetest/validation.go
-4
View File
@@ -30,7 +30,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/sqle"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
"github.com/dolthub/dolt/go/store/val"
)
@@ -46,9 +45,6 @@ func ValidateDatabase(ctx context.Context, db sql.Database) (err error) {
}
func ValidateDoltDatabase(ctx context.Context, db sqle.Database) (err error) {
if !types.IsFormat_DOLT(db.GetDoltDB().Format()) {
return nil
}
for _, stage := range validationStages {
if err = stage(ctx, db); err != nil {
return err
go/libraries/doltcore/sqle/index/index_reader.go
+18 -99
View File
@@ -58,13 +58,9 @@ func RowIterForIndexLookup(ctx *sql.Context, t DoltTableable, lookup sql.IndexLo
return nil, err
}
return RowIterForProllyRange(ctx, idx, prollyRanges[0], pkSch, columns, durableState, lookup.IsReverse)
} else {
nomsRanges, err := idx.nomsRanges(ctx, mysqlRanges...)
if err != nil {
return nil, err
}
return RowIterForNomsRanges(ctx, idx, nomsRanges, columns, durableState)
}
panic("Unsupported format for RowIterForIndexLookup")
}
func RowIterForProllyRange(ctx *sql.Context, idx DoltIndex, r prolly.Range, pkSch sql.PrimaryKeySchema, projections []uint64, durableState *durableIndexState, reverse bool) (sql.RowIter, error) {
@@ -85,49 +81,27 @@ func RowIterForProllyRange(ctx *sql.Context, idx DoltIndex, r prolly.Range, pkSc
return newProllyIndexIter(ctx, idx, r, pkSch, projections, durableState.Primary, durableState.Secondary)
}
func RowIterForNomsRanges(ctx *sql.Context, idx DoltIndex, ranges []*noms.ReadRange, columns []uint64, durableState *durableIndexState) (sql.RowIter, error) {
if len(columns) == 0 {
columns = idx.Schema().GetAllCols().Tags
}
m := durable.NomsMapFromIndex(durableState.Secondary)
nrr := noms.NewNomsRangeReader(idx.valueReadWriter(), idx.IndexSchema(), m, ranges)
covers := idx.coversColumns(durableState, columns)
if covers || idx.ID() == "PRIMARY" {
return NewCoveringIndexRowIterAdapter(ctx, idx, nrr, columns), nil
} else {
return NewIndexLookupRowIterAdapter(ctx, idx, durableState, nrr, columns)
}
}
type IndexLookupKeyIterator interface {
// NextKey returns the next key if it exists, and io.EOF if it does not.
NextKey(ctx *sql.Context) (row.TaggedValues, error)
}
func NewRangePartitionIter(ctx *sql.Context, t DoltTableable, lookup sql.IndexLookup, isDoltFmt bool) (sql.PartitionIter, error) {
func NewRangePartitionIter(ctx *sql.Context, t DoltTableable, lookup sql.IndexLookup) (sql.PartitionIter, error) {
mysqlRanges := lookup.Ranges.(sql.MySQLRangeCollection)
idx := lookup.Index.(*doltIndex)
if lookup.IsPointLookup && isDoltFmt {
if lookup.IsPointLookup {
return newPointPartitionIter(ctx, lookup, idx)
}
var prollyRanges []prolly.Range
var nomsRanges []*noms.ReadRange
var err error
if isDoltFmt {
prollyRanges, err = idx.prollyRanges(ctx, idx.ns, mysqlRanges...)
} else {
nomsRanges, err = idx.nomsRanges(ctx, mysqlRanges...)
}
prollyRanges, err = idx.prollyRanges(ctx, idx.ns, mysqlRanges...)
if err != nil {
return nil, err
}
return &rangePartitionIter{
nomsRanges: nomsRanges,
prollyRanges: prollyRanges,
curr: 0,
isDoltFmt: isDoltFmt,
isReverse: lookup.IsReverse,
}, nil
}
@@ -167,10 +141,8 @@ func (p *pointPartition) Next(c *sql.Context) (sql.Partition, error) {
}
type rangePartitionIter struct {
nomsRanges []*noms.ReadRange
prollyRanges []prolly.Range
curr int
isDoltFmt bool
isReverse bool
}
@@ -181,10 +153,7 @@ func (itr *rangePartitionIter) Close(*sql.Context) error {
// Next returns the next partition if there is one, or io.EOF if there isn't.
func (itr *rangePartitionIter) Next(_ *sql.Context) (sql.Partition, error) {
if itr.isDoltFmt {
return itr.nextProllyPartition()
}
return itr.nextNomsPartition()
return itr.nextProllyPartition()
}
func (itr *rangePartitionIter) nextProllyPartition() (sql.Partition, error) {
@@ -204,23 +173,6 @@ func (itr *rangePartitionIter) nextProllyPartition() (sql.Partition, error) {
}, nil
}
func (itr *rangePartitionIter) nextNomsPartition() (sql.Partition, error) {
if itr.curr >= len(itr.nomsRanges) {
return nil, io.EOF
}
var bytes [4]byte
binary.BigEndian.PutUint32(bytes[:], uint32(itr.curr))
nr := itr.nomsRanges[itr.curr]
itr.curr += 1
return rangePartition{
nomsRange: nr,
key: bytes[:],
isReverse: itr.isReverse,
}, nil
}
type rangePartition struct {
nomsRange *noms.ReadRange
key []byte
@@ -310,7 +262,6 @@ func NewIndexReaderBuilder(
key doltdb.DataCacheKey,
projections []uint64,
pkSch sql.PrimaryKeySchema,
isDoltFormat bool,
) (IndexScanBuilder, error) {
if projections == nil {
projections = idx.Schema().GetAllCols().Tags
@@ -328,27 +279,21 @@ func NewIndexReaderBuilder(
projections: projections,
}
if isDoltFormat {
secondaryIndex := durable.MapFromIndex(s.Secondary)
base.ns = secondaryIndex.NodeStore()
base.secKd, base.secVd = secondaryIndex.Descriptors()
base.prefDesc = base.secKd.PrefixDesc(len(di.columns))
switch si := secondaryIndex.(type) {
case prolly.Map:
base.sec = si
case prolly.ProximityMap:
base.proximitySecondary = si
default:
return nil, fmt.Errorf("unknown index type %v", secondaryIndex)
}
secondaryIndex := durable.MapFromIndex(s.Secondary)
base.ns = secondaryIndex.NodeStore()
base.secKd, base.secVd = secondaryIndex.Descriptors()
base.prefDesc = base.secKd.PrefixDesc(len(di.columns))
switch si := secondaryIndex.(type) {
case prolly.Map:
base.sec = si
case prolly.ProximityMap:
base.proximitySecondary = si
default:
return nil, fmt.Errorf("unknown index type %v", secondaryIndex)
}
switch {
case !isDoltFormat:
return &nomsIndexImplBuilder{
baseIndexImplBuilder: base,
s: s,
}, nil
case sql.IsKeyless(pkSch.Schema):
return &keylessIndexImplBuilder{
baseIndexImplBuilder: base,
@@ -412,7 +357,6 @@ func newNonCoveringLookupBuilder(s *durableIndexState, b *baseIndexImplBuilder)
}
var _ IndexScanBuilder = (*baseIndexImplBuilder)(nil)
var _ IndexScanBuilder = (*nomsIndexImplBuilder)(nil)
var _ IndexScanBuilder = (*coveringIndexImplBuilder)(nil)
var _ IndexScanBuilder = (*keylessIndexImplBuilder)(nil)
var _ IndexScanBuilder = (*nonCoveringIndexImplBuilder)(nil)
@@ -875,31 +819,6 @@ func (ib *keylessIndexImplBuilder) NewSecondaryIter(strict bool, cnt int, nullSa
}, nil
}
type nomsIndexImplBuilder struct {
*baseIndexImplBuilder
s *durableIndexState
}
func (ib *nomsIndexImplBuilder) OutputSchema() schema.Schema {
return ib.baseIndexImplBuilder.idx.Schema()
}
// NewPartitionRowIter implements IndexScanBuilder
func (ib *nomsIndexImplBuilder) NewPartitionRowIter(ctx *sql.Context, part sql.Partition) (sql.RowIter, error) {
p := part.(rangePartition)
ranges := []*noms.ReadRange{p.nomsRange}
return RowIterForNomsRanges(ctx, ib.idx, ranges, ib.projections, ib.s)
}
// NewRangeMapIter implements IndexScanBuilder
func (ib *nomsIndexImplBuilder) NewRangeMapIter(ctx context.Context, r prolly.Range, reverse bool) (prolly.MapIter, error) {
panic("cannot call NewMapIter on *nomsIndexImplBuilder")
}
func (ib *nomsIndexImplBuilder) NewSecondaryIter(strict bool, cnt int, nullSafe []bool) (SecondaryLookupIterGen, error) {
panic("cannot call NewSecondaryIter on *nomsIndexImplBuilder")
}
// boundsCase determines the case upon which the bounds are tested.
type boundsCase byte
go/libraries/doltcore/sqle/index/mergeable_indexes_setup_test.go
-219
View File
@@ -1,219 +0,0 @@
// Copyright 2020 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package index_test
import (
"context"
"fmt"
"io"
"testing"
sqle "github.com/dolthub/go-mysql-server"
"github.com/dolthub/go-mysql-server/sql"
"github.com/stretchr/testify/require"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/dtestutils"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
"github.com/dolthub/dolt/go/store/types"
)
func setupIndexes(t *testing.T, tableName, insertQuery string) (*sqle.Engine, *sql.Context, []*indexTuple) {
ctx := context.Background()
dEnv := dtestutils.CreateTestEnv()
tmpDir, err := dEnv.TempTableFilesDir()
require.NoError(t, err)
deaf, err := dEnv.DbEaFactory(ctx)
require.NoError(t, err)
opts := editor.Options{Deaf: deaf, Tempdir: tmpDir}
db, err := dsqle.NewDatabase(context.Background(), "dolt", dEnv.DbData(ctx), opts)
require.NoError(t, err)
engine, sqlCtx, err := dsqle.NewTestEngine(dEnv, context.Background(), db)
require.NoError(t, err)
_, iter, _, err := engine.Query(sqlCtx, fmt.Sprintf(`CREATE TABLE %s (
pk bigint PRIMARY KEY,
v1 bigint,
v2 bigint,
INDEX idxv1 (v1),
INDEX idxv2v1 (v2,v1)
)`, tableName))
require.NoError(t, err)
require.NoError(t, drainIter(sqlCtx, iter))
_, iter, _, err = engine.Query(sqlCtx, insertQuery)
require.NoError(t, err)
require.NoError(t, drainIter(sqlCtx, iter))
sqlTbl, ok, err := db.GetTableInsensitive(sqlCtx, tableName)
require.NoError(t, err)
require.True(t, ok)
tbl, ok := sqlTbl.(*dsqle.AlterableDoltTable)
require.True(t, ok)
sch := dsqle.DoltSchemaFromAlterableTable(tbl)
idxv1, ok := sch.Indexes().GetByNameCaseInsensitive("idxv1")
require.True(t, ok)
table := dsqle.DoltTableFromAlterableTable(sqlCtx, tbl)
idxv1RowData, err := table.GetIndexRowData(context.Background(), idxv1.Name())
require.NoError(t, err)
idxv1Cols := make([]schema.Column, idxv1.Count())
for i, tag := range idxv1.IndexedColumnTags() {
idxv1Cols[i], _ = idxv1.GetColumn(tag)
}
idxv1ToTuple := &indexTuple{
nbf: idxv1RowData.Format(),
cols: idxv1Cols,
}
idxv2v1, ok := sch.Indexes().GetByNameCaseInsensitive("idxv2v1")
require.True(t, ok)
idxv2v1RowData, err := table.GetNomsIndexRowData(context.Background(), idxv2v1.Name())
require.NoError(t, err)
idxv2v1Cols := make([]schema.Column, idxv2v1.Count())
for i, tag := range idxv2v1.IndexedColumnTags() {
idxv2v1Cols[i], _ = idxv2v1.GetColumn(tag)
}
idxv2v1ToTuple := &indexTuple{
nbf: idxv2v1RowData.Format(),
cols: idxv2v1Cols,
}
mrEnv, err := env.MultiEnvForDirectory(context.Background(), dEnv.Config.WriteableConfig(), dEnv.FS, dEnv.Version, dEnv)
require.NoError(t, err)
b := env.GetDefaultInitBranch(dEnv.Config)
pro, err := dsqle.NewDoltDatabaseProviderWithDatabase(b, mrEnv.FileSystem(), db, dEnv.FS, sql.EngineOverrides{})
if err != nil {
return nil, nil, nil
}
pro = pro.WithDbFactoryUrl(doltdb.InMemDoltDB)
engine = sqle.NewDefault(pro)
it := []*indexTuple{
idxv1ToTuple,
idxv2v1ToTuple,
{
nbf: idxv2v1RowData.Format(),
cols: idxv2v1Cols[:len(idxv2v1Cols)-1],
},
}
return engine, sqlCtx, it
}
// indexTuple converts integers into the appropriate tuple for comparison against ranges
type indexTuple struct {
nbf *types.NomsBinFormat
cols []schema.Column
}
func (it *indexTuple) tuple(vals ...int) types.Tuple {
if len(it.cols) != len(vals) {
panic("len of columns in index does not match the given number of values")
}
valsWithTags := make([]types.Value, len(vals)*2)
for i, val := range vals {
valsWithTags[2*i] = types.Uint(it.cols[i].Tag)
valsWithTags[2*i+1] = types.Int(val)
}
tpl, err := types.NewTuple(it.nbf, valsWithTags...)
if err != nil {
panic(err)
}
return tpl
}
func (it *indexTuple) nilTuple() types.Tuple {
valsWithTags := make([]types.Value, len(it.cols)*2)
for i := 0; i < len(it.cols); i++ {
valsWithTags[2*i] = types.Uint(it.cols[i].Tag)
valsWithTags[2*i+1] = types.NullValue
}
tpl, err := types.NewTuple(it.nbf, valsWithTags...)
if err != nil {
panic(err)
}
return tpl
}
func drainIter(ctx *sql.Context, iter sql.RowIter) error {
for {
_, err := iter.Next(ctx)
if err == io.EOF {
break
} else if err != nil {
closeErr := iter.Close(ctx)
if closeErr != nil {
panic(fmt.Errorf("%v\n%v", err, closeErr))
}
return err
}
}
return iter.Close(ctx)
}
func getDbState(t *testing.T, db sql.Database, dEnv *env.DoltEnv) (dsess.InitialDbState, error) {
ctx := context.Background()
headSpec, err := dEnv.RepoStateReader().CWBHeadSpec(ctx)
if err != nil {
return dsess.InitialDbState{}, err
}
headRef, err := dEnv.RepoStateReader().CWBHeadRef(ctx)
if err != nil {
return dsess.InitialDbState{}, err
}
optCmt, err := dEnv.DoltDB(ctx).Resolve(ctx, headSpec, headRef)
require.NoError(t, err)
headCommit, ok := optCmt.ToCommit()
require.True(t, ok)
ws, err := dEnv.WorkingSet(ctx)
require.NoError(t, err)
return dsess.InitialDbState{
Db: db,
HeadCommit: headCommit,
WorkingSet: ws,
DbData: env.DbData[*sql.Context]{
Ddb: dEnv.DoltDB(ctx),
Rsr: forwardCtxDbData{dEnv.DbData(ctx).Rsr},
Rsw: dEnv.DbData(ctx).Rsw,
},
Remotes: dEnv.RepoState.Remotes,
}, nil
}
type forwardCtxDbData struct {
env.RepoStateReader[context.Context]
}
func (d forwardCtxDbData) CWBHeadSpec(ctx *sql.Context) (*doltdb.CommitSpec, error) {
return d.RepoStateReader.CWBHeadSpec(ctx)
}
func (d forwardCtxDbData) CWBHeadRef(ctx *sql.Context) (ref.DoltRef, error) {
return d.RepoStateReader.CWBHeadRef(ctx)
}
go/libraries/doltcore/sqle/index/mergeable_indexes_test.go
-1588
View File
File diff suppressed because it is too large Load Diff
go/libraries/doltcore/sqle/index/noms_index_iter.go
-290
View File
@@ -1,290 +0,0 @@
// Copyright 2020 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package index
import (
"context"
"io"
"sync"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms"
"github.com/dolthub/dolt/go/libraries/utils/async"
"github.com/dolthub/dolt/go/store/types"
)
const (
ringBufferAllocSize = 1024
)
var resultBufferPool = &sync.Pool{
New: func() interface{} {
return async.NewRingBuffer(ringBufferAllocSize)
},
}
type indexLookupRowIterAdapter struct {
idx DoltIndex
keyIter nomsKeyIter
tableRows types.Map
lookupTags map[uint64]int
conv *KVToSqlRowConverter
resultBuf *async.RingBuffer
cancelF func()
read uint64
count uint64
}
// NewIndexLookupRowIterAdapter returns a new indexLookupRowIterAdapter.
func NewIndexLookupRowIterAdapter(ctx *sql.Context, idx DoltIndex, durableState *durableIndexState, keyIter nomsKeyIter, columns []uint64) (*indexLookupRowIterAdapter, error) {
rows := durable.NomsMapFromIndex(durableState.Primary)
resBuf := resultBufferPool.Get().(*async.RingBuffer)
epoch := resBuf.Reset()
queueCtx, cancelF := context.WithCancel(ctx)
iter := &indexLookupRowIterAdapter{
idx: idx,
keyIter: keyIter,
tableRows: rows,
conv: idx.sqlRowConverter(durableState, columns),
lookupTags: idx.lookupTags(durableState),
cancelF: cancelF,
resultBuf: resBuf,
}
go iter.queueRows(queueCtx, epoch)
return iter, nil
}
// Next returns the next row from the iterator.
func (i *indexLookupRowIterAdapter) Next(ctx *sql.Context) (sql.Row, error) {
for i.count == 0 || i.read < i.count {
item, err := i.resultBuf.Pop()
if err != nil {
return nil, err
}
res := item.(lookupResult)
i.read++
if res.err != nil {
if res.err == io.EOF {
i.count = res.idx
continue
}
return nil, res.err
}
return res.r, res.err
}
return nil, io.EOF
}
func (i *indexLookupRowIterAdapter) Close(*sql.Context) error {
i.cancelF()
resultBufferPool.Put(i.resultBuf)
return nil
}
// queueRows reads each key from the key iterator and writes it to lookups.toLookupCh which manages a pool of worker
// routines which will process the requests in parallel.
func (i *indexLookupRowIterAdapter) queueRows(ctx context.Context, epoch int) {
for idx := uint64(1); ; idx++ {
indexKey, err := i.keyIter.ReadKey(ctx)
if err != nil {
i.resultBuf.Push(lookupResult{
idx: idx,
r: nil,
err: err,
}, epoch)
return
}
lookup := toLookup{
idx: idx,
t: indexKey,
tupleToRow: i.processKey,
resBuf: i.resultBuf,
epoch: epoch,
ctx: ctx,
}
select {
case lookups.toLookupCh <- lookup:
case <-ctx.Done():
err := ctx.Err()
if err == nil {
err = io.EOF
}
i.resultBuf.Push(lookupResult{
idx: idx,
r: nil,
err: err,
}, epoch)
return
}
}
}
func (i *indexLookupRowIterAdapter) indexKeyToTableKey(nbf *types.NomsBinFormat, indexKey types.Tuple) (types.Tuple, error) {
tplItr, err := indexKey.Iterator()
if err != nil {
return types.Tuple{}, err
}
resVals := make([]types.Value, len(i.lookupTags)*2)
for {
_, tag, err := tplItr.NextUint64()
if err != nil {
if err == io.EOF {
break
}
return types.Tuple{}, err
}
idx, inKey := i.lookupTags[tag]
if inKey {
_, valVal, err := tplItr.Next()
if err != nil {
return types.Tuple{}, err
}
resVals[idx*2] = types.Uint(tag)
resVals[idx*2+1] = valVal
} else {
err := tplItr.Skip()
if err != nil {
return types.Tuple{}, err
}
}
}
return types.NewTuple(nbf, resVals...)
}
// processKey is called within queueRows and processes each key, sending the resulting row to the row channel.
func (i *indexLookupRowIterAdapter) processKey(ctx context.Context, indexKey types.Tuple) (sql.Row, error) {
pkTupleVal, err := i.indexKeyToTableKey(i.idx.Format(), indexKey)
if err != nil {
return nil, err
}
fieldsVal, ok, err := i.tableRows.MaybeGetTuple(ctx, pkTupleVal)
if err != nil {
return nil, err
}
if !ok {
return nil, nil
}
sqlRow, err := i.conv.ConvertKVTuplesToSqlRow(pkTupleVal, fieldsVal)
if err != nil {
return nil, err
}
return sqlRow, nil
}
type CoveringIndexRowIterAdapter struct {
idx DoltIndex
rr *noms.NomsRangeReader
conv *KVToSqlRowConverter
ctx *sql.Context
pkCols *schema.ColCollection
nonPKCols *schema.ColCollection
nbf *types.NomsBinFormat
}
func NewCoveringIndexRowIterAdapter(ctx *sql.Context, idx DoltIndex, keyIter *noms.NomsRangeReader, resultCols []uint64) *CoveringIndexRowIterAdapter {
idxCols := idx.IndexSchema().GetPKCols()
tblPKCols := idx.Schema().GetPKCols()
sch := idx.Schema()
allCols := sch.GetAllCols().GetColumns()
cols := make([]schema.Column, 0)
for _, col := range allCols {
for _, tag := range resultCols {
if col.Tag == tag {
cols = append(cols, col)
}
}
}
tagToSqlColIdx := make(map[uint64]int)
isPrimaryKeyIdx := idx.ID() == "PRIMARY"
resultColSet := make(map[uint64]bool)
for _, k := range resultCols {
resultColSet[k] = true
}
for i, col := range cols {
_, partOfIdxKey := idxCols.GetByNameCaseInsensitive(col.Name)
// Either this is a primary key index or the key is a part of the index and this part of the result column set.
if (partOfIdxKey || isPrimaryKeyIdx) && (len(resultCols) == 0 || resultColSet[col.Tag]) {
tagToSqlColIdx[col.Tag] = i
}
}
// TAGS: Use of tags here is safe since it's constrained to a single table
for i, col := range cols {
_, partOfIndexKey := idxCols.GetByTag(col.Tag)
_, partOfTableKeys := tblPKCols.GetByTag(col.Tag)
if partOfIndexKey != partOfTableKeys {
cols[i], _ = schema.NewColumnWithTypeInfo(col.Name, col.Tag, col.TypeInfo, partOfIndexKey, col.Default, col.AutoIncrement, col.Comment, col.Constraints...)
}
}
return &CoveringIndexRowIterAdapter{
idx: idx,
rr: keyIter,
conv: NewKVToSqlRowConverter(idx.Format(), tagToSqlColIdx, cols, len(cols)),
ctx: ctx,
pkCols: sch.GetPKCols(),
nonPKCols: sch.GetNonPKCols(),
nbf: idx.Format(),
}
}
// Next returns the next row from the iterator.
func (ci *CoveringIndexRowIterAdapter) Next(ctx *sql.Context) (sql.Row, error) {
key, value, err := ci.rr.ReadKV(ctx)
if err != nil {
return nil, err
}
return ci.conv.ConvertKVTuplesToSqlRow(key, value)
}
func (ci *CoveringIndexRowIterAdapter) Close(*sql.Context) error {
return nil
}
go/libraries/doltcore/sqle/indexed_dolt_table.go
+12 -17
View File
@@ -20,7 +20,6 @@ import (
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/store/types"
)
// IndexedDoltTable is a wrapper for a DoltTable. It implements the sql.Table interface like
@@ -30,16 +29,14 @@ type IndexedDoltTable struct {
idx index.DoltIndex
lb index.IndexScanBuilder
*DoltTable
mu *sync.Mutex
isDoltFormat bool
mu *sync.Mutex
}
func NewIndexedDoltTable(t *DoltTable, idx index.DoltIndex) *IndexedDoltTable {
return &IndexedDoltTable{
DoltTable: t,
idx: idx,
isDoltFormat: types.IsFormat_DOLT(t.Format()),
mu: &sync.Mutex{},
DoltTable: t,
idx: idx,
mu: &sync.Mutex{},
}
}
@@ -58,13 +55,13 @@ func (t *IndexedDoltTable) LookupBuilder(ctx *sql.Context) (index.IndexScanBuild
return nil, err
}
if t.lb == nil || !canCache || t.lb.Key() != key {
return index.NewIndexReaderBuilder(ctx, t.DoltTable, t.idx, key, t.DoltTable.projectedCols, t.DoltTable.sqlSch, t.isDoltFormat)
return index.NewIndexReaderBuilder(ctx, t.DoltTable, t.idx, key, t.DoltTable.projectedCols, t.DoltTable.sqlSch)
}
return t.lb, nil
}
func (idt *IndexedDoltTable) LookupPartitions(ctx *sql.Context, lookup sql.IndexLookup) (sql.PartitionIter, error) {
return index.NewRangePartitionIter(ctx, idt.DoltTable, lookup, idt.isDoltFormat)
return index.NewRangePartitionIter(ctx, idt.DoltTable, lookup)
}
func (idt *IndexedDoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
@@ -80,7 +77,7 @@ func (idt *IndexedDoltTable) PartitionRows(ctx *sql.Context, part sql.Partition)
}
if idt.lb == nil || !canCache || idt.lb.Key() != key {
idt.lb, err = index.NewIndexReaderBuilder(ctx, idt.DoltTable, idt.idx, key, idt.DoltTable.projectedCols, idt.DoltTable.sqlSch, idt.isDoltFormat)
idt.lb, err = index.NewIndexReaderBuilder(ctx, idt.DoltTable, idt.idx, key, idt.DoltTable.projectedCols, idt.DoltTable.sqlSch)
if err != nil {
return nil, err
}
@@ -97,7 +94,7 @@ func (idt *IndexedDoltTable) PartitionRows2(ctx *sql.Context, part sql.Partition
return nil, err
}
if idt.lb == nil || !canCache || idt.lb.Key() != key {
idt.lb, err = index.NewIndexReaderBuilder(ctx, idt.DoltTable, idt.idx, key, idt.DoltTable.projectedCols, idt.DoltTable.sqlSch, idt.isDoltFormat)
idt.lb, err = index.NewIndexReaderBuilder(ctx, idt.DoltTable, idt.idx, key, idt.DoltTable.projectedCols, idt.DoltTable.sqlSch)
if err != nil {
return nil, err
}
@@ -117,7 +114,6 @@ func NewWritableIndexedDoltTable(t *WritableDoltTable, idx index.DoltIndex) *Wri
return &WritableIndexedDoltTable{
WritableDoltTable: t,
idx: idx,
isDoltFormat: types.IsFormat_DOLT(idx.Format()),
mu: &sync.Mutex{},
}
}
@@ -126,8 +122,7 @@ type WritableIndexedDoltTable struct {
idx index.DoltIndex
lb index.IndexScanBuilder
*WritableDoltTable
mu *sync.Mutex
isDoltFormat bool
mu *sync.Mutex
}
func (t *WritableIndexedDoltTable) Index() index.DoltIndex {
@@ -142,7 +137,7 @@ func (t *WritableIndexedDoltTable) LookupBuilder(ctx *sql.Context) (index.IndexS
return nil, err
}
if t.lb == nil || !canCache || t.lb.Key() != key {
return index.NewIndexReaderBuilder(ctx, t.DoltTable, t.idx, key, t.DoltTable.projectedCols, t.DoltTable.sqlSch, t.isDoltFormat)
return index.NewIndexReaderBuilder(ctx, t.DoltTable, t.idx, key, t.DoltTable.projectedCols, t.DoltTable.sqlSch)
}
return t.lb, nil
}
@@ -151,7 +146,7 @@ func (t *WritableIndexedDoltTable) LookupPartitions(ctx *sql.Context, lookup sql
if lookup.VectorOrderAndLimit.OrderBy != nil {
return index.NewVectorPartitionIter(lookup)
}
return index.NewRangePartitionIter(ctx, t.DoltTable, lookup, t.isDoltFormat)
return index.NewRangePartitionIter(ctx, t.DoltTable, lookup)
}
func (t *WritableIndexedDoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
@@ -166,7 +161,7 @@ func (t *WritableIndexedDoltTable) PartitionRows(ctx *sql.Context, part sql.Part
return nil, err
}
if t.lb == nil || !canCache || t.lb.Key() != key {
t.lb, err = index.NewIndexReaderBuilder(ctx, t.DoltTable, t.idx, key, t.projectedCols, t.sqlSch, t.isDoltFormat)
t.lb, err = index.NewIndexReaderBuilder(ctx, t.DoltTable, t.idx, key, t.projectedCols, t.sqlSch)
if err != nil {
return nil, err
}
go/libraries/doltcore/sqle/integration_test/stockmarket_test.go
-20271
View File
File diff suppressed because it is too large Load Diff
go/libraries/doltcore/sqle/kvexec/builder.go
+2 -2
View File
@@ -397,7 +397,7 @@ func getSourceKv(ctx *sql.Context, n sql.Node, isSrc bool) (prolly.Map, prolly.M
return prolly.Map{}, nil, nil, nil, nil, nil, err
}
if rowData.Format() != types.Format_DOLT {
return prolly.Map{}, nil, nil, nil, nil, nil, nil
panic("Unsupported index format in lookup join: " + rowData.Format().VersionString())
}
priMap, err = durable.ProllyMapFromIndex(rowData)
if err != nil {
@@ -568,7 +568,7 @@ func getMergeKv(ctx *sql.Context, n sql.Node) (mergeState, error) {
}
if idx.Format() != types.Format_DOLT {
return ms, nil
panic("Unsupported index format in merge join: " + idx.Format().VersionString())
}
secIdx, err := index.GetDurableIndex(ctx, doltTable, idx)
go/libraries/doltcore/sqle/rows.go
+2 -172
View File
@@ -23,8 +23,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/libraries/doltcore/table"
"github.com/dolthub/dolt/go/store/types"
)
@@ -58,13 +56,6 @@ func (k keylessRowIter) Close(ctx *sql.Context) error {
return k.keyedIter.Close(ctx)
}
// An iterator over the rows of a table.
type doltTableRowIter struct {
sql.RowIter
ctx context.Context
reader table.SqlTableReader
}
// Returns a new row iterator for the table given
func newRowIterator(ctx context.Context, tbl *doltdb.Table, projCols []uint64, partition doltTablePartition) (sql.RowIter, error) {
sch, err := tbl.GetSchema(ctx)
@@ -73,106 +64,8 @@ func newRowIterator(ctx context.Context, tbl *doltdb.Table, projCols []uint64, p
return nil, err
}
if types.IsFormat_DOLT(tbl.Format()) {
return ProllyRowIterFromPartition(ctx, sch, projCols, partition)
}
mapIter, err := iterForPartition(ctx, partition)
if err != nil {
return nil, err
}
if schema.IsKeyless(sch) {
// would be more optimal to project columns into keyless tables also
return newKeylessRowIterator(ctx, tbl, projCols, mapIter)
} else {
return newKeyedRowIter(ctx, tbl, projCols, mapIter)
}
}
func newKeylessRowIterator(ctx context.Context, tbl *doltdb.Table, projectedCols []uint64, mapIter types.MapTupleIterator) (sql.RowIter, error) {
cols, tagToSqlColIdx, err := getTagToResColIdx(ctx, tbl, projectedCols)
if err != nil {
return nil, err
}
idxOfCardinality := len(cols)
tagToSqlColIdx[schema.KeylessRowCardinalityTag] = idxOfCardinality
colsCopy := make([]schema.Column, len(cols), len(cols)+1)
copy(colsCopy, cols)
colsCopy = append(colsCopy, schema.NewColumn("__cardinality__", schema.KeylessRowCardinalityTag, types.UintKind, false))
conv := index.NewKVToSqlRowConverter(tbl.Format(), tagToSqlColIdx, colsCopy, len(colsCopy))
keyedItr, err := index.NewDoltMapIter(mapIter.NextTuple, nil, conv), nil
if err != nil {
return nil, err
}
return &keylessRowIter{
keyedIter: keyedItr,
cardIdx: idxOfCardinality,
nonCardCols: len(cols),
}, nil
}
func newKeyedRowIter(ctx context.Context, tbl *doltdb.Table, projectedCols []uint64, mapIter types.MapTupleIterator) (sql.RowIter, error) {
cols, tagToSqlColIdx, err := getTagToResColIdx(ctx, tbl, projectedCols)
if err != nil {
return nil, err
}
conv := index.NewKVToSqlRowConverter(tbl.Format(), tagToSqlColIdx, cols, len(cols))
return index.NewDoltMapIter(mapIter.NextTuple, nil, conv), nil
}
func iterForPartition(ctx context.Context, partition doltTablePartition) (types.MapTupleIterator, error) {
if partition.end == NoUpperBound {
c, err := partition.rowData.Count()
if err != nil {
return nil, err
}
partition.end = c
}
return partition.IteratorForPartition(ctx, partition.rowData)
}
func getTagToResColIdx(ctx context.Context, tbl *doltdb.Table, projectedCols []uint64) ([]schema.Column, map[uint64]int, error) {
sch, err := tbl.GetSchema(ctx)
if err != nil {
return nil, nil, err
}
allCols := sch.GetAllCols().GetColumns()
tagToSqlColIdx := make(map[uint64]int)
if projectedCols != nil {
outCols := make([]schema.Column, len(projectedCols))
for i := range projectedCols {
t := projectedCols[i]
idx := sch.GetAllCols().TagToIdx[t]
tagToSqlColIdx[t] = i
outCols[i] = allCols[idx]
}
return outCols, tagToSqlColIdx, nil
}
for i, col := range allCols {
tagToSqlColIdx[col.Tag] = i
}
return allCols, tagToSqlColIdx, nil
}
// Next returns the next row in this row iterator, or an io.EOF error if there aren't any more.
func (itr *doltTableRowIter) Next() (sql.Row, error) {
return itr.reader.ReadSqlRow(itr.ctx)
}
// Close required by sql.RowIter interface
func (itr *doltTableRowIter) Close(*sql.Context) error {
return nil
types.AssertFormat_DOLT(tbl.Format())
return ProllyRowIterFromPartition(ctx, sch, projCols, partition)
}
func ProllyRowIterFromPartition(
@@ -227,66 +120,3 @@ func SqlTableToRowIter(ctx *sql.Context, table *DoltTable, columns []uint64) (sq
return newRowIterator(ctx, t, columns, p)
}
// DoltTablePartitionToRowIter returns a sql.RowIter for a partition of the clustered index of |table|.
func DoltTablePartitionToRowIter(ctx *sql.Context, name string, table *doltdb.Table, start uint64, end uint64) (sql.Schema, sql.RowIter, error) {
sch, err := table.GetSchema(ctx)
if err != nil {
return nil, nil, err
}
pkSch, err := sqlutil.FromDoltSchema("", name, sch)
if err != nil {
return nil, nil, err
}
data, err := table.GetRowData(ctx)
if err != nil {
return nil, nil, err
}
if types.IsFormat_DOLT(data.Format()) {
idx, err := durable.ProllyMapFromIndex(data)
if err != nil {
return nil, nil, err
}
c, err := idx.Count()
if err != nil {
return nil, nil, err
}
if end > uint64(c) {
end = uint64(c)
}
iter, err := idx.IterOrdinalRange(ctx, start, end)
if err != nil {
return nil, nil, err
}
rowIter := index.NewProllyRowIterForMap(sch, idx, iter, nil)
if err != nil {
return nil, nil, err
}
return pkSch.Schema, rowIter, nil
}
idx := durable.NomsMapFromIndex(data)
iterAt, err := idx.IteratorAt(ctx, start)
if err != nil {
return nil, nil, err
}
iter := types.NewLimitingMapIterator(iterAt, end-start)
var rowIter sql.RowIter
if schema.IsKeyless(sch) {
rowIter, err = newKeylessRowIterator(ctx, table, nil, iter)
if err != nil {
return nil, nil, err
}
} else {
rowIter, err = newKeyedRowIter(ctx, table, nil, iter)
if err != nil {
return nil, nil, err
}
}
return pkSch.Schema, rowIter, nil
}
go/libraries/doltcore/sqle/sqlddl_test.go
+9 -450
View File
@@ -32,12 +32,10 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/gcctx"
"github.com/dolthub/dolt/go/libraries/doltcore/dtestutils"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer"
"github.com/dolthub/dolt/go/store/types"
)
func TestCreateTable(t *testing.T) {
@@ -154,15 +152,15 @@ func TestCreateTable(t *testing.T) {
schemaNewColumn(t, "c10", 1965, gmstypes.TinyText, false),
schemaNewColumn(t, "c11", 12860, gmstypes.MediumText, false),
schemaNewColumn(t, "c12", 7155, gmstypes.LongText, false),
//schemaNewColumn(t, "c13", 113, sql.TinyBlob, false),
//schemaNewColumn(t, "c14", 114, sql.Blob, false),
//schemaNewColumn(t, "c15", 115, sql.LongBlob, false),
// schemaNewColumn(t, "c13", 113, sql.TinyBlob, false),
// schemaNewColumn(t, "c14", 114, sql.Blob, false),
// schemaNewColumn(t, "c15", 115, sql.LongBlob, false),
schemaNewColumn(t, "c16", 15859, gmstypes.MustCreateStringWithDefaults(sqltypes.Char, 5), false),
schemaNewColumn(t, "c17", 11710, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 255), false),
schemaNewColumn(t, "c18", 6838, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false),
schemaNewColumn(t, "c19", 9377, gmstypes.Float32, false),
schemaNewColumn(t, "c20", 15979, gmstypes.Float64, false),
//schemaNewColumn(t, "c21", 121, sql.MustCreateDecimalType(10, 5), false),
// schemaNewColumn(t, "c21", 121, sql.MustCreateDecimalType(10, 5), false),
schemaNewColumn(t, "c22", 2910, gmstypes.Uint32, false),
schemaNewColumn(t, "c23", 8740, gmstypes.Uint8, false),
schemaNewColumn(t, "c24", 8689, gmstypes.Uint16, false),
@@ -366,445 +364,6 @@ func TestDropTable(t *testing.T) {
}
}
func TestAddColumn(t *testing.T) {
tests := []struct {
expectedSchema schema.Schema
name string
query string
expectedErr string
expectedRows []row.Row
}{
{
name: "alter add string column no default",
query: "alter table people add (newColumn varchar(80))",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumn(t, "newColumn", 4208, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false)),
expectedRows: addColToRows(t, AllPeopleRows, 4208, nil),
},
{
name: "alter add float column without default",
query: "alter table people add (newColumn float)",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumn(t, "newColumn", 4208, gmstypes.Float32, false)),
expectedRows: addColToRows(t, AllPeopleRows, 4208, nil),
},
{
name: "alter add uint column without default",
query: "alter table people add (newColumn bigint unsigned)",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumn(t, "newColumn", 4208, gmstypes.Uint64, false)),
expectedRows: addColToRows(t, AllPeopleRows, 4208, nil),
},
{
name: "alter add int column default",
query: "alter table people add (newColumn int default 2)",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumnWDefVal(t, "newColumn", 2803, gmstypes.Int32, false, "2")),
expectedRows: addColToRows(t, AllPeopleRows, 2803, types.Int(int32(2))),
},
{
name: "alter add uint column default",
query: "alter table people add (newColumn bigint unsigned default 20)",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumnWDefVal(t, "newColumn", 517, gmstypes.Uint64, false, "20")),
expectedRows: addColToRows(t, AllPeopleRows, 517, types.Uint(uint64(20))),
},
{
name: "alter add string column with default",
query: "alter table people add (newColumn varchar(80) default 'hi')",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumnWDefVal(t, "newColumn", 13690, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false, `'hi'`)),
expectedRows: addColToRows(t, AllPeopleRows, 13690, types.String("hi")),
},
{
name: "alter add column first",
query: "alter table people add newColumn varchar(80) first",
expectedSchema: dtestutils.CreateSchema(
schemaNewColumn(t, "newColumn", 4208, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false),
schema.NewColumn("id", IdTag, types.IntKind, true, schema.NotNullConstraint{}),
schema.NewColumn("first_name", FirstNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("last_name", LastNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("is_married", IsMarriedTag, types.IntKind, false),
schema.NewColumn("age", AgeTag, types.IntKind, false),
schema.NewColumn("rating", RatingTag, types.FloatKind, false),
schema.NewColumn("uuid", UuidTag, types.StringKind, false),
schema.NewColumn("num_episodes", NumEpisodesTag, types.UintKind, false),
),
expectedRows: addColToRows(t, AllPeopleRows, 4208, nil),
},
{
name: "alter add column middle",
query: "alter table people add newColumn varchar(80) after last_name",
expectedSchema: dtestutils.CreateSchema(
schema.NewColumn("id", IdTag, types.IntKind, true, schema.NotNullConstraint{}),
schema.NewColumn("first_name", FirstNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("last_name", LastNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schemaNewColumn(t, "newColumn", 4208, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false),
schema.NewColumn("is_married", IsMarriedTag, types.IntKind, false),
schema.NewColumn("age", AgeTag, types.IntKind, false),
schema.NewColumn("rating", RatingTag, types.FloatKind, false),
schema.NewColumn("uuid", UuidTag, types.StringKind, false),
schema.NewColumn("num_episodes", NumEpisodesTag, types.UintKind, false),
),
expectedRows: addColToRows(t, AllPeopleRows, 4208, nil),
},
{
name: "alter add column not null",
query: "alter table people add (newColumn varchar(80) not null default 'default')",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumnWDefVal(t, "newColumn", 13690, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false, `'default'`, schema.NotNullConstraint{})),
expectedRows: addColToRows(t, AllPeopleRows, 13690, types.String("default")),
},
{
name: "alter add column not null with expression default",
query: "alter table people add (newColumn int not null default (2+2/2))",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumnWDefVal(t, "newColumn", 2803, gmstypes.Int32, false, "((2 + (2 / 2)))", schema.NotNullConstraint{})),
expectedRows: addColToRows(t, AllPeopleRows, 2803, types.Int(3)),
},
{
name: "alter add column not null with negative expression",
query: "alter table people add (newColumn float not null default -1.1)",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumnWDefVal(t, "newColumn", 12469, gmstypes.Float32, false, "-1.1", schema.NotNullConstraint{})),
expectedRows: addColToRows(t, AllPeopleRows, 12469, types.Float(float32(-1.1))),
},
{
name: "alter add column not null with type mismatch in default",
query: "alter table people add (newColumn float not null default 'not a number')",
expectedErr: "incompatible type",
},
{
name: "alter add column column not found",
query: "alter table people add column newColumn float after notFound",
expectedErr: `table "people" does not have column "notFound"`,
},
{
name: "alter add column table not found",
query: "alter table notFound add column newColumn float",
expectedErr: "table not found: notFound",
},
{
name: "alter add column not null without default",
query: "alter table people add (newColumn varchar(80) not null)",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumnWDefVal(t, "newColumn", 13690, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false, "", schema.NotNullConstraint{})),
expectedRows: addColToRows(t, AllPeopleRows, 13690, types.String("")),
},
{
name: "alter add column nullable",
query: "alter table people add (newColumn bigint)",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumn(t, "newColumn", 4435, gmstypes.Int64, false)),
expectedRows: AllPeopleRows,
},
{
name: "alter add column with optional column keyword",
query: "alter table people add column (newColumn varchar(80))",
expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
schemaNewColumn(t, "newColumn", 4208, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false)),
expectedRows: AllPeopleRows,
},
{
name: "alter table add column name clash",
query: "alter table people add column(age int)",
expectedErr: `Column "age" already exists`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := context.Background()
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB(ctx).Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
updatedRoot, err := ExecuteSql(ctx, dEnv, root, tt.query)
if tt.expectedErr == "" {
require.NoError(t, err)
} else {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.expectedErr)
return
}
assert.NotNil(t, updatedRoot)
table, _, err := updatedRoot.GetTable(ctx, doltdb.TableName{Name: PeopleTableName})
assert.NoError(t, err)
sch, err := table.GetSchema(ctx)
assert.NoError(t, err)
equalSchemas(t, tt.expectedSchema, sch)
if types.Format_Default != types.Format_LD_1 {
return // todo: convert these to enginetests
}
updatedTable, ok, err := updatedRoot.GetTable(ctx, doltdb.TableName{Name: "people"})
assert.NoError(t, err)
require.True(t, ok)
rowData, err := updatedTable.GetNomsRowData(ctx)
assert.NoError(t, err)
var foundRows []row.Row
err = rowData.Iter(ctx, func(key, value types.Value) (stop bool, err error) {
r, err := row.FromNoms(tt.expectedSchema, key.(types.Tuple), value.(types.Tuple))
assert.NoError(t, err)
foundRows = append(foundRows, r)
return false, nil
})
assert.NoError(t, err)
assert.Equal(t, tt.expectedRows, foundRows)
})
}
}
func TestRenameColumn(t *testing.T) {
tests := []struct {
expectedSchema schema.Schema
name string
query string
expectedErr string
expectedRows []row.Row
}{
{
name: "alter rename column with column and as keywords",
query: "alter table people rename column rating as newRating",
expectedSchema: dtestutils.CreateSchema(
schema.NewColumn("id", IdTag, types.IntKind, true, schema.NotNullConstraint{}),
schema.NewColumn("first_name", FirstNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("last_name", LastNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("is_married", IsMarriedTag, types.IntKind, false),
schema.NewColumn("age", AgeTag, types.IntKind, false),
schema.NewColumn("newRating", RatingTag, types.FloatKind, false),
schema.NewColumn("uuid", UuidTag, types.StringKind, false),
schema.NewColumn("num_episodes", NumEpisodesTag, types.UintKind, false),
),
expectedRows: AllPeopleRows,
},
{
name: "alter rename column with column and to keyword",
query: "alter table people rename column rating to newRating",
expectedSchema: dtestutils.CreateSchema(
schema.NewColumn("id", IdTag, types.IntKind, true, schema.NotNullConstraint{}),
schema.NewColumn("first_name", FirstNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("last_name", LastNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("is_married", IsMarriedTag, types.IntKind, false),
schema.NewColumn("age", AgeTag, types.IntKind, false),
schema.NewColumn("newRating", RatingTag, types.FloatKind, false),
schema.NewColumn("uuid", UuidTag, types.StringKind, false),
schema.NewColumn("num_episodes", NumEpisodesTag, types.UintKind, false),
),
expectedRows: AllPeopleRows,
},
{
name: "alter rename primary key column",
query: "alter table people rename column id to newId",
expectedSchema: dtestutils.CreateSchema(
schema.NewColumn("newId", IdTag, types.IntKind, true, schema.NotNullConstraint{}),
schema.NewColumn("first_name", FirstNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("last_name", LastNameTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("is_married", IsMarriedTag, types.IntKind, false),
schema.NewColumn("age", AgeTag, types.IntKind, false),
schema.NewColumn("rating", RatingTag, types.FloatKind, false),
schema.NewColumn("uuid", UuidTag, types.StringKind, false),
schema.NewColumn("num_episodes", NumEpisodesTag, types.UintKind, false),
),
expectedRows: AllPeopleRows,
},
{
name: "table not found",
query: "alter table notFound rename column id to newId",
expectedErr: "table not found: notFound",
},
{
name: "column not found",
query: "alter table people rename column notFound to newNotFound",
expectedErr: `table "people" does not have column "notFound"`,
},
{
name: "column name collision",
query: "alter table people rename column id to AGE",
expectedErr: "Column \"AGE\" already exists",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := context.Background()
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB(ctx).Close()
root, _ := dEnv.WorkingRoot(ctx)
updatedRoot, err := ExecuteSql(ctx, dEnv, root, tt.query)
if tt.expectedErr == "" {
require.NoError(t, err)
} else {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.expectedErr)
return
}
require.NotNil(t, updatedRoot)
table, _, err := updatedRoot.GetTable(ctx, doltdb.TableName{Name: PeopleTableName})
assert.NoError(t, err)
sch, err := table.GetSchema(ctx)
require.NoError(t, err)
assert.Equal(t, tt.expectedSchema, sch)
if types.Format_Default != types.Format_LD_1 {
return // todo: convert these to enginetests
}
updatedTable, ok, err := updatedRoot.GetTable(ctx, doltdb.TableName{Name: "people"})
assert.NoError(t, err)
require.True(t, ok)
rowData, err := updatedTable.GetNomsRowData(ctx)
assert.NoError(t, err)
var foundRows []row.Row
err = rowData.Iter(ctx, func(key, value types.Value) (stop bool, err error) {
updatedSch, err := updatedTable.GetSchema(ctx)
assert.NoError(t, err)
r, err := row.FromNoms(updatedSch, key.(types.Tuple), value.(types.Tuple))
assert.NoError(t, err)
foundRows = append(foundRows, r)
return false, nil
})
assert.NoError(t, err)
assert.Equal(t, tt.expectedRows, foundRows)
})
}
}
func TestRenameTableStatements(t *testing.T) {
tests := []struct {
expectedSchema schema.Schema
name string
query string
oldTableName string
newTableName string
expectedErr string
expectedRows []row.Row
}{
{
name: "alter rename table",
query: "rename table people to newPeople",
oldTableName: "people",
newTableName: "newPeople",
expectedSchema: PeopleTestSchema,
expectedRows: AllPeopleRows,
},
{
name: "alter rename table with alter syntax",
query: "alter table people rename to newPeople",
oldTableName: "people",
newTableName: "newPeople",
expectedSchema: PeopleTestSchema,
expectedRows: AllPeopleRows,
},
{
name: "rename multiple tables",
query: "rename table people to newPeople, appearances to newAppearances",
oldTableName: "appearances",
newTableName: "newAppearances",
expectedSchema: AppearancesTestSchema,
expectedRows: AllAppsRows,
},
{
name: "alter rename table with alter syntax",
query: "alter table people rename to 123People",
oldTableName: "people",
newTableName: "123People",
expectedSchema: PeopleTestSchema,
expectedRows: AllPeopleRows,
},
{
name: "table not found",
query: "rename table notFound to newNowFound",
expectedErr: "table not found: notFound",
},
{
name: "invalid table name",
query: "rename table people to `a!trailing^space*is%the(worst) `",
expectedErr: "Invalid table name",
},
{
name: "reserved table name",
query: "rename table people to dolt_table",
expectedErr: "Invalid table name",
},
{
name: "table name in use",
query: "rename table people to appearances",
expectedErr: "already exists",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := context.Background()
dEnv, err := CreateTestDatabase()
require.NoError(t, err)
defer dEnv.DoltDB(ctx).Close()
root, err := dEnv.WorkingRoot(ctx)
require.NoError(t, err)
updatedRoot, err := ExecuteSql(ctx, dEnv, root, tt.query)
if len(tt.expectedErr) > 0 {
require.Error(t, err)
assert.Contains(t, err.Error(), tt.expectedErr)
return
} else {
require.NoError(t, err)
}
require.NotNil(t, updatedRoot)
has, err := updatedRoot.HasTable(ctx, doltdb.TableName{Name: tt.oldTableName})
require.NoError(t, err)
assert.False(t, has)
newTable, ok, err := updatedRoot.GetTable(ctx, doltdb.TableName{Name: tt.newTableName})
require.NoError(t, err)
require.True(t, ok)
sch, err := newTable.GetSchema(ctx)
require.NoError(t, err)
require.Equal(t, tt.expectedSchema, sch)
if types.Format_Default != types.Format_LD_1 {
return // todo: convert these to enginetests
}
rowData, err := newTable.GetNomsRowData(ctx)
require.NoError(t, err)
var foundRows []row.Row
err = rowData.Iter(ctx, func(key, value types.Value) (stop bool, err error) {
r, err := row.FromNoms(tt.expectedSchema, key.(types.Tuple), value.(types.Tuple))
require.NoError(t, err)
foundRows = append(foundRows, r)
return false, nil
})
require.NoError(t, err)
// Some test cases deal with rows declared in a different order than noms returns them, so use an order-
// insensitive comparison here.
assert.ElementsMatch(t, tt.expectedRows, foundRows)
})
}
}
func TestAlterSystemTables(t *testing.T) {
systemTableNames := []string{"dolt_log", "dolt_history_people", "dolt_diff_people", "dolt_commit_diff_people", "dolt_schemas"}
reservedTableNames := []string{"dolt_query_catalog", "dolt_docs", "dolt_procedures", "dolt_ignore"}
@@ -967,15 +526,15 @@ func TestParseCreateTableStatement(t *testing.T) {
schemaNewColumn(t, "c10", 1965, gmstypes.TinyText, false),
schemaNewColumn(t, "c11", 12860, gmstypes.MediumText, false),
schemaNewColumn(t, "c12", 7155, gmstypes.LongText, false),
//schemaNewColumn(t, "c13", 113, sql.TinyBlob, false),
//schemaNewColumn(t, "c14", 114, sql.Blob, false),
//schemaNewColumn(t, "c15", 115, sql.LongBlob, false),
// schemaNewColumn(t, "c13", 113, sql.TinyBlob, false),
// schemaNewColumn(t, "c14", 114, sql.Blob, false),
// schemaNewColumn(t, "c15", 115, sql.LongBlob, false),
schemaNewColumn(t, "c16", 15859, gmstypes.MustCreateStringWithDefaults(sqltypes.Char, 5), false),
schemaNewColumn(t, "c17", 11710, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 255), false),
schemaNewColumn(t, "c18", 6838, gmstypes.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false),
schemaNewColumn(t, "c19", 9377, gmstypes.Float32, false),
schemaNewColumn(t, "c20", 15979, gmstypes.Float64, false),
//schemaNewColumn(t, "c21", 121, sql.MustCreateDecimalType(10, 5), false),
// schemaNewColumn(t, "c21", 121, sql.MustCreateDecimalType(10, 5), false),
schemaNewColumn(t, "c22", 2910, gmstypes.Uint32, false),
schemaNewColumn(t, "c23", 8740, gmstypes.Uint8, false),
schemaNewColumn(t, "c24", 8689, gmstypes.Uint16, false),
@@ -1078,7 +637,7 @@ func TestParseCreateTableStatement(t *testing.T) {
dEnv := dtestutils.CreateTestEnv()
defer dEnv.DoltDB(ctx).Close()
root, _ := dEnv.WorkingRoot(ctx)
//eng, dbName, _ := engine.NewSqlEngineForEnv(ctx, dEnv)
// eng, dbName, _ := engine.NewSqlEngineForEnv(ctx, dEnv)
eng, sqlCtx := newTestEngine(ctx, dEnv)
_, iter, _, err := eng.Query(sqlCtx, "create database test")
go/libraries/doltcore/sqle/sqlselect_test.go
+2 -181
View File
@@ -25,10 +25,8 @@ import (
"github.com/stretchr/testify/require"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
"github.com/dolthub/dolt/go/libraries/doltcore/dtestutils"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
@@ -37,7 +35,7 @@ import (
)
// Set to the name of a single test to run just that test, useful for debugging
const singleSelectQueryTest = "" //"Natural join with join clause"
const singleSelectQueryTest = "" // "Natural join with join clause"
// Set to false to run tests known to be broken
const skipBrokenSelect = true
@@ -1270,18 +1268,6 @@ func TestSelect(t *testing.T) {
}
}
func TestAsOfQueries(t *testing.T) {
if types.Format_Default != types.Format_LD_1 {
t.Skip("") // todo: convert to enginetests
}
for _, test := range AsOfTests {
t.Run(test.Name, func(t *testing.T) {
// AS OF queries use the same history as the diff tests, so exercise the same test setup
testSelectDiffQuery(t, test)
})
}
}
func TestJoins(t *testing.T) {
for _, tt := range JoinTests {
if tt.Name == "Join from table with two key columns to table with one key column" {
@@ -1379,7 +1365,7 @@ func testSelectQuery(t *testing.T, test SelectTest) {
expectedRows := unwrapRows(t, test.ExpectedRows)
actualRows = unwrapRows(t, actualRows)
// JSON columns must be compared using like so
assert.Equal(t, len(expectedRows), len(actualRows))
require.Equal(t, len(expectedRows), len(actualRows))
for i := 0; i < len(expectedRows); i++ {
assert.Equal(t, len(expectedRows[i]), len(actualRows[i]))
for j := 0; j < len(expectedRows[i]); j++ {
@@ -1548,63 +1534,6 @@ var DiffSchema = dtestutils.MustSchema(
schema.NewColumn("diff_type", 14, types.StringKind, false),
)
func testSelectDiffQuery(t *testing.T, test SelectTest) {
validateTest(t, test)
ctx := context.Background()
cleanup := installTestCommitClock()
defer cleanup()
dEnv := dtestutils.CreateTestEnv()
initializeWithHistory(t, ctx, dEnv, CreateHistory(ctx, dEnv, t)...)
if test.AdditionalSetup != nil {
test.AdditionalSetup(t, dEnv)
}
cs, err := doltdb.NewCommitSpec("main")
require.NoError(t, err)
optCmt, err := dEnv.DoltDB(ctx).Resolve(ctx, cs, nil)
require.NoError(t, err)
cm, ok := optCmt.ToCommit()
require.True(t, ok)
root, err := cm.GetRootValue(ctx)
require.NoError(t, err)
err = dEnv.UpdateStagedRoot(ctx, root)
require.NoError(t, err)
err = dEnv.UpdateWorkingRoot(ctx, root)
require.NoError(t, err)
root, err = dEnv.WorkingRoot(context.Background())
require.NoError(t, err)
root = updateTables(t, ctx, root, createWorkingRootUpdate())
err = dEnv.UpdateWorkingRoot(ctx, root)
require.NoError(t, err)
actualRows, sch, err := executeSelect(t, ctx, dEnv, root, test.Query)
if len(test.ExpectedErr) > 0 {
require.Error(t, err)
return
} else {
require.NoError(t, err)
}
assert.Equal(t, test.ExpectedRows, actualRows)
var sqlSchema sql.Schema
if test.ExpectedSqlSchema != nil {
sqlSchema = test.ExpectedSqlSchema
} else {
sqlSchema = mustSqlSchema(test.ExpectedSchema)
}
assertSchemasEqual(t, sqlSchema, sch)
}
// TODO: this shouldn't be here
func createWorkingRootUpdate() map[string]TableUpdate {
return map[string]TableUpdate{
@@ -1618,114 +1547,6 @@ func createWorkingRootUpdate() map[string]TableUpdate {
}
}
func updateTables(t *testing.T, ctx context.Context, root doltdb.RootValue, tblUpdates map[string]TableUpdate) doltdb.RootValue {
for tblName, updates := range tblUpdates {
tbl, ok, err := root.GetTable(ctx, doltdb.TableName{Name: tblName})
require.NoError(t, err)
var sch schema.Schema
if updates.NewSch != nil {
sch = updates.NewSch
} else {
sch, err = tbl.GetSchema(ctx)
require.NoError(t, err)
}
var rowData types.Map
if updates.NewRowData == nil {
if ok {
rowData, err = tbl.GetNomsRowData(ctx)
require.NoError(t, err)
} else {
rowData, err = types.NewMap(ctx, root.VRW())
require.NoError(t, err)
}
} else {
rowData = *updates.NewRowData
}
if updates.RowUpdates != nil {
me := rowData.Edit()
for _, r := range updates.RowUpdates {
me = me.Set(r.NomsMapKey(sch), r.NomsMapValue(sch))
}
rowData, err = me.Map(ctx)
require.NoError(t, err)
}
var indexData durable.IndexSet
require.NoError(t, err)
if tbl != nil {
indexData, err = tbl.GetIndexSet(ctx)
require.NoError(t, err)
}
tbl, err = doltdb.NewNomsTable(ctx, root.VRW(), root.NodeStore(), sch, rowData, indexData, nil)
require.NoError(t, err)
root, err = root.PutTable(ctx, doltdb.TableName{Name: tblName}, tbl)
require.NoError(t, err)
}
return root
}
// initializeWithHistory will go through the provided historyNodes and create the intended commit graph
func initializeWithHistory(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, historyNodes ...HistoryNode) {
for _, node := range historyNodes {
cs, err := doltdb.NewCommitSpec(env.DefaultInitBranch)
require.NoError(t, err)
optCmt, err := dEnv.DoltDB(ctx).Resolve(ctx, cs, nil)
require.NoError(t, err)
cm, ok := optCmt.ToCommit()
require.True(t, ok)
processNode(t, ctx, dEnv, node, cm)
}
}
func processNode(t *testing.T, ctx context.Context, dEnv *env.DoltEnv, node HistoryNode, parent *doltdb.Commit) {
branchRef := ref.NewBranchRef(node.Branch)
ok, err := dEnv.DoltDB(ctx).HasRef(ctx, branchRef)
require.NoError(t, err)
if !ok {
err = dEnv.DoltDB(ctx).NewBranchAtCommit(ctx, branchRef, parent, nil)
require.NoError(t, err)
}
cs, err := doltdb.NewCommitSpec(branchRef.String())
require.NoError(t, err)
optCmt, err := dEnv.DoltDB(ctx).Resolve(ctx, cs, nil)
require.NoError(t, err)
cm, ok := optCmt.ToCommit()
require.True(t, ok)
root, err := cm.GetRootValue(ctx)
require.NoError(t, err)
root = updateTables(t, ctx, root, node.Updates)
r, h, err := dEnv.DoltDB(ctx).WriteRootValue(ctx, root)
require.NoError(t, err)
root = r
meta, err := datas.NewCommitMeta("Ash Ketchum", "ash@poke.mon", node.CommitMsg)
require.NoError(t, err)
cm, err = dEnv.DoltDB(ctx).Commit(ctx, h, branchRef, meta)
require.NoError(t, err)
for _, child := range node.Children {
processNode(t, ctx, dEnv, child, cm)
}
}
func validateTest(t *testing.T, test SelectTest) {
if (test.ExpectedRows == nil) != (test.ExpectedSchema == nil && test.ExpectedSqlSchema == nil) {
require.Fail(t, "Incorrect test setup: schema and rows must both be provided if one is")
go/libraries/doltcore/sqle/table_editor_fk_test.go
+30
View File
@@ -890,3 +890,33 @@ ALTER TABLE three ADD FOREIGN KEY (v1, v2) REFERENCES two(v1, v2) ON DELETE CASC
})
}
}
func convertSqlRowToInt64(sqlRows []sql.Row) []sql.Row {
if sqlRows == nil {
return nil
}
newSqlRows := make([]sql.Row, len(sqlRows))
for i, sqlRow := range sqlRows {
newSqlRow := make(sql.Row, len(sqlRow))
for j := range sqlRow {
switch v := sqlRow[j].(type) {
case int:
newSqlRow[j] = int64(v)
case int8:
newSqlRow[j] = int64(v)
case int16:
newSqlRow[j] = int64(v)
case int32:
newSqlRow[j] = int64(v)
case int64:
newSqlRow[j] = v
case nil:
newSqlRow[j] = nil
default:
return sqlRows
}
}
newSqlRows[i] = newSqlRow
}
return newSqlRows
}
go/libraries/doltcore/sqle/table_editor_index_test.go
-386
View File
@@ -1,386 +0,0 @@
// Copyright 2020 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqle
import (
"context"
"strings"
"testing"
"github.com/dolthub/go-mysql-server/sql"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/dtestutils"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
"github.com/dolthub/dolt/go/store/types"
)
func setupEditorIndexTest(ctx context.Context, t *testing.T) (*env.DoltEnv, doltdb.RootValue) {
index_dEnv := dtestutils.CreateTestEnv()
root, err := index_dEnv.WorkingRoot(context.Background())
require.NoError(t, err)
index_initialRoot, err := ExecuteSql(ctx, index_dEnv, root, `
CREATE TABLE onepk (
pk1 BIGINT PRIMARY KEY,
v1 BIGINT,
v2 BIGINT
);
CREATE TABLE twopk (
pk1 BIGINT,
pk2 BIGINT,
v1 BIGINT,
v2 BIGINT,
PRIMARY KEY (pk1,pk2)
);
CREATE TABLE oneuni (
pk1 BIGINT PRIMARY KEY,
v1 BIGINT,
v2 BIGINT
);
CREATE TABLE twouni (
pk1 BIGINT,
pk2 BIGINT,
v1 BIGINT,
v2 BIGINT,
PRIMARY KEY (pk1,pk2)
);
CREATE INDEX idx_v1 ON onepk(v1);
CREATE INDEX idx_v2v1 ON twopk(v2, v1);
CREATE UNIQUE INDEX idx_v1 ON oneuni(v1);
CREATE UNIQUE INDEX idx_v1v2 ON twouni(v1, v2);
`)
require.NoError(t, err)
return index_dEnv, index_initialRoot
}
func TestTableEditorIndexResults(t *testing.T) {
tests := []struct {
sqlStatement string
expectedIdxv1 []sql.Row
expectedIdxv2v1 []sql.Row
}{
{
`
INSERT INTO onepk VALUES (1, 2, 3), (4, 5, 6), (7, 8, 9);
INSERT INTO onepk VALUES (3, 2, 1), (6, 5, 4), (9, 8, 7);
`,
[]sql.Row{{2, 1}, {2, 3}, {5, 4}, {5, 6}, {8, 7}, {8, 9}},
[]sql.Row{},
},
{
`
INSERT INTO onepk VALUES (1, 11, 111), (2, 22, 222), (3, 33, 333);
UPDATE onepk SET v1 = v1 - 1 WHERE pk1 > 1;
REPLACE INTO onepk VALUES (3, 55, 555), (4, 44, 444);
`,
[]sql.Row{{11, 1}, {21, 2}, {44, 4}, {55, 3}},
[]sql.Row{},
},
{
`
INSERT INTO onepk VALUES (1, 11, 111), (2, 22, 222), (3, 33, 333);
INSERT INTO twopk VALUES (4, 44, 444, 4444);
REPLACE INTO twopk VALUES (4, 44, 111, 4444), (5, 55, 222, 5555), (6, 66, 333, 6666);
DELETE FROM onepk WHERE v2 = 222;
`,
[]sql.Row{{11, 1}, {33, 3}},
[]sql.Row{{4444, 111, 4, 44}, {5555, 222, 5, 55}, {6666, 333, 6, 66}},
},
{
`
INSERT INTO onepk VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3);
DELETE FROM onepk WHERE pk1 % 2 = 1;
REPLACE INTO onepk VALUES (3, 6, 2), (-1, 4, -3);
UPDATE onepk SET pk1 = v1 + pk1 ORDER BY pk1 DESC;
`,
[]sql.Row{{2, 4}, {4, 3}, {6, 9}},
[]sql.Row{},
},
}
for _, test := range tests {
t.Run(test.sqlStatement, func(t *testing.T) {
ctx := context.Background()
dEnv, initialRoot := setupEditorIndexTest(ctx, t)
defer dEnv.DoltDB(ctx).Close()
root := initialRoot
for _, sqlStatement := range strings.Split(test.sqlStatement, ";") {
var err error
root, err = executeModify(t, context.Background(), dEnv, root, sqlStatement)
require.NoError(t, err)
}
onepk, ok, err := root.GetTable(context.Background(), doltdb.TableName{Name: "onepk"})
require.NoError(t, err)
require.True(t, ok)
twopk, ok, err := root.GetTable(context.Background(), doltdb.TableName{Name: "twopk"})
require.NoError(t, err)
require.True(t, ok)
onepkSch, err := onepk.GetSchema(context.Background())
require.NoError(t, err)
twopkSch, err := twopk.GetSchema(context.Background())
require.NoError(t, err)
idx_v1 := onepkSch.Indexes().GetByName("idx_v1")
require.NotNil(t, idx_v1)
idx_v2v1 := twopkSch.Indexes().GetByName("idx_v2v1")
require.NotNil(t, idx_v2v1)
if types.Format_Default != types.Format_LD_1 {
t.Skip("need a prolly sql row iter")
}
idx_v1RowData, err := onepk.GetNomsIndexRowData(context.Background(), idx_v1.Name())
require.NoError(t, err)
idx_v2v1RowData, err := twopk.GetNomsIndexRowData(context.Background(), idx_v2v1.Name())
require.NoError(t, err)
if assert.Equal(t, uint64(len(test.expectedIdxv1)), idx_v1RowData.Len()) && len(test.expectedIdxv1) > 0 {
var sqlRows []sql.Row
_ = idx_v1RowData.IterAll(context.Background(), func(key, value types.Value) error {
r, err := row.FromNoms(idx_v1.Schema(), key.(types.Tuple), value.(types.Tuple))
assert.NoError(t, err)
sqlRow, err := sqlutil.DoltRowToSqlRow(r, idx_v1.Schema())
assert.NoError(t, err)
sqlRows = append(sqlRows, sqlRow)
return nil
})
assert.ElementsMatch(t, convertSqlRowToInt64(test.expectedIdxv1), sqlRows)
}
if assert.Equal(t, uint64(len(test.expectedIdxv2v1)), idx_v2v1RowData.Len()) && len(test.expectedIdxv2v1) > 0 {
var sqlRows []sql.Row
_ = idx_v2v1RowData.IterAll(context.Background(), func(key, value types.Value) error {
r, err := row.FromNoms(idx_v2v1.Schema(), key.(types.Tuple), value.(types.Tuple))
assert.NoError(t, err)
sqlRow, err := sqlutil.DoltRowToSqlRow(r, idx_v2v1.Schema())
assert.NoError(t, err)
sqlRows = append(sqlRows, sqlRow)
return nil
})
assert.ElementsMatch(t, convertSqlRowToInt64(test.expectedIdxv2v1), sqlRows)
}
})
}
}
func TestTableEditorUniqueIndexResults(t *testing.T) {
tests := []struct {
sqlStatement string
expectedIdxv1 []sql.Row
expectedIdxv1v2 []sql.Row
expectedErr bool
}{
{
`
INSERT INTO oneuni VALUES (1, 3, 2), (4, 6, 5), (7, 9, 8);
INSERT INTO oneuni VALUES (3, 1, 2), (6, 4, 5), (9, 7, 8);
`,
[]sql.Row{{1, 3}, {3, 1}, {4, 6}, {6, 4}, {7, 9}, {9, 7}},
[]sql.Row{},
false,
},
{
`
INSERT INTO oneuni VALUES (1, 11, 111), (2, 22, 222), (3, 33, 333);
UPDATE oneuni SET v1 = v1 - 1 WHERE pk1 > 1;
REPLACE INTO oneuni VALUES (3, 55, 555), (4, 44, 444);
`,
[]sql.Row{{11, 1}, {21, 2}, {44, 4}, {55, 3}},
[]sql.Row{},
false,
},
{
`
INSERT INTO oneuni VALUES (1, 11, 111), (2, 22, 222), (3, 33, 333);
REPLACE INTO oneuni VALUES (1, 11, 444);
INSERT INTO twouni VALUES (4, 44, 444, 4444);
REPLACE INTO twouni VALUES (4, 44, 111, 4444), (5, 55, 222, 5555), (6, 66, 333, 6666);
DELETE FROM oneuni WHERE v1 = 22;
`,
[]sql.Row{{11, 1}, {33, 3}},
[]sql.Row{{111, 4444, 4, 44}, {222, 5555, 5, 55}, {333, 6666, 6, 66}},
false,
},
{
`
INSERT INTO oneuni VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3);
DELETE FROM oneuni WHERE pk1 % 2 = 1;
REPLACE INTO oneuni VALUES (3, 6, 2), (-1, 4, -3);
UPDATE oneuni SET pk1 = v1 + v2;
`,
[]sql.Row{{2, 4}, {4, 1}, {6, 8}},
[]sql.Row{},
false,
},
{
`
INSERT INTO oneuni VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3);
DELETE FROM oneuni WHERE v1 < 3;
REPLACE INTO oneuni VALUES (4, 2, 2), (5, 3, 3), (3, 1, 1);
`,
[]sql.Row{{1, 3}, {2, 4}, {3, 5}},
[]sql.Row{},
false,
},
{
`
INSERT INTO oneuni VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3);
DELETE FROM oneuni WHERE v1 < 3;
REPLACE INTO oneuni VALUES (4, 2, 2), (5, 2, 3), (3, 1, 1);
`,
[]sql.Row{{1, 3}, {2, 5}},
[]sql.Row{},
false,
},
{
`
INSERT INTO oneuni VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3);
REPLACE INTO oneuni VALUES (1, 1, 1), (2, 2, 2), (3, 2, 3);
`,
[]sql.Row{{1, 1}, {2, 3}},
[]sql.Row{},
false,
},
{
`
INSERT INTO oneuni VALUES (1, 1, 1), (2, 1, 2), (3, 3, 3);
`,
[]sql.Row{},
[]sql.Row{},
true,
},
{
`
INSERT INTO oneuni VALUES (1, 2, 3), (2, 1, 4);
UPDATE oneuni SET v1 = v1 + pk1;
`,
[]sql.Row{},
[]sql.Row{},
true,
},
}
for _, test := range tests {
t.Run(test.sqlStatement, func(t *testing.T) {
ctx := context.Background()
dEnv, initialRoot := setupEditorIndexTest(ctx, t)
defer dEnv.DoltDB(ctx).Close()
root := initialRoot
var err error
for _, sqlStatement := range strings.Split(test.sqlStatement, ";") {
root, err = executeModify(t, context.Background(), dEnv, root, sqlStatement)
if err != nil {
break
}
}
if test.expectedErr {
require.Error(t, err)
return
}
require.NoError(t, err)
oneuni, ok, err := root.GetTable(context.Background(), doltdb.TableName{Name: "oneuni"})
require.NoError(t, err)
require.True(t, ok)
twouni, ok, err := root.GetTable(context.Background(), doltdb.TableName{Name: "twouni"})
require.NoError(t, err)
require.True(t, ok)
oneuniSch, err := oneuni.GetSchema(context.Background())
require.NoError(t, err)
twouniSch, err := twouni.GetSchema(context.Background())
require.NoError(t, err)
idx_v1 := oneuniSch.Indexes().GetByName("idx_v1")
require.NotNil(t, idx_v1)
idx_v1v2 := twouniSch.Indexes().GetByName("idx_v1v2")
require.NotNil(t, idx_v1v2)
if types.Format_Default != types.Format_LD_1 {
t.Skip("need a prolly sql row iter")
}
idx_v1RowData, err := oneuni.GetNomsIndexRowData(context.Background(), idx_v1.Name())
require.NoError(t, err)
idx_v1v2RowData, err := twouni.GetNomsIndexRowData(context.Background(), idx_v1v2.Name())
require.NoError(t, err)
if assert.Equal(t, uint64(len(test.expectedIdxv1)), idx_v1RowData.Len()) && len(test.expectedIdxv1) > 0 {
var sqlRows []sql.Row
_ = idx_v1RowData.IterAll(context.Background(), func(key, value types.Value) error {
r, err := row.FromNoms(idx_v1.Schema(), key.(types.Tuple), value.(types.Tuple))
assert.NoError(t, err)
sqlRow, err := sqlutil.DoltRowToSqlRow(r, idx_v1.Schema())
assert.NoError(t, err)
sqlRows = append(sqlRows, sqlRow)
return nil
})
assert.ElementsMatch(t, convertSqlRowToInt64(test.expectedIdxv1), sqlRows)
}
if assert.Equal(t, uint64(len(test.expectedIdxv1v2)), idx_v1v2RowData.Len()) && len(test.expectedIdxv1v2) > 0 {
var sqlRows []sql.Row
_ = idx_v1v2RowData.IterAll(context.Background(), func(key, value types.Value) error {
r, err := row.FromNoms(idx_v1v2.Schema(), key.(types.Tuple), value.(types.Tuple))
assert.NoError(t, err)
sqlRow, err := sqlutil.DoltRowToSqlRow(r, idx_v1v2.Schema())
assert.NoError(t, err)
sqlRows = append(sqlRows, sqlRow)
return nil
})
assert.ElementsMatch(t, convertSqlRowToInt64(test.expectedIdxv1v2), sqlRows)
}
})
}
}
func convertSqlRowToInt64(sqlRows []sql.Row) []sql.Row {
if sqlRows == nil {
return nil
}
newSqlRows := make([]sql.Row, len(sqlRows))
for i, sqlRow := range sqlRows {
newSqlRow := make(sql.Row, len(sqlRow))
for j := range sqlRow {
switch v := sqlRow[j].(type) {
case int:
newSqlRow[j] = int64(v)
case int8:
newSqlRow[j] = int64(v)
case int16:
newSqlRow[j] = int64(v)
case int32:
newSqlRow[j] = int64(v)
case int64:
newSqlRow[j] = v
case nil:
newSqlRow[j] = nil
default:
return sqlRows
}
}
newSqlRows[i] = newSqlRow
}
return newSqlRows
}
go/libraries/doltcore/sqle/tables.go
+8 -105
View File
@@ -1090,40 +1090,12 @@ func (t *WritableDoltTable) truncate(
}
func copyConstraintViolationsAndConflicts(ctx context.Context, from, to *doltdb.Table) (*doltdb.Table, error) {
if !types.IsFormat_DOLT(to.Format()) {
if has, err := from.HasConflicts(ctx); err != nil {
return nil, err
} else if has {
confSch, conf, err := from.GetConflicts(ctx)
if err != nil {
return nil, err
}
to, err = to.SetConflicts(ctx, confSch, conf)
if err != nil {
return nil, err
}
}
viols, err := from.GetConstraintViolations(ctx)
if err != nil {
return nil, err
}
to, err = to.SetConstraintViolations(ctx, viols)
if err != nil {
return nil, err
}
} else {
arts, err := from.GetArtifacts(ctx)
if err != nil {
return nil, err
}
to, err = to.SetArtifacts(ctx, arts)
if err != nil {
return nil, err
}
types.AssertFormat_DOLT(to.Format())
arts, err := from.GetArtifacts(ctx)
if err != nil {
return nil, err
}
return to, nil
return to.SetArtifacts(ctx, arts)
}
// Updater implements sql.UpdatableTable
@@ -1264,7 +1236,7 @@ func (t *DoltTable) GetDeclaredForeignKeys(ctx *sql.Context) ([]sql.ForeignKeyCo
for i, fk := range declaredFks {
if len(fk.UnresolvedFKDetails.TableColumns) > 0 && len(fk.UnresolvedFKDetails.ReferencedTableColumns) > 0 {
//TODO: implement multi-db support for foreign keys
// TODO: implement multi-db support for foreign keys
toReturn[i] = sql.ForeignKeyConstraint{
Name: fk.Name,
Database: t.db.Name(),
@@ -1496,14 +1468,6 @@ func (p doltTablePartition) Key() []byte {
return []byte(strconv.FormatUint(p.start, 10) + " >= i < " + strconv.FormatUint(p.end, 10))
}
// IteratorForPartition returns a types.MapIterator implementation which will iterate through the values
// for index = start; index < end. This iterator is not thread safe and should only be used from a single go routine
// unless paired with a mutex
func (p doltTablePartition) IteratorForPartition(ctx context.Context, idx durable.Index) (types.MapTupleIterator, error) {
m := durable.NomsMapFromIndex(idx)
return m.RangeIterator(ctx, p.start, p.end)
}
// AlterableDoltTable allows altering the schema of the table. It implements sql.AlterableTable.
type AlterableDoltTable struct {
WritableDoltTable
@@ -2189,66 +2153,6 @@ func (t *AlterableDoltTable) DropColumn(*sql.Context, string) error {
return fmt.Errorf("not implemented: AlterableDoltTable.DropColumn()")
}
// dropColumnData drops values for the specified column from the underlying storage layer
func (t *AlterableDoltTable) dropColumnData(ctx *sql.Context, updatedTable *doltdb.Table, sch schema.Schema, columnName string) (*doltdb.Table, error) {
nomsRowData, err := updatedTable.GetNomsRowData(ctx)
if err != nil {
return nil, err
}
column, ok := sch.GetAllCols().GetByName(columnName)
if !ok {
return nil, sql.ErrColumnNotFound.New(columnName)
}
mapEditor := nomsRowData.Edit()
defer mapEditor.Close(ctx)
err = nomsRowData.Iter(ctx, func(key, value types.Value) (stop bool, err error) {
if t, ok := value.(types.Tuple); ok {
newTuple, err := types.NewTuple(nomsRowData.Format())
if err != nil {
return true, err
}
idx := uint64(0)
for idx < t.Len() {
tTag, err := t.Get(idx)
if err != nil {
return true, err
}
tValue, err := t.Get(idx + 1)
if err != nil {
return true, err
}
if tTag.Equals(types.Uint(column.Tag)) == false {
newTuple, err = newTuple.Append(tTag, tValue)
if err != nil {
return true, err
}
}
idx += 2
}
mapEditor.Set(key, newTuple)
}
return false, nil
})
if err != nil {
return nil, err
}
newMapData, err := mapEditor.Map(ctx)
if err != nil {
return nil, err
}
return updatedTable.UpdateNomsRows(ctx, newMapData)
}
// ModifyColumn implements sql.AlterableTable. ModifyColumn operations are only used for operations that change only
// the schema of a table, not the data. For those operations, |RewriteInserter| is used.
func (t *AlterableDoltTable) ModifyColumn(ctx *sql.Context, columnName string, column *sql.Column, order *sql.ColumnOrder) error {
@@ -2512,9 +2416,8 @@ func (t *AlterableDoltTable) RenameIndex(ctx *sql.Context, fromIndexName string,
// CreateFulltextIndex implements fulltext.IndexAlterableTable
func (t *AlterableDoltTable) CreateFulltextIndex(ctx *sql.Context, idx sql.IndexDef, keyCols fulltext.KeyColumns, tableNames fulltext.IndexTableNames) error {
if !types.IsFormat_DOLT(t.Format()) {
return fmt.Errorf("FULLTEXT is not supported on storage format %s. Run `dolt migrate` to upgrade to the latest storage format.", t.Format().VersionString())
}
types.AssertFormat_DOLT(t.Format())
if err := dsess.CheckAccessForDb(ctx, t.db, branch_control.Permissions_Write); err != nil {
return err
}
go/libraries/doltcore/sqle/testutil.go
+1 -14
View File
@@ -551,20 +551,7 @@ func SqlRowsFromDurableIndex(idx durable.Index, sch schema.Schema) ([]sql.Row, e
}
} else {
// types.Format_LD_1
rowData := durable.NomsMapFromIndex(idx)
_ = rowData.IterAll(ctx, func(key, value types.Value) error {
r, err := row.FromNoms(sch, key.(types.Tuple), value.(types.Tuple))
if err != nil {
return err
}
sqlRow, err := sqlutil.DoltRowToSqlRow(r, sch)
if err != nil {
return err
}
sqlRows = append(sqlRows, sqlRow)
return nil
})
panic("Unsupported format: " + idx.Format().VersionString())
}
return sqlRows, nil
}
go/libraries/doltcore/table/editor/index_editor_test.go
-115
View File
@@ -19,38 +19,20 @@ import (
"sync"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
const (
// The number of rows we expect the test to end up with
indexEditorConcurrencyFinalCount = 100
idTag = 0
firstTag = 1
lastTag = 2
isMarriedTag = 3
ageTag = 4
emptyTag = 5
testSchemaIndexName = "idx_name"
testSchemaIndexAge = "idx_age"
)
var id0, _ = uuid.NewRandom()
var id1, _ = uuid.NewRandom()
var id2, _ = uuid.NewRandom()
var id3, _ = uuid.NewRandom()
func TestIndexEditorConcurrency(t *testing.T) {
format := types.Format_LD_1
_, vrw, _, err := dbfactory.MemFactory{}.CreateDB(context.Background(), format, nil, nil)
@@ -451,100 +433,3 @@ func TestIndexEditorCapacityExceeded(t *testing.T) {
_, err = indexEditor.Map(ctx)
require.Contains(t, err.Error(), "unrecoverable state")
}
func createTestRowData(t *testing.T, vrw types.ValueReadWriter, sch schema.Schema) (types.Map, []row.Row) {
return createTestRowDataFromTaggedValues(t, vrw, sch,
row.TaggedValues{
idTag: types.InlineBlob(id0[:]), firstTag: types.String("bill"), lastTag: types.String("billerson"), ageTag: types.Uint(53)},
row.TaggedValues{
idTag: types.InlineBlob(id1[:]), firstTag: types.String("eric"), lastTag: types.String("ericson"), isMarriedTag: types.Int(1), ageTag: types.Uint(21)},
row.TaggedValues{
idTag: types.InlineBlob(id2[:]), firstTag: types.String("john"), lastTag: types.String("johnson"), isMarriedTag: types.Int(0), ageTag: types.Uint(53)},
row.TaggedValues{
idTag: types.InlineBlob(id3[:]), firstTag: types.String("robert"), lastTag: types.String("robertson"), ageTag: types.Uint(36)},
)
}
func createUpdatedTestRowData(t *testing.T, vrw types.ValueReadWriter, sch schema.Schema) (types.Map, []row.Row) {
return createTestRowDataFromTaggedValues(t, vrw, sch,
row.TaggedValues{
idTag: types.InlineBlob(id0[:]), firstTag: types.String("jack"), lastTag: types.String("space"), ageTag: types.Uint(20)},
row.TaggedValues{
idTag: types.InlineBlob(id1[:]), firstTag: types.String("rick"), lastTag: types.String("drive"), isMarriedTag: types.Int(0), ageTag: types.Uint(21)},
row.TaggedValues{
idTag: types.InlineBlob(id2[:]), firstTag: types.String("tyler"), lastTag: types.String("eat"), isMarriedTag: types.Int(1), ageTag: types.Uint(22)},
row.TaggedValues{
idTag: types.InlineBlob(id3[:]), firstTag: types.String("moore"), lastTag: types.String("walk"), ageTag: types.Uint(23)},
)
}
func createTestRowDataFromTaggedValues(t *testing.T, vrw types.ValueReadWriter, sch schema.Schema, vals ...row.TaggedValues) (types.Map, []row.Row) {
var err error
rows := make([]row.Row, len(vals))
m, err := types.NewMap(context.Background(), vrw)
assert.NoError(t, err)
ed := m.Edit()
for i, val := range vals {
r, err := row.New(types.Format_LD_1, sch, val)
require.NoError(t, err)
rows[i] = r
ed = ed.Set(r.NomsMapKey(sch), r.NomsMapValue(sch))
}
m, err = ed.Map(context.Background())
assert.NoError(t, err)
return m, rows
}
func createTestSchema(t *testing.T) schema.Schema {
colColl := schema.NewColCollection(
schema.NewColumn("id", idTag, types.InlineBlobKind, true, schema.NotNullConstraint{}),
schema.NewColumn("first", firstTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("last", lastTag, types.StringKind, false, schema.NotNullConstraint{}),
schema.NewColumn("is_married", isMarriedTag, types.IntKind, false),
schema.NewColumn("age", ageTag, types.UintKind, false),
schema.NewColumn("empty", emptyTag, types.IntKind, false),
)
sch, err := schema.SchemaFromCols(colColl)
require.NoError(t, err)
_, err = sch.Indexes().AddIndexByColTags(testSchemaIndexName, []uint64{firstTag, lastTag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""})
require.NoError(t, err)
_, err = sch.Indexes().AddIndexByColTags(testSchemaIndexAge, []uint64{ageTag}, nil, schema.IndexProperties{IsUnique: false, Comment: ""})
require.NoError(t, err)
return sch
}
func createTableWithoutIndexRebuilding(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeStore, sch schema.Schema, rowData types.Map) (*doltdb.Table, error) {
return doltdb.NewNomsTable(ctx, vrw, ns, sch, rowData, nil, nil)
}
func rowsToIndexRows(t *testing.T, rows []row.Row, indexName schema.Index, indexAge schema.Index) (indexNameExpectedRows []row.Row, indexAgeExpectedRows []row.Row) {
indexNameExpectedRows = make([]row.Row, len(rows))
indexAgeExpectedRows = make([]row.Row, len(rows))
indexNameSch := indexName.Schema()
indexAgeSch := indexAge.Schema()
var err error
for i, r := range rows {
indexNameKey := make(row.TaggedValues)
for _, tag := range indexName.AllTags() {
val, ok := r.GetColVal(tag)
require.True(t, ok)
indexNameKey[tag] = val
}
indexNameExpectedRows[i], err = row.New(types.Format_LD_1, indexNameSch, indexNameKey)
require.NoError(t, err)
indexAgeKey := make(row.TaggedValues)
for _, tag := range indexAge.AllTags() {
val, ok := r.GetColVal(tag)
require.True(t, ok)
indexAgeKey[tag] = val
}
indexAgeExpectedRows[i], err = row.New(types.Format_LD_1, indexAgeSch, indexAgeKey)
require.NoError(t, err)
}
return
}
go/libraries/doltcore/table/table_iterator.go
+2 -25
View File
@@ -60,36 +60,13 @@ func (i rowIterImpl) Close(ctx context.Context) error {
// NewTableIterator creates a RowIter that iterates sql.Row's from |idx|.
// |offset| can be supplied to read at some start point in |idx|.
func NewTableIterator(ctx context.Context, sch schema.Schema, idx durable.Index) (RowIter, error) {
var rowItr sql.RowIter
if types.IsFormat_DOLT(idx.Format()) {
m := durable.MapFromIndex(idx)
itr, err := m.IterAll(ctx)
if err != nil {
return nil, err
}
rowItr = index.NewProllyRowIterForMap(sch, m, itr, nil)
if err != nil {
return nil, err
}
} else {
noms := durable.NomsMapFromIndex(idx)
itr, err := noms.IteratorAt(ctx, 0)
if err != nil {
return nil, err
}
conv := makeNomsConverter(idx.Format(), sch)
rowItr = index.NewDoltMapIter(itr.NextTuple, nil, conv)
return NewRowIter(index.NewProllyRowIterForMap(sch, m, itr, nil)), nil
}
return NewRowIter(rowItr), nil
}
// makeNomsConverter creates a *index.KVToSqlRowConverter.
func makeNomsConverter(nbf *types.NomsBinFormat, sch schema.Schema) *index.KVToSqlRowConverter {
cols := sch.GetAllCols().GetColumns()
tagToSqlColIdx := make(map[uint64]int)
for i, col := range cols {
tagToSqlColIdx[col.Tag] = i
}
return index.NewKVToSqlRowConverter(nbf, tagToSqlColIdx, cols, len(cols))
panic("Unsupported format: " + idx.Format().VersionString())
}
go/store/types/format.go
+6 -2
View File
@@ -16,6 +16,7 @@ package types
import (
"errors"
"fmt"
"sync"
"github.com/dolthub/dolt/go/store/constants"
@@ -58,8 +59,11 @@ func IsFormat_DOLT(nbf *NomsBinFormat) bool {
return nbf.tag == formatTag_DOLT
}
func IsFormat_LD(nbf *NomsBinFormat) bool {
return nbf.tag == formatTag_LD_1
// AssertFormat_DOLT panics if the provided NomsBinFormat is not compatible with the DOLT format.
func AssertFormat_DOLT(nbf *NomsBinFormat) {
if !IsFormat_DOLT(nbf) {
panic(fmt.Sprintf("Unsupported binary format %s, please migrate database to newer format", nbf.VersionString()))
}
}
func GetFormatForVersionString(s string) (*NomsBinFormat, error) {
integration-tests/bats/no-repo.bats
-7
View File
@@ -54,7 +54,6 @@ teardown() {
[[ "$output" =~ "tag - Create, list, delete tags." ]] || false
[[ "$output" =~ "blame - Show what revision and author last modified each row of a table." ]] || false
[[ "$output" =~ "constraints - Commands for handling constraints." ]] || false
[[ "$output" =~ "migrate - Executes a database migration to use the latest Dolt data format." ]] || false
[[ "$output" =~ "read-tables - Fetch table(s) at a specific commit into a new dolt repo" ]] || false
[[ "$output" =~ "gc - Cleans up unreferenced data from the repository." ]] || false
[[ "$output" =~ "filter-branch - Edits the commit history using the provided query." ]] || false
@@ -343,12 +342,6 @@ NOT_VALID_REPO_ERROR="The current directory is not a valid dolt repository."
[[ "$output" =~ "resolve -" ]] || false
}
@test "no-repo: dolt migrate outside of a dolt repository" {
run dolt migrate
[ "$status" -ne 0 ]
[ "${lines[0]}" = "$NOT_VALID_REPO_ERROR" ]
}
@test "no-repo: initializing a dolt repository" {
mkdir dolt-repo-$$-new
cd dolt-repo-$$-new