mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-01-23 10:27:34 -06:00
Code Issues Packages Projects Releases 123 Wiki Activity

first pass at table renames

Browse Source
This commit is contained in:
Andy Arthur
2020-06-02 11:54:32 -05:00
parent 8d5d6057a5
commit 0da8e893bd
6 changed files with 343 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
bats/sql-diff.bats
View File

@@ -200,7 +200,7 @@ SQL
dolt diff --sql newbranch firstbranch > query
dolt checkout firstbranch
dolt sql < query
rm query
cat query
dolt add test
dolt commit -m "Reconciled with newbranch"
@@ -476,9 +476,7 @@ SQL
dolt diff --sql newbranch firstbranch > query
dolt checkout firstbranch
skip "add + drop doesn't work, we have to track renames"
dolt sql < query
rm query
dolt add .
dolt commit -m "Reconciled with newbranch"
@@ -486,6 +484,7 @@ SQL
run dolt diff --sql newbranch firstbranch
[ "$status" -eq 0 ]
[ "$output" = "" ]
grep 'RENAME' query
}
@test "diff sql recreates tables with all types" {

159
go/cmd/dolt/commands/diff.go
View File

@@ -17,6 +17,7 @@ package commands
import (
"context"
"fmt"
"github.com/liquidata-inc/dolt/go/libraries/utils/set"
"reflect"
"strconv"
"strings"
@@ -103,6 +104,7 @@ In order to filter which diffs are displayed {{.EmphasisLeft}}--where key=value{
type diffArgs struct {
diffParts diffPart
diffOutput diffOutput
tableSet *set.StrSet
limit int
where string
}
@@ -182,7 +184,7 @@ func (cmd DiffCmd) Exec(ctx context.Context, commandStr string, args []string, d
if verr == nil {
whereClause := apr.GetValueOrDefault(whereParam, "")
verr = diffRoots(ctx, r1, r2, tables, docs, dEnv, &diffArgs{diffParts, diffOutput, limit, whereClause})
verr = diffRoots(ctx, r1, r2, docs, dEnv, &diffArgs{diffParts, diffOutput, set.NewStrSet(tables), limit, whereClause})
}
if verr != nil {
@@ -296,64 +298,56 @@ func getRootForCommitSpecStr(ctx context.Context, csStr string, dEnv *env.DoltEn
return h.String(), r, nil
}
func diffRoots(ctx context.Context, r1, r2 *doltdb.RootValue, tblNames []string, docDetails []doltdb.DocDetails, dEnv *env.DoltEnv, dArgs *diffArgs) errhand.VerboseError {
func diffRoots(ctx context.Context, r1, r2 *doltdb.RootValue, docDetails []doltdb.DocDetails, dEnv *env.DoltEnv, dArgs *diffArgs) errhand.VerboseError {
var err error
if len(tblNames) == 0 {
tblNames, err = doltdb.UnionTableNames(ctx, r1, r2)
if dArgs.tableSet.Size() == 0 {
utn, err := doltdb.UnionTableNames(ctx, r1, r2)
if err != nil {
return errhand.BuildDError("error: failed to get table names").AddCause(err).Build()
}
dArgs.tableSet.Add(utn...)
}
// todo: root ordering?
tableDeltas, err := diff.GetTableDeltas(ctx, r1, r2)
if err != nil {
return errhand.BuildDError("error: unable to read tables").AddCause(err).Build()
return errhand.BuildDError("error: unable to diff tables").AddCause(err).Build()
}
if dArgs.diffOutput == SQLDiffOutput {
err = diff.PrintSqlTableDiffs(ctx, r1, r2, iohelp.NopWrCloser(cli.CliOut))
if err != nil {
return errhand.BuildDError("error: unable to diff tables").AddCause(err).Build()
}
}
for _, tblName := range tblNames {
tbl1, ok1, err := r1.GetTable(ctx, tblName)
for _, td := range tableDeltas {
if err != nil {
return errhand.BuildDError("error: failed to get table '%s'", tblName).AddCause(err).Build()
// todo: match tableSet by newName, oldName or both?
if !dArgs.tableSet.Contains(td.NewName) {
continue
}
tbl2, ok2, err := r2.GetTable(ctx, tblName)
// todo: old/new vs to/from
tblName := td.NewName
tbl1 := td.OldTable
tbl2 := td.NewTable
if err != nil {
return errhand.BuildDError("error: failed to get table '%s'", tblName).AddCause(err).Build()
}
if !ok1 && !ok2 {
bdr := errhand.BuildDError("Table could not be found.")
bdr.AddDetails("The table %s does not exist.", tblName)
cli.PrintErrln(bdr.Build())
} else if tbl1 != nil && tbl2 != nil {
h1, err := tbl1.HashOf()
if err != nil {
return errhand.BuildDError("error: failed to get table hash").Build()
}
h2, err := tbl2.HashOf()
if err != nil {
return errhand.BuildDError("error: failed to get table hash").Build()
}
if h1 == h2 {
continue
}
if tbl1 == nil && tbl2 == nil {
return errhand.BuildDError("error: both tables in tableDelta are nil").Build()
}
if dArgs.diffOutput == TabularDiffOutput {
printTableDiffSummary(ctx, dEnv, tblName, tbl1, tbl2, docDetails)
// if we're in standard output mode, follow Git convention
// and don't print data diffs for added/dropped tables
if td.IsDrop() || td.IsAdd() {
continue
}
}
if tbl1 == nil || tbl2 == nil || tblName == doltdb.DocTableName {
if tblName == doltdb.DocTableName {
continue
}
@@ -361,58 +355,49 @@ func diffRoots(ctx context.Context, r1, r2 *doltdb.RootValue, tblNames []string,
var sch2 schema.Schema
var sch1Hash hash.Hash
var sch2Hash hash.Hash
rowData1, err := types.NewMap(ctx, dEnv.DoltDB.ValueReadWriter())
rowData1 := types.EmptyMap
rowData2 := types.EmptyMap
if err != nil {
return errhand.BuildDError("").AddCause(err).Build()
}
rowData2, err := types.NewMap(ctx, dEnv.DoltDB.ValueReadWriter())
if err != nil {
return errhand.BuildDError("").AddCause(err).Build()
}
if ok1 {
if tbl1 != nil {
sch1, err = tbl1.GetSchema(ctx)
if err != nil {
return errhand.BuildDError("error: failed to get schema").AddCause(err).Build()
}
schRef, err := tbl1.GetSchemaRef()
if err != nil {
return errhand.BuildDError("error: failed to get schema ref").AddCause(err).Build()
}
sch1Hash = schRef.TargetHash()
rowData1, err = tbl1.GetRowData(ctx)
if err != nil {
return errhand.BuildDError("error: failed to get row data").AddCause(err).Build()
}
}
if ok2 {
if tbl2 != nil {
sch2, err = tbl2.GetSchema(ctx)
if err != nil {
return errhand.BuildDError("error: failed to get schema").AddCause(err).Build()
}
schRef, err := tbl2.GetSchemaRef()
if tbl1 == nil {
sch1 = sch2
}
schRef, err := tbl2.GetSchemaRef()
if err != nil {
return errhand.BuildDError("error: failed to get schema ref").AddCause(err).Build()
}
sch2Hash = schRef.TargetHash()
rowData2, err = tbl2.GetRowData(ctx)
if err != nil {
return errhand.BuildDError("error: failed to get row data").AddCause(err).Build()
}
} else {
sch2 = sch1
}
var verr errhand.VerboseError
@@ -423,7 +408,7 @@ func diffRoots(ctx context.Context, r1, r2 *doltdb.RootValue, tblNames []string,
}
if dArgs.diffParts&SchemaOnlyDiff != 0 && sch1Hash != sch2Hash {
verr = diffSchemas(tblName, sch2, sch1, dArgs)
verr = diffSchemas(ctx, td, dArgs)
}
if dArgs.diffParts&DataOnlyDiff != 0 {
@@ -438,18 +423,25 @@ func diffRoots(ctx context.Context, r1, r2 *doltdb.RootValue, tblNames []string,
return nil
}
func diffSchemas(tableName string, sch1 schema.Schema, sch2 schema.Schema, dArgs *diffArgs) errhand.VerboseError {
diffs, unionTags := diff.DiffSchemas(sch1, sch2)
func diffSchemas(ctx context.Context, td diff.TableDelta, dArgs *diffArgs) errhand.VerboseError {
if dArgs.diffOutput == TabularDiffOutput {
if verr := tabularSchemaDiff(tableName, unionTags, diffs); verr != nil {
return verr
if td.IsDrop() || td.IsAdd() {
panic("cannot perform tabular schema diff for added/dropped tables")
}
} else {
sqlSchemaDiff(tableName, unionTags, diffs)
sch1, sch2, err := td.GetSchemas(ctx)
if err != nil {
return errhand.BuildDError("cannot retrieve schema for table %s", td.NewName).AddCause(err).Build()
}
diffs, unionTags := diff.DiffSchemas(sch1, sch2)
return tabularSchemaDiff(td.NewName, unionTags, diffs)
}
return nil
return sqlSchemaDiff(ctx, td)
}
func tabularSchemaDiff(tableName string, tags []uint64, diffs map[uint64]diff.SchemaDifference) errhand.VerboseError {
@@ -530,19 +522,38 @@ func tabularSchemaDiff(tableName string, tags []uint64, diffs map[uint64]diff.Sc
return nil
}
func sqlSchemaDiff(tableName string, tags []uint64, diffs map[uint64]diff.SchemaDifference) {
for _, tag := range tags {
dff := diffs[tag]
switch dff.DiffType {
case diff.SchDiffNone:
case diff.SchDiffColAdded:
cli.Println(sqlfmt.AlterTableAddColStmt(tableName, sqlfmt.FmtColWithTag(0, 0, 0, *dff.New)))
case diff.SchDiffColRemoved:
cli.Print(sqlfmt.AlterTableDropColStmt(tableName, dff.Old.Name))
case diff.SchDiffColModified:
cli.Print(sqlfmt.AlterTableRenameColStmt(tableName, dff.Old.Name, dff.New.Name))
func sqlSchemaDiff(ctx context.Context, td diff.TableDelta) errhand.VerboseError{
sch1, sch2, err := td.GetSchemas(ctx)
if err != nil {
return errhand.BuildDError("cannot retrieve schema for table %s", td.NewName).AddCause(err).Build()
}
if sch1 == nil {
cli.Println(sqlfmt.DropTableStmt(td.OldName))
} else if sch2 == nil {
cli.Println(sqlfmt.CreateTableStmtWithTags(td.NewName, sch1))
} else {
if td.OldName != td.NewName {
cli.Println(sqlfmt.RenameTableStmt(td.OldName, td.NewName))
}
colDiffs, unionTags := diff.DiffSchemas(sch1, sch2)
for _, tag := range unionTags {
cd := colDiffs[tag]
switch cd.DiffType {
case diff.SchDiffNone:
case diff.SchDiffColAdded:
cli.Println(sqlfmt.AlterTableAddColStmt(td.NewName, sqlfmt.FmtCol(0, 0, 0, *cd.New)))
case diff.SchDiffColRemoved:
cli.Print(sqlfmt.AlterTableDropColStmt(td.NewName, cd.Old.Name))
case diff.SchDiffColModified:
cli.Print(sqlfmt.AlterTableRenameColStmt(td.NewName, cd.Old.Name, cd.New.Name))
}
}
}
return nil
}
func dumbDownSchema(in schema.Schema) (schema.Schema, error) {

2
go/libraries/doltcore/diff/diff_splitter.go
View File

@@ -23,7 +23,7 @@ import (
)
const (
// DiffTypeProp is the name of a property added to each split row which tells if its added, removed, the modified
// DiffTypeProp is the name of a property added to each split row which tells if its added, dropped, the modified
// old value, or the new value after modification
DiffTypeProp = "difftype"

226
go/libraries/doltcore/diff/diffs.go
View File

@@ -16,6 +16,8 @@ package diff
import (
"context"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/schema"
"github.com/liquidata-inc/dolt/go/store/hash"
"sort"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/doltdb"
@@ -28,6 +30,7 @@ const (
AddedTable TableDiffType = iota
ModifiedTable
RemovedTable
RenamedTable
)
type TableDiffs struct {
@@ -36,6 +39,9 @@ type TableDiffs struct {
NumRemoved int
TableToType map[string]TableDiffType
Tables []string
// renamed tables are included in TableToType by their new name
NewNameToOldName map[string]string
}
type DocDiffType int
@@ -89,32 +95,45 @@ func (rvu RootValueUnreadable) Error() string {
}
func NewTableDiffs(ctx context.Context, newer, older *doltdb.RootValue) (*TableDiffs, error) {
added, modified, removed, err := newer.TableDiff(ctx, older)
matches, err := matchTablesForRoots(ctx, newer, older)
if err != nil {
return nil, err
}
var tbls []string
tbls = append(tbls, added...)
tbls = append(tbls, modified...)
tbls = append(tbls, removed...)
sort.Strings(tbls)
tbls = append(tbls, matches.added..., )
tbls = append(tbls, matches.modified...)
tbls = append(tbls, matches.dropped...)
tblToType := make(map[string]TableDiffType)
for _, tbl := range added {
for _, tbl := range matches.added {
tblToType[tbl] = AddedTable
}
for _, tbl := range modified {
for _, tbl := range matches.modified {
tblToType[tbl] = ModifiedTable
}
for _, tbl := range removed {
for _, tbl := range matches.dropped {
tblToType[tbl] = RemovedTable
}
return &TableDiffs{len(added), len(modified), len(removed), tblToType, tbls}, err
for newName, _ := range matches.renamed {
tblToType[newName] = RenamedTable
tbls = append(tbls, newName)
}
sort.Strings(tbls)
return &TableDiffs{
NumAdded: len(matches.added),
NumModified: len(matches.modified),
NumRemoved: len(matches.dropped),
TableToType: tblToType,
Tables: tbls,
NewNameToOldName: matches.renamed,
}, err
}
func (td *TableDiffs) Len() int {
@@ -238,3 +257,192 @@ func GetDocDiffs(ctx context.Context, dEnv *env.DoltEnv) (*DocDiffs, *DocDiffs,
return stagedDocDiffs, notStagedDocDiffs, nil
}
type tableMatches struct {
added []string
dropped []string
modified []string
unchanged []string
renamed map[string]string
}
// matchTablesForRoots matches all tables that exist in both roots and finds the tables that only exist in one root.
// Tables are matched by the column tag of the first primary key column.
func matchTablesForRoots(ctx context.Context, newer, older *doltdb.RootValue) (tableMatches, error) {
tm := tableMatches{}
tm.renamed = make(map[string]string)
oldTableNames := make(map[uint64]string)
oldTableHashes := make(map[uint64]hash.Hash)
err := older.IterTables(ctx, func(name string, table *doltdb.Table) (stop bool, err error) {
sch, err := table.GetSchema(ctx)
if err != nil {
return true, err
}
th, err := table.HashOf()
if err != nil {
return true, err
}
pkTag := sch.GetPKCols().GetColumns()[0].Tag
oldTableNames[pkTag] = name
oldTableHashes[pkTag] = th
return false, nil
})
if err != nil {
return tm, err
}
err = newer.IterTables(ctx, func(name string, table *doltdb.Table) (stop bool, err error) {
sch, err := table.GetSchema(ctx)
if err != nil {
return true, err
}
th, err := table.HashOf()
if err != nil {
return true, err
}
pkTag := sch.GetPKCols().GetColumns()[0].Tag
oldName, ok := oldTableNames[pkTag]
switch {
case !ok:
tm.added = append(tm.added, name)
case oldName != name:
tm.renamed[name] = oldName
case oldTableHashes[pkTag] != th:
tm.modified = append(tm.modified, name)
default:
tm.unchanged = append(tm.unchanged, name)
}
if ok {
delete(oldTableNames, pkTag) // consume table name
}
return false, nil
})
if err != nil {
return tm, err
}
// all unmatched tables from older must have been dropped
for _, oldName := range oldTableNames {
tm.dropped = append(tm.dropped, oldName)
}
return tm, nil
}
// todo: to vs from
type TableDelta struct {
NewName string
OldName string
NewTable *doltdb.Table
OldTable *doltdb.Table
}
func GetTableDeltas(ctx context.Context, older, newer *doltdb.RootValue) ([]TableDelta, error) {
var deltas []TableDelta
oldTables := make(map[uint64]*doltdb.Table)
oldTableNames := make(map[uint64]string)
oldTableHashes := make(map[uint64]hash.Hash)
err := older.IterTables(ctx, func(name string, table *doltdb.Table) (stop bool, err error) {
sch, err := table.GetSchema(ctx)
if err != nil {
return true, err
}
th, err := table.HashOf()
if err != nil {
return true, err
}
pkTag := sch.GetPKCols().GetColumns()[0].Tag
oldTables[pkTag] = table
oldTableNames[pkTag] = name
oldTableHashes[pkTag] = th
return false, nil
})
if err != nil {
return nil, err
}
err = newer.IterTables(ctx, func(name string, table *doltdb.Table) (stop bool, err error) {
sch, err := table.GetSchema(ctx)
if err != nil {
return true, err
}
th, err := table.HashOf()
if err != nil {
return true, err
}
pkTag := sch.GetPKCols().GetColumns()[0].Tag
oldName, ok := oldTableNames[pkTag]
if !ok {
deltas = append(deltas, TableDelta{NewName: name, NewTable: table})
} else if oldName != name || oldTableHashes[pkTag] != th {
deltas = append(deltas, TableDelta{
NewName: name,
OldName: oldTableNames[pkTag],
NewTable: table,
OldTable: oldTables[pkTag],
})
}
if ok {
delete(oldTableNames, pkTag) // consume table name
}
return false, nil
})
if err != nil {
return nil, err
}
// all unmatched tables from older must have been dropped
for pkTag, oldName := range oldTableNames {
deltas = append(deltas, TableDelta{OldName: oldName, OldTable: oldTables[pkTag]})
}
return deltas, nil
}
func (td TableDelta) IsAdd() bool {
return td.OldTable == nil && td.NewTable != nil
}
func (td TableDelta) IsDrop() bool {
return td.OldTable != nil && td.NewTable == nil
}
func (td TableDelta) GetSchemas(ctx context.Context) (new, old schema.Schema, err error) {
if td.OldTable != nil {
old, err = td.OldTable.GetSchema(ctx)
if err != nil {
return nil, nil, err
}
}
if td.NewTable != nil {
new, err = td.NewTable.GetSchema(ctx)
if err != nil {
return nil, nil, err
}
}
return new, old, nil
}

2
go/libraries/doltcore/diff/schema_diff_test.go
View File

@@ -25,7 +25,7 @@ import (
func TestDiffSchemas(t *testing.T) {
oldCols := []schema.Column{
schema.NewColumn("unchanged", 0, types.StringKind, true, schema.NotNullConstraint{}),
schema.NewColumn("removed", 1, types.StringKind, true),
schema.NewColumn("dropped", 1, types.StringKind, true),
schema.NewColumn("renamed", 2, types.StringKind, false),
schema.NewColumn("type_changed", 3, types.StringKind, false),
schema.NewColumn("moved_to_pk", 4, types.StringKind, false),

38
go/libraries/doltcore/doltdb/root_val.go
View File

@@ -348,7 +348,6 @@ func (root *RootValue) getSuperSchemaAtLastCommit(ctx context.Context, tName str
return ss, true, nil
}
// TODO: get or create from history of root
func (root *RootValue) getOrCreateSuperSchemaMap(ctx context.Context) (types.Map, error) {
v, found, err := root.valueSt.MaybeGet(superSchemasKey)
@@ -558,6 +557,43 @@ func (root *RootValue) HasConflicts(ctx context.Context) (bool, error) {
return len(cnfTbls) > 0, nil
}
func (root *RootValue) IterTables(ctx context.Context, cb func(name string, table *Table) (stop bool, err error)) error {
tm, err := root.getTableMap()
if err != nil {
return err
}
itr, err := tm.Iterator(ctx)
if err != nil {
return err
}
for {
nm, tableRef, err := itr.Next(ctx)
if err != nil || nm == nil || tableRef == nil {
return err
}
tableStruct, err := tableRef.(types.Ref).TargetValue(ctx, root.vrw)
if err != nil {
return err
}
name := string(nm.(types.String))
table := &Table{root.vrw, tableStruct.(types.Struct)}
stop, err := cb(name, table)
if err != nil || stop {
return err
}
}
}
// PutSuperSchema writes a new map entry for the table name and super schema supplied, it will overwrite an existing entry.
func (root *RootValue) PutSuperSchema(ctx context.Context, tName string, ss *schema.SuperSchema) (*RootValue, error) {
newRoot := root