Merge branch 'main' into andy/prolly-indexes

.github/scripts/performance-benchmarking/get-mysql-dolt-job-json.sh

@@ -17,7 +17,7 @@ actorprefix="$7"
 format="$8"
 precision="1"
 
-readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan')"
+readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan', 'index_scan', 'table_scan', 'groupby_scan')"
 medianLatencyMultiplierReadsQuery="select f.test_name as read_tests, f.server_name, f.server_version, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, t.server_name, t.server_version, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) < 1.0 then 1.0 else ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) end as multiplier from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;"
 meanMultiplierReadsQuery="select round(avg(multipliers), $precision) as reads_mean_multiplier from (select case when (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) < 1.0 then 1.0 else (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) end as multipliers from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name)"
 

go/go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
-	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
 	github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
 	github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible
@@ -19,7 +18,7 @@ require (
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi v0.0.0-20201005193433-3ee972b1d078
 	github.com/dolthub/fslock v0.0.3
-	github.com/dolthub/go-mysql-server v0.11.1-0.20220119215817-a848aa9c7e52
+	github.com/dolthub/go-mysql-server v0.11.1-0.20220120003117-69f4a35700d3
 	github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371
 	github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66
 	github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
@@ -27,29 +26,26 @@ require (
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fatih/color v1.9.0
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
-	github.com/go-kit/kit v0.10.0 // indirect
 	github.com/go-openapi/errors v0.19.6 // indirect
 	github.com/go-openapi/strfmt v0.19.5 // indirect
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gocraft/dbr/v2 v2.7.2
-	github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.1
 	github.com/google/go-cmp v0.5.6
 	github.com/google/uuid v1.2.0
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/jedib0t/go-pretty v4.3.1-0.20191104025401-85fe5d6a7c4d+incompatible
 	github.com/jpillora/backoff v1.0.0
 	github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d
-	github.com/lestrrat-go/strftime v1.0.4 // indirect
+	github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6
 	github.com/mattn/go-isatty v0.0.12
 	github.com/mattn/go-runewidth v0.0.9
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b
-	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.3.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.5.0
+	github.com/prometheus/client_golang v1.11.0
 	github.com/rivo/uniseg v0.1.0
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.2+incompatible
@@ -63,66 +59,21 @@ require (
 	github.com/tklauser/go-sysconf v0.3.5 // indirect
 	github.com/uber/jaeger-client-go v2.25.0+incompatible
 	github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
+	github.com/xitongsys/parquet-go v1.6.1
+	github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371
 	go.mongodb.org/mongo-driver v1.7.0 // indirect
 	go.uber.org/zap v1.15.0
 	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
 	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20211019181941-9d821ace8654
+	golang.org/x/tools v0.1.8 // indirect
 	google.golang.org/api v0.32.0
-	google.golang.org/genproto v0.0.0-20210506142907-4a47615972c2 // indirect
 	google.golang.org/grpc v1.37.0
 	google.golang.org/protobuf v1.26.0
 	gopkg.in/square/go-jose.v2 v2.5.1
 	gopkg.in/src-d/go-errors.v1 v1.0.0
 	gopkg.in/yaml.v2 v2.3.0
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
-)
-
-require (
-	github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6
-	github.com/prometheus/client_golang v1.11.0
-	github.com/xitongsys/parquet-go v1.6.1
-	github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371
-)
-
-require (
-	cloud.google.com/go v0.66.0 // indirect
-	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
-	github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/go-ole/go-ole v1.2.1 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/jstemmer/go-junit-report v0.9.1 // indirect
-	github.com/klauspost/compress v1.10.10 // indirect
-	github.com/mattn/go-colorable v0.1.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 // indirect
-	github.com/pierrec/lz4/v4 v4.1.6 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.26.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/src-d/go-oniguruma v1.1.0 // indirect
-	github.com/tklauser/numcpus v0.2.2 // indirect
-	go.opencensus.io v0.22.4 // indirect
-	go.uber.org/atomic v1.6.0 // indirect
-	go.uber.org/multierr v1.5.0 // indirect
-	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
-	golang.org/x/mod v0.5.1 // indirect
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.8 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
 )
 
 replace (

go/go.sum

@@ -175,10 +175,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
-github.com/dolthub/go-mysql-server v0.11.1-0.20220118204939-ed04cdef372c h1:wb2wHS7Yizn6xHVVFnhHi+GF++avJRnupxlRzAdb9v8=
-github.com/dolthub/go-mysql-server v0.11.1-0.20220118204939-ed04cdef372c/go.mod h1:tyrWU1vUzj/ilniOAefGJquvOpHNSrFSUHVWJqlSIFc=
-github.com/dolthub/go-mysql-server v0.11.1-0.20220119215817-a848aa9c7e52 h1:uY5VWCU6Rv3z3GFb1buploWnA9YyHJTcODNKVVo1NY4=
-github.com/dolthub/go-mysql-server v0.11.1-0.20220119215817-a848aa9c7e52/go.mod h1:tyrWU1vUzj/ilniOAefGJquvOpHNSrFSUHVWJqlSIFc=
+github.com/dolthub/go-mysql-server v0.11.1-0.20220120003117-69f4a35700d3 h1:LeUle75vefgXEYWenKDnhZ2ZlK8lOoDZsVhpZ5D0ztg=
+github.com/dolthub/go-mysql-server v0.11.1-0.20220120003117-69f4a35700d3/go.mod h1:tyrWU1vUzj/ilniOAefGJquvOpHNSrFSUHVWJqlSIFc=
 github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g=
 github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms=
 github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8=

go/libraries/doltcore/doltdb/foreign_key_coll.go

@@ -317,33 +317,125 @@ func (fkc *ForeignKeyCollection) GetByNameCaseInsensitive(foreignKeyName string)
 	return ForeignKey{}, false
 }
 
-// GetByTags gets the Foreign Key defined over the parent and child columns corresponding to tags parameters.
-func (fkc *ForeignKeyCollection) GetByTags(childTags, parentTags []uint64) (match ForeignKey, ok bool) {
+// GetByTags gets the ForeignKey defined over the parent and child columns corresponding to their tags.
+func (fkc *ForeignKeyCollection) GetByTags(childTags, parentTags []uint64) (ForeignKey, bool) {
 	if len(childTags) == 0 || len(parentTags) == 0 {
-		return match, false
+		return ForeignKey{}, false
 	}
-	_ = fkc.Iter(func(fk ForeignKey) (stop bool, err error) {
+OuterLoop:
+	for _, fk := range fkc.foreignKeys {
 		if len(fk.ReferencedTableColumns) != len(parentTags) {
-			return false, nil
+			continue
 		}
 		for i, t := range fk.ReferencedTableColumns {
 			if t != parentTags[i] {
-				return false, nil
+				continue OuterLoop
 			}
 		}
 
 		if len(fk.TableColumns) != len(childTags) {
-			return false, nil
+			continue
 		}
 		for i, t := range fk.TableColumns {
 			if t != childTags[i] {
-				return false, nil
+				continue OuterLoop
 			}
 		}
-		match, ok = fk, true
-		return true, nil
-	})
-	return match, ok
+		return fk, true
+	}
+	return ForeignKey{}, false
+}
+
+// GetMatchingKey gets the ForeignKey defined over the parent and child columns. If the given foreign key is resolved,
+// then both resolved and unresolved keys are checked for a match. If the given foreign key is unresolved, then ONLY
+// unresolved keys may be found.
+//
+// This discrepancy is due to the primary uses for this function. It is assumed that the ForeignKeyCollection is an
+// ancestor collection compared to the collection that the given key comes from. Therefore, the key found in the
+// ancestor will usually be the unresolved version of the given key, hence the comparison is valid. However, if the
+// given key is unresolved, it is treated as a new key, which cannot be matched to a resolved key that previously
+// existed.
+//
+// The given schema map is keyed by table name, and is used in the event that the given key is resolved and any keys in
+// the collection are unresolved. A "dirty resolution" is performed, which matches the column names to tags, and then a
+// standard tag comparison is performed. If a table or column is not in the map, then the foreign key is ignored.
+func (fkc *ForeignKeyCollection) GetMatchingKey(fk ForeignKey, allSchemas map[string]schema.Schema) (ForeignKey, bool) {
+	if !fk.IsResolved() {
+		// The given foreign key is unresolved, so we only look for matches on unresolved keys
+	OuterLoopUnresolved:
+		for _, existingFk := range fkc.foreignKeys {
+			// For unresolved keys, the table name is important (column tags are globally unique, column names are not)
+			if existingFk.IsResolved() ||
+				fk.TableName != existingFk.TableName ||
+				fk.ReferencedTableName != existingFk.ReferencedTableName ||
+				len(fk.UnresolvedFKDetails.TableColumns) != len(existingFk.UnresolvedFKDetails.TableColumns) ||
+				len(fk.UnresolvedFKDetails.ReferencedTableColumns) != len(existingFk.UnresolvedFKDetails.ReferencedTableColumns) {
+				continue
+			}
+			for i, fkCol := range fk.UnresolvedFKDetails.TableColumns {
+				if fkCol != existingFk.UnresolvedFKDetails.TableColumns[i] {
+					continue OuterLoopUnresolved
+				}
+			}
+			for i, fkCol := range fk.UnresolvedFKDetails.ReferencedTableColumns {
+				if fkCol != existingFk.UnresolvedFKDetails.ReferencedTableColumns[i] {
+					continue OuterLoopUnresolved
+				}
+			}
+			return existingFk, true
+		}
+		return ForeignKey{}, false
+	}
+	// The given foreign key is resolved, so we may match both resolved and unresolved keys
+OuterLoopResolved:
+	for _, existingFk := range fkc.foreignKeys {
+		if existingFk.IsResolved() {
+			// When both are resolved, we do a standard tag comparison
+			if len(fk.TableColumns) != len(existingFk.TableColumns) ||
+				len(fk.ReferencedTableColumns) != len(existingFk.ReferencedTableColumns) {
+				continue
+			}
+			for i, tag := range fk.TableColumns {
+				if tag != existingFk.TableColumns[i] {
+					continue OuterLoopResolved
+				}
+			}
+			for i, tag := range fk.ReferencedTableColumns {
+				if tag != existingFk.ReferencedTableColumns[i] {
+					continue OuterLoopResolved
+				}
+			}
+			return existingFk, true
+		} else {
+			// Since the existing key is unresolved, we reference the schema map to get tags we can use
+			if len(fk.TableColumns) != len(existingFk.UnresolvedFKDetails.TableColumns) ||
+				len(fk.ReferencedTableColumns) != len(existingFk.UnresolvedFKDetails.ReferencedTableColumns) {
+				continue
+			}
+			tblSch, ok := allSchemas[existingFk.TableName]
+			if !ok {
+				continue
+			}
+			refTblSch, ok := allSchemas[existingFk.ReferencedTableName]
+			if !ok {
+				continue
+			}
+			for i, tag := range fk.TableColumns {
+				col, ok := tblSch.GetAllCols().GetByNameCaseInsensitive(existingFk.UnresolvedFKDetails.TableColumns[i])
+				if !ok || tag != col.Tag {
+					continue OuterLoopResolved
+				}
+			}
+			for i, tag := range fk.ReferencedTableColumns {
+				col, ok := refTblSch.GetAllCols().GetByNameCaseInsensitive(existingFk.UnresolvedFKDetails.ReferencedTableColumns[i])
+				if !ok || tag != col.Tag {
+					continue OuterLoopResolved
+				}
+			}
+			return existingFk, true
+		}
+	}
+	return ForeignKey{}, false
 }
 
 func (fkc *ForeignKeyCollection) Iter(cb func(fk ForeignKey) (stop bool, err error)) error {

go/libraries/doltcore/dtestutils/schema.go

@@ -55,6 +55,8 @@ func NewRow(sch schema.Schema, values ...types.Value) row.Row {
 
 // AddColumnToSchema returns a new schema by adding the given column to the given schema. Will panic on an invalid
 // schema, e.g. tag collision.
+// Note the AddColumnToSchema relies on being called from the engine (GMS) to correctly update defaults. Directly calling
+// this method in Dolt only adds a new column to schema but does not apply the default.
 func AddColumnToSchema(sch schema.Schema, col schema.Column) schema.Schema {
 	columns := sch.GetAllCols()
 	columns = columns.Append(col)

go/libraries/doltcore/merge/merge_schema.go

@@ -143,22 +143,29 @@ func ForeignKeysMerge(ctx context.Context, mergedRoot, ourRoot, theirRoot, ancRo
 		return nil, nil, err
 	}
 
+	ancSchs, err := ancRoot.GetAllSchemas(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	common, conflicts, err := foreignKeysInCommon(ours, theirs, anc)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	ourNewFKs, err := fkCollSetDifference(ours, anc)
+	ourNewFKs, err := fkCollSetDifference(ours, anc, ancSchs)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	theirNewFKs, err := fkCollSetDifference(theirs, anc)
+	theirNewFKs, err := fkCollSetDifference(theirs, anc, ancSchs)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	// check for conflicts between foreign keys added on each branch since the ancestor
+	//TODO: figure out the best way to handle unresolved foreign keys here if one branch added an unresolved one and
+	// another branch added the same one but resolved
 	_ = ourNewFKs.Iter(func(ourFK doltdb.ForeignKey) (stop bool, err error) {
 		theirFK, ok := theirNewFKs.GetByTags(ourFK.TableColumns, ourFK.ReferencedTableColumns)
 		if ok && !ourFK.DeepEquals(theirFK) {
@@ -316,7 +323,8 @@ func mergeIndexes(mergedCC *schema.ColCollection, ourSch, theirSch, ancSch schem
 	// check for conflicts between indexes added on each branch since the ancestor
 	_ = ourNewIdxs.Iter(func(ourIdx schema.Index) (stop bool, err error) {
 		theirIdx, ok := theirNewIdxs.GetByNameCaseInsensitive(ourIdx.Name())
-		if ok {
+		// If both indexes are exactly equal then there isn't a conflict
+		if ok && !ourIdx.DeepEquals(theirIdx) {
 			conflicts = append(conflicts, IdxConflict{
 				Kind:   NameCollision,
 				Ours:   ourIdx,
@@ -497,10 +505,13 @@ func foreignKeysInCommon(ourFKs, theirFKs, ancFKs *doltdb.ForeignKeyCollection)
 	return common, conflicts, nil
 }
 
-func fkCollSetDifference(left, right *doltdb.ForeignKeyCollection) (d *doltdb.ForeignKeyCollection, err error) {
+// fkCollSetDifference returns a collection of all foreign keys that are in the given collection but not the ancestor
+// collection. This is specifically for finding differences between a descendant and an ancestor, and therefore should
+// not be used in the general case.
+func fkCollSetDifference(fkColl, ancestorFkColl *doltdb.ForeignKeyCollection, ancSchs map[string]schema.Schema) (d *doltdb.ForeignKeyCollection, err error) {
 	d, _ = doltdb.NewForeignKeyCollection()
-	err = left.Iter(func(fk doltdb.ForeignKey) (stop bool, err error) {
-		_, ok := right.GetByTags(fk.TableColumns, fk.ReferencedTableColumns)
+	err = fkColl.Iter(func(fk doltdb.ForeignKey) (stop bool, err error) {
+		_, ok := ancestorFkColl.GetMatchingKey(fk, ancSchs)
 		if !ok {
 			err = d.AddKeys(fk)
 		}

go/libraries/doltcore/schema/alterschema/addcolumn.go

@@ -22,11 +22,8 @@ import (
 	"github.com/dolthub/go-mysql-server/sql"
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
-	"github.com/dolthub/dolt/go/libraries/doltcore/row"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
-	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
-	"github.com/dolthub/dolt/go/store/types"
 )
 
 // Nullable represents whether a column can have a null value.
@@ -79,31 +76,7 @@ func AddColumnToTable(
 		return nil, err
 	}
 
-	return updateTableWithNewSchema(ctx, tblName, tbl, tag, newSchema, defaultVal)
-}
-
-// updateTableWithNewSchema updates the existing table with a new schema and new values for the new column as necessary,
-// and returns the new table.
-func updateTableWithNewSchema(
-	ctx context.Context,
-	tblName string,
-	tbl *doltdb.Table,
-	tag uint64,
-	newSchema schema.Schema,
-	defaultVal *sql.ColumnDefaultValue,
-) (*doltdb.Table, error) {
-	var err error
-	tbl, err = tbl.UpdateSchema(ctx, newSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	tbl, err = applyDefaultValue(ctx, tblName, tbl, tag, newSchema, defaultVal)
-	if err != nil {
-		return nil, err
-	}
-
-	return tbl, nil
+	return tbl.UpdateSchema(ctx, newSchema)
 }
 
 // addColumnToSchema creates a new schema with a column as specified by the params.
@@ -214,61 +187,3 @@ func validateNewColumn(
 
 	return nil
 }
-
-func applyDefaultValue(
-	ctx context.Context,
-	tblName string,
-	tbl *doltdb.Table,
-	tag uint64,
-	newSchema schema.Schema,
-	defaultVal *sql.ColumnDefaultValue,
-) (*doltdb.Table, error) {
-	rowData, err := tbl.GetNomsRowData(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	me := rowData.Edit()
-
-	newSqlSchema, err := sqlutil.FromDoltSchema(tblName, newSchema)
-	if err != nil {
-		return nil, err
-	}
-
-	columnIndex := -1
-	for i, colTag := range newSchema.GetAllCols().Tags {
-		if colTag == tag {
-			columnIndex = i
-			break
-		}
-	}
-	if columnIndex == -1 {
-		return nil, fmt.Errorf("could not find tag `%d` in new schema", tag)
-	}
-
-	// FromDoltSchema doesn't reify the expression for a default, so set it explicitly
-	newSqlSchema.Schema[columnIndex].Default = defaultVal
-
-	err = rowData.Iter(ctx, func(k, v types.Value) (stop bool, err error) {
-		oldRow, err := row.FromNoms(newSchema, k.(types.Tuple), v.(types.Tuple))
-		if err != nil {
-			return true, err
-		}
-		newRow, err := sqlutil.ApplyDefaults(ctx, tbl.ValueReadWriter(), newSchema, newSqlSchema.Schema, columnIndex, oldRow)
-		if err != nil {
-			return true, err
-		}
-		me.Set(newRow.NomsMapKey(newSchema), newRow.NomsMapValue(newSchema))
-		return false, nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	newRowData, err := me.Map(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	return tbl.UpdateNomsRows(ctx, newRowData)
-}

go/libraries/doltcore/schema/alterschema/addcolumn_test.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/go-mysql-server/sql/parse"
-	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -47,46 +46,6 @@ func TestAddColumnToTable(t *testing.T) {
 		expectedRows   []row.Row
 		expectedErr    string
 	}{
-		{
-			name:       "string column no default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.StringKind,
-			nullable:   Null,
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schema.NewColumn("newCol", dtestutils.NextTag, types.StringKind, false)),
-			expectedRows: dtestutils.TypedRows,
-		},
-		{
-			name:       "int column no default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.IntKind,
-			nullable:   Null,
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schema.NewColumn("newCol", dtestutils.NextTag, types.IntKind, false)),
-			expectedRows: dtestutils.TypedRows,
-		},
-		{
-			name:       "uint column no default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.UintKind,
-			nullable:   Null,
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schema.NewColumn("newCol", dtestutils.NextTag, types.UintKind, false)),
-			expectedRows: dtestutils.TypedRows,
-		},
-		{
-			name:       "float column no default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.FloatKind,
-			nullable:   Null,
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schema.NewColumn("newCol", dtestutils.NextTag, types.FloatKind, false)),
-			expectedRows: dtestutils.TypedRows,
-		},
 		{
 			name:       "bool column no default",
 			tag:        dtestutils.NextTag,
@@ -97,83 +56,6 @@ func TestAddColumnToTable(t *testing.T) {
 				schema.NewColumn("newCol", dtestutils.NextTag, types.BoolKind, false)),
 			expectedRows: dtestutils.TypedRows,
 		},
-		{
-			name:       "uuid column no default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.UUIDKind,
-			nullable:   Null,
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schema.NewColumn("newCol", dtestutils.NextTag, types.UUIDKind, false)),
-			expectedRows: dtestutils.TypedRows,
-		},
-		{
-			name:       "string column with default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.StringKind,
-			nullable:   NotNull,
-			defaultVal: mustStringToColumnDefault(`("default")`),
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schemaNewColumn("newCol", dtestutils.NextTag, types.StringKind, false, `("default")`, schema.NotNullConstraint{})),
-			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.String("default")),
-		},
-		{
-			name:       "int column with default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.IntKind,
-			nullable:   NotNull,
-			defaultVal: mustStringToColumnDefault("42"),
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schemaNewColumn("newCol", dtestutils.NextTag, types.IntKind, false, "42", schema.NotNullConstraint{})),
-			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.Int(42)),
-		},
-		{
-			name:       "uint column with default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.UintKind,
-			nullable:   NotNull,
-			defaultVal: mustStringToColumnDefault("64"),
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schemaNewColumn("newCol", dtestutils.NextTag, types.UintKind, false, "64", schema.NotNullConstraint{})),
-			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.Uint(64)),
-		},
-		{
-			name:       "float column with default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.FloatKind,
-			nullable:   NotNull,
-			defaultVal: mustStringToColumnDefault("33.33"),
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schemaNewColumn("newCol", dtestutils.NextTag, types.FloatKind, false, "33.33", schema.NotNullConstraint{})),
-			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.Float(33.33)),
-		},
-		{
-			name:       "bool column with default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.BoolKind,
-			nullable:   NotNull,
-			defaultVal: mustStringToColumnDefault("true"),
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schemaNewColumn("newCol", dtestutils.NextTag, types.BoolKind, false, "true", schema.NotNullConstraint{})),
-			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.Bool(true)),
-		},
-		{
-			name:       "uuid column with default",
-			tag:        dtestutils.NextTag,
-			newColName: "newCol",
-			colKind:    types.UUIDKind,
-			nullable:   NotNull,
-			defaultVal: mustStringToColumnDefault(`"00000000-0000-0000-0000-000000000000"`),
-			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
-				schemaNewColumn("newCol", dtestutils.NextTag, types.UUIDKind, false, `"00000000-0000-0000-0000-000000000000"`, schema.NotNullConstraint{})),
-			expectedRows: dtestutils.AddColToRows(t,
-				dtestutils.TypedRows, dtestutils.NextTag, types.UUID(uuid.MustParse("00000000-0000-0000-0000-000000000000"))),
-		},
 		{
 			name:       "nullable with nil default",
 			tag:        dtestutils.NextTag,
@@ -193,7 +75,7 @@ func TestAddColumnToTable(t *testing.T) {
 			defaultVal: mustStringToColumnDefault("42"),
 			expectedSchema: dtestutils.AddColumnToSchema(dtestutils.TypedSchema,
 				schemaNewColumn("newCol", dtestutils.NextTag, types.IntKind, false, "42")),
-			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.Int(42)),
+			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.NullValue),
 		},
 		{
 			name:       "first order",
@@ -211,7 +93,7 @@ func TestAddColumnToTable(t *testing.T) {
 				schema.NewColumn("is_married", dtestutils.IsMarriedTag, types.BoolKind, false, schema.NotNullConstraint{}),
 				schema.NewColumn("title", dtestutils.TitleTag, types.StringKind, false),
 			),
-			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.Int(42)),
+			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.NullValue),
 		},
 		{
 			name:       "middle order",
@@ -229,7 +111,7 @@ func TestAddColumnToTable(t *testing.T) {
 				schema.NewColumn("is_married", dtestutils.IsMarriedTag, types.BoolKind, false, schema.NotNullConstraint{}),
 				schema.NewColumn("title", dtestutils.TitleTag, types.StringKind, false),
 			),
-			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.Int(42)),
+			expectedRows: dtestutils.AddColToRows(t, dtestutils.TypedRows, dtestutils.NextTag, types.NullValue),
 		},
 		{
 			name:        "tag collision",

go/libraries/doltcore/schema/index.go

@@ -32,6 +32,9 @@ type Index interface {
 	Comment() string
 	// Count returns the number of indexed columns in this index.
 	Count() int
+	// DeepEquals returns whether this Index is equivalent to another. This function is similar to Equals, however it
+	// does take the table's primary keys into consideration.
+	DeepEquals(other Index) bool
 	// Equals returns whether this Index is equivalent to another. This does not check for column names, thus those may
 	// be renamed and the index equivalence will be preserved. It also does not depend on the table's primary keys.
 	Equals(other Index) bool
@@ -124,6 +127,26 @@ func (ix *indexImpl) Equals(other Index) bool {
 		ix.Name() == other.Name()
 }
 
+// DeepEquals implements Index.
+func (ix *indexImpl) DeepEquals(other Index) bool {
+	if ix.Count() != other.Count() {
+		return false
+	}
+
+	// we're only interested in columns the index is defined over, not the table's primary keys
+	tt := ix.AllTags()
+	ot := other.AllTags()
+	for i := range tt {
+		if tt[i] != ot[i] {
+			return false
+		}
+	}
+
+	return ix.IsUnique() == other.IsUnique() &&
+		ix.Comment() == other.Comment() &&
+		ix.Name() == other.Name()
+}
+
 // GetColumn implements Index.
 func (ix *indexImpl) GetColumn(tag uint64) (Column, bool) {
 	return ix.indexColl.colColl.GetByTag(tag)

go/libraries/doltcore/sqle/enginetest/dolt_transaction_commit_test.go

@@ -363,3 +363,75 @@ func TestDoltTransactionCommitAutocommit(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, "Initialize data repository", cm0.Description)
 }
+
+func TestDoltTransactionCommitLateFkResolution(t *testing.T) {
+	harness := newDoltHarness(t)
+	enginetest.TestTransactionScript(t, harness, enginetest.TransactionTest{
+		Name: "delayed foreign key resolution with transaction commits",
+		SetUpScript: []string{
+			"SET foreign_key_checks=0;",
+			"CREATE TABLE child (pk BIGINT PRIMARY KEY, v1 BIGINT, CONSTRAINT fk_late FOREIGN KEY (v1) REFERENCES parent (pk));",
+			"SET foreign_key_checks=1;",
+			"CREATE TABLE parent (pk BIGINT PRIMARY KEY);",
+			"INSERT INTO parent VALUES (1), (2);",
+		},
+		Assertions: []enginetest.ScriptTestAssertion{
+			{
+				Query:    "/* client a */ SET @@autocommit=0;",
+				Expected: []sql.Row{{}},
+			},
+			{
+				Query:    "/* client b */ SET @@autocommit=0;",
+				Expected: []sql.Row{{}},
+			},
+			{
+				Query:    "/* client a */ START TRANSACTION;",
+				Expected: []sql.Row{},
+			},
+			{
+				Query:    "/* client b */ START TRANSACTION;",
+				Expected: []sql.Row{},
+			},
+			{
+				Query:    "/* client a */ INSERT INTO child VALUES (1, 1);",
+				Expected: []sql.Row{{sql.NewOkResult(1)}},
+			},
+			{
+				Query:    "/* client b */ INSERT INTO child VALUES (2, 2);",
+				Expected: []sql.Row{{sql.NewOkResult(1)}},
+			},
+			{
+				Query:    "/* client a */ COMMIT;",
+				Expected: []sql.Row{},
+			},
+			{
+				Query:    "/* client b */ COMMIT;",
+				Expected: []sql.Row{},
+			},
+			{
+				Query:    "/* client a */ SELECT * FROM child ORDER BY pk;",
+				Expected: []sql.Row{{1, 1}, {2, 2}},
+			},
+			{
+				Query:    "/* client b */ SELECT * FROM child ORDER BY pk;",
+				Expected: []sql.Row{{1, 1}, {2, 2}},
+			},
+			{ // This uses the index, which is automatically created by the late fk resolution, so it's also tested here
+				Query:    "/* client a */ SELECT * FROM child WHERE v1 > 0 ORDER BY pk;",
+				Expected: []sql.Row{{1, 1}, {2, 2}},
+			},
+			{ // This uses the index, which is automatically created by the late fk resolution, so it's also tested here
+				Query:    "/* client b */ SELECT * FROM child WHERE v1 > 0 ORDER BY pk;",
+				Expected: []sql.Row{{1, 1}, {2, 2}},
+			},
+			{
+				Query:       "/* client a */ INSERT INTO child VALUES (3, 3);",
+				ExpectedErr: sql.ErrForeignKeyChildViolation,
+			},
+			{
+				Query:       "/* client b */ INSERT INTO child VALUES (3, 3);",
+				ExpectedErr: sql.ErrForeignKeyChildViolation,
+			},
+		},
+	})
+}

go/libraries/doltcore/sqle/sqlddl_test.go

@@ -369,12 +369,47 @@ func TestAddColumn(t *testing.T) {
 		expectedErr    string
 	}{
 		{
-			name:  "alter add column",
+			name:  "alter add string column no default",
 			query: "alter table people add (newColumn varchar(80))",
 			expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
 				schemaNewColumn(t, "newColumn", 4208, sql.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false)),
 			expectedRows: dtestutils.AddColToRows(t, AllPeopleRows, 4208, nil),
 		},
+		{
+			name:  "alter add float column without default",
+			query: "alter table people add (newColumn float)",
+			expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
+				schemaNewColumn(t, "newColumn", 4208, sql.Float32, false)),
+			expectedRows: dtestutils.AddColToRows(t, AllPeopleRows, 4208, nil),
+		},
+		{
+			name:  "alter add uint column without default",
+			query: "alter table people add (newColumn bigint unsigned)",
+			expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
+				schemaNewColumn(t, "newColumn", 4208, sql.Uint64, false)),
+			expectedRows: dtestutils.AddColToRows(t, AllPeopleRows, 4208, nil),
+		},
+		{
+			name:  "alter add int column default",
+			query: "alter table people add (newColumn int default 2)",
+			expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
+				schemaNewColumnWDefVal(t, "newColumn", 4435, sql.Int32, false, "2")),
+			expectedRows: dtestutils.AddColToRows(t, AllPeopleRows, 4435, types.Int(int32(2))),
+		},
+		{
+			name:  "alter add uint column default",
+			query: "alter table people add (newColumn bigint unsigned default 20)",
+			expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
+				schemaNewColumnWDefVal(t, "newColumn", 6535, sql.Uint64, false, "20")),
+			expectedRows: dtestutils.AddColToRows(t, AllPeopleRows, 6535, types.Uint(uint64(20))),
+		},
+		{
+			name:  "alter add string column with default",
+			query: "alter table people add (newColumn varchar(80) default 'hi')",
+			expectedSchema: dtestutils.AddColumnToSchema(PeopleTestSchema,
+				schemaNewColumnWDefVal(t, "newColumn", 4208, sql.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false, `"hi"`)),
+			expectedRows: dtestutils.AddColToRows(t, AllPeopleRows, 4208, types.String("hi")),
+		},
 		{
 			name:  "alter add column first",
 			query: "alter table people add newColumn varchar(80) first",
@@ -464,6 +499,11 @@ func TestAddColumn(t *testing.T) {
 				schemaNewColumn(t, "newColumn", 4208, sql.MustCreateStringWithDefaults(sqltypes.VarChar, 80), false)),
 			expectedRows: AllPeopleRows,
 		},
+		{
+			name:        "alter table add column name clash",
+			query:       "alter table people add column(age int)",
+			expectedErr: `Column "age" already exists`,
+		},
 	}
 
 	for _, tt := range tests {

go/libraries/doltcore/sqle/sqlutil/schema.go

@@ -17,62 +17,14 @@ package sqlutil
 import (
 	"context"
 
-	sqle "github.com/dolthub/go-mysql-server"
 	"github.com/dolthub/go-mysql-server/sql"
 	"github.com/dolthub/go-mysql-server/sql/parse"
 	"github.com/dolthub/vitess/go/vt/sqlparser"
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
-	"github.com/dolthub/dolt/go/libraries/doltcore/row"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
-	"github.com/dolthub/dolt/go/store/types"
 )
 
-// ApplyDefaults applies the default values to the given indices, returning the resulting row.
-func ApplyDefaults(
-	ctx context.Context,
-	vrw types.ValueReadWriter,
-	doltSchema schema.Schema,
-	sqlSchema sql.Schema,
-	colIdx int,
-	dRow row.Row,
-) (row.Row, error) {
-	sqlCtx, ok := ctx.(*sql.Context)
-	if !ok {
-		sqlCtx = sql.NewContext(ctx)
-	}
-	doltCols := doltSchema.GetAllCols()
-	oldSqlRow := make(sql.Row, len(sqlSchema))
-	for i, tag := range doltCols.Tags {
-		val, ok := dRow.GetColVal(tag)
-		if ok {
-			var err error
-			oldSqlRow[i], err = doltCols.TagToCol[tag].TypeInfo.ConvertNomsValueToValue(val)
-			if err != nil {
-				return nil, err
-			}
-		} else {
-			oldSqlRow[i] = nil
-		}
-	}
-	newSqlRow, err := sqle.ApplyDefaults(sqlCtx, sqlSchema, []int{colIdx}, oldSqlRow)
-	if err != nil {
-		return nil, err
-	}
-	newRow := make(row.TaggedValues)
-	for i, tag := range doltCols.Tags {
-		if newSqlRow[i] == nil {
-			continue
-		}
-		val, err := doltCols.TagToCol[tag].TypeInfo.ConvertValueToNomsValue(ctx, vrw, newSqlRow[i])
-		if err != nil {
-			return nil, err
-		}
-		newRow[tag] = val
-	}
-	return row.New(dRow.Format(), doltSchema, newRow)
-}
-
 // ParseCreateTableStatement will parse a CREATE TABLE ddl statement and use it to create a Dolt Schema. A RootValue
 // is used to generate unique tags for the Schema
 func ParseCreateTableStatement(ctx context.Context, root *doltdb.RootValue, query string) (string, schema.Schema, error) {

go/libraries/doltcore/sqle/sqlutil/sql_row.go

@@ -110,9 +110,6 @@ func DoltKeyValueAndMappingFromSqlRow(ctx context.Context, vrw types.ValueReadWr
 	for i, c := range doltSchema.GetAllCols().GetColumns() {
 		val := r[i]
 		if val == nil {
-			if !c.IsNullable() {
-				return types.Tuple{}, types.Tuple{}, nil, fmt.Errorf("column <%v> received nil but is non-nullable", c.Name)
-			}
 			continue
 		}
 
@@ -183,9 +180,6 @@ func DoltKeyAndMappingFromSqlRow(ctx context.Context, vrw types.ValueReadWriter,
 		schCol := allCols.GetAtIndex(i)
 		val := r[i]
 		if val == nil {
-			if !schCol.IsNullable() {
-				return types.Tuple{}, nil, fmt.Errorf("column <%v> received nil but is non-nullable", schCol.Name)
-			}
 			continue
 		}
 
@@ -232,9 +226,6 @@ func pkDoltRowFromSqlRow(ctx context.Context, vrw types.ValueReadWriter, r sql.R
 			if err != nil {
 				return nil, err
 			}
-		} else if !schCol.IsNullable() {
-			// TODO: this isn't an error in the case of result set construction (where non-null columns can indeed be null)
-			return nil, fmt.Errorf("column <%v> received nil but is non-nullable", schCol.Name)
 		}
 	}
 	return row.New(vrw.Format(), doltSchema, taggedVals)