update dolt_column_diff logic to use tags, add more tests

go/libraries/doltcore/sqle/dtables/column_diff_table.go

@@ -19,11 +19,9 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"strings"
 	"time"
 
 	"github.com/dolthub/go-mysql-server/sql"
-	"github.com/dolthub/go-mysql-server/sql/expression"
 	"github.com/dolthub/go-mysql-server/sql/plan"
 	"github.com/dolthub/go-mysql-server/sql/types"
 
@@ -32,7 +30,6 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
-	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
 	"github.com/dolthub/dolt/go/store/datas"
 	"github.com/dolthub/dolt/go/store/hash"
 	dtypes "github.com/dolthub/dolt/go/store/types"
@@ -77,8 +74,7 @@ func (dt *ColumnDiffTable) Schema() sql.Schema {
 		{Name: "email", Type: types.Text, Source: doltdb.ColumnDiffTableName, PrimaryKey: false},
 		{Name: "date", Type: types.Datetime, Source: doltdb.ColumnDiffTableName, PrimaryKey: false},
 		{Name: "message", Type: types.Text, Source: doltdb.ColumnDiffTableName, PrimaryKey: false},
-		{Name: "data_change", Type: types.Boolean, Source: doltdb.ColumnDiffTableName, PrimaryKey: false},
-		{Name: "schema_change", Type: types.Boolean, Source: doltdb.ColumnDiffTableName, PrimaryKey: false},
+		{Name: "diff_type", Type: types.Text, Source: doltdb.ColumnDiffTableName, PrimaryKey: false},
 	}
 }
 
@@ -189,6 +185,8 @@ type doltColDiffWorkingSetRowItr struct {
 	colIndex            int
 	stagedTableDeltas   []diff.TableDelta
 	unstagedTableDeltas []diff.TableDelta
+	colNames            []string
+	diffTypes           []string
 }
 
 func (dt *ColumnDiffTable) newWorkingSetRowItr(ctx *sql.Context) (sql.RowIter, error) {
@@ -218,17 +216,11 @@ func (dt *ColumnDiffTable) newWorkingSetRowItr(ctx *sql.Context) (sql.RowIter, e
 
 // incrementColIndex increments the column index and table changes index.  When the end of the column names array is
 // reached, moves to the next table changes delta.
-func (d *doltColDiffWorkingSetRowItr) incrementColIndex(tableDelta diff.TableDelta, staged bool) {
+func (d *doltColDiffWorkingSetRowItr) incrementColIndex(staged bool) {
 	d.colIndex++
-	var numCols int
-	if tableDelta.ToSch != nil {
-		numCols = tableDelta.ToSch.GetAllCols().Size()
-	} else {
-		numCols = tableDelta.FromSch.GetAllCols().Size()
-	}
 
 	// move to next table once all modified columns are iterated through
-	if d.colIndex >= numCols {
+	if d.colIndex >= len(d.colNames) {
 		d.colIndex = 0
 		if staged {
 			d.stagedIndex++
@@ -247,42 +239,59 @@ func (d *doltColDiffWorkingSetRowItr) Next(ctx *sql.Context) (sql.Row, error) {
 		changeSet = "STAGED"
 		tableDelta = d.stagedTableDeltas[d.stagedIndex]
 		staged = true
+		// only need to load new changes when we're finished iterating through the previous tableDelta
+		if d.colIndex == 0 {
+			d.loadColumnChanges(tableDelta)
+		}
 	} else if d.unstagedIndex < len(d.unstagedTableDeltas) {
 		changeSet = "WORKING"
 		tableDelta = d.unstagedTableDeltas[d.unstagedIndex]
+		// only need to load new changes when we're finished iterating through the previous tableDelta
+		if d.colIndex == 0 {
+			d.loadColumnChanges(tableDelta)
+		}
 	} else {
 		return nil, io.EOF
 	}
 
-	defer d.incrementColIndex(tableDelta, staged)
+	defer d.incrementColIndex(staged)
 
 	change, err := tableDelta.GetSummary(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	var colName string
-	if tableDelta.FromSch != nil {
-		colName = tableDelta.FromSch.GetAllCols().GetColumnNames()[d.colIndex]
-	} else {
-		colName = tableDelta.ToSch.GetAllCols().GetColumnNames()[d.colIndex]
-	}
-
 	sqlRow := sql.NewRow(
 		changeSet,
 		change.TableName,
-		colName,
+		d.colNames[d.colIndex],
 		nil, // committer
 		nil, // email
 		nil, // date
 		nil, // message
-		change.DataChange,
-		change.SchemaChange,
+		d.diffTypes[d.colIndex],
 	)
 
 	return sqlRow, nil
 }
 
+// loadColumnChanges loads the list of column names for columns that have changed and the corresponding diff_type for
+// those column changes
+func (d *doltColDiffWorkingSetRowItr) loadColumnChanges(tableDelta diff.TableDelta) {
+	var toCols, fromCols *schema.ColCollection
+	if tableDelta.ToSch != nil {
+		toCols = tableDelta.ToSch.GetAllCols()
+	}
+	if tableDelta.FromSch != nil {
+		fromCols = tableDelta.FromSch.GetAllCols()
+	}
+	_, _, _, colNames, diffTypes := calculateColSchemaDiff(toCols, fromCols)
+	// TODO (steph): if a row has been modified, all columns will be marked as modified instead of just the columns which contain the changes
+
+	d.colNames = colNames
+	d.diffTypes = diffTypes
+}
+
 func (d *doltColDiffWorkingSetRowItr) Close(c *sql.Context) error {
 	return nil
 }
@@ -369,6 +378,7 @@ func (itr *doltColDiffCommitHistoryRowItr) Next(ctx *sql.Context) (sql.Row, erro
 	meta := itr.meta
 	h := itr.hash
 	col := tableChange.colNames[itr.colIdx]
+	diffType := tableChange.diffTypes[itr.colIdx]
 
 	return sql.NewRow(
 		h.String(),
@@ -378,13 +388,11 @@ func (itr *doltColDiffCommitHistoryRowItr) Next(ctx *sql.Context) (sql.Row, erro
 		meta.Email,
 		meta.Time(),
 		meta.Description,
-		tableChange.dataChange,
-		tableChange.schemaChange,
+		diffType,
 	), nil
 }
 
-// loadTableChanges loads the current commit's table changes and metadata
-// into the iterator.
+// loadTableChanges loads the current commit's table changes and metadata into the iterator.
 func (itr *doltColDiffCommitHistoryRowItr) loadTableChanges(ctx context.Context, commit *doltdb.Commit) error {
 	tableChanges, err := itr.calculateTableChanges(ctx, commit)
 	if err != nil {
@@ -440,13 +448,25 @@ func (itr *doltColDiffCommitHistoryRowItr) calculateTableChanges(ctx context.Con
 	}
 
 	tableChanges := make([]tableColChange, len(deltas))
+	tableChangeIdx := 0
 	for i := 0; i < len(deltas); i++ {
 		change, err := processTableColDelta(itr.ctx, itr.ddb, deltas[i])
 		if err != nil {
 			return nil, err
 		}
 
-		tableChanges[i] = *change
+		// filter out changes that have no modified columns
+		if len(change.colNames) != 0 {
+			tableChanges[tableChangeIdx] = *change
+			tableChangeIdx++
+		} else {
+			// remove table changes that don't have any modified columns
+			if tableChangeIdx == len(tableChanges)-1 {
+				tableChanges = tableChanges[:tableChangeIdx]
+			} else {
+				tableChanges = append(tableChanges[:tableChangeIdx], tableChanges[tableChangeIdx+1:]...)
+			}
+		}
 	}
 
 	// Not all commits mutate tables (e.g. empty commits)
@@ -465,10 +485,9 @@ func (itr *doltColDiffCommitHistoryRowItr) Close(*sql.Context) error {
 // tableColChange is an internal data structure used to hold the results of processing
 // a diff.TableDelta structure into the output data for this system table.
 type tableColChange struct {
-	tableName    string
-	colNames     []string
-	dataChange   bool
-	schemaChange bool
+	tableName string
+	colNames  []string
+	diffTypes []string
 }
 
 // processTableColDelta processes the specified TableDelta to determine what kind of change it was (i.e. table drop,
@@ -476,60 +495,35 @@ type tableColChange struct {
 func processTableColDelta(ctx *sql.Context, ddb *doltdb.DoltDB, delta diff.TableDelta) (*tableColChange, error) {
 	// Dropping a table is always a schema change, and also a data change if the table contained data
 	if delta.IsDrop() {
-		isEmpty, err := isTableDataEmpty(ctx, delta.FromTable)
-		if err != nil {
-			return nil, err
+		diffTypes := make([]string, delta.FromSch.GetAllCols().Size())
+		for i := range diffTypes {
+			diffTypes[i] = diffTypeRemoved
 		}
 
 		return &tableColChange{
-			tableName:    delta.FromName,
-			colNames:     delta.FromSch.GetAllCols().GetColumnNames(),
-			dataChange:   !isEmpty,
-			schemaChange: true,
+			tableName: delta.FromName,
+			colNames:  delta.FromSch.GetAllCols().GetColumnNames(),
+			diffTypes: diffTypes,
 		}, nil
 	}
 
-	// Renaming a table is always a schema change, and also a data change if the table data differs
-	if delta.IsRename() {
-		dataChanged, err := delta.HasHashChanged()
-		if err != nil {
-			return nil, err
-		}
-
-		return &tableColChange{
-			tableName:    delta.ToName,
-			colNames:     delta.ToSch.GetAllCols().GetColumnNames(),
-			dataChange:   dataChanged,
-			schemaChange: true,
-		}, nil
-	}
+	// Renaming a table does not affect columns necessarily, if table data was changed it will be checked below
 
 	// Creating a table is always a schema change, and also a data change if data was inserted
 	if delta.IsAdd() {
-		isEmpty, err := isTableDataEmpty(ctx, delta.ToTable)
-		if err != nil {
-			return nil, err
+		diffTypes := make([]string, delta.ToSch.GetAllCols().Size())
+		for i := range diffTypes {
+			diffTypes[i] = diffTypeAdded
 		}
 
 		return &tableColChange{
-			tableName:    delta.ToName,
-			colNames:     delta.ToSch.GetAllCols().GetColumnNames(),
-			dataChange:   !isEmpty,
-			schemaChange: true,
+			tableName: delta.ToName,
+			colNames:  delta.ToSch.GetAllCols().GetColumnNames(),
+			diffTypes: diffTypes,
 		}, nil
 	}
 
-	dataChanged, err := delta.HasHashChanged()
-	if err != nil {
-		return nil, err
-	}
-
-	schemaChanged, err := delta.HasSchemaChanged(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	// calculate which columns have changed
+	// calculate which columns have been modified
 	diffTableSchema, j, err := GetDiffTableSchemaAndJoiner(delta.ToTable.Format(), delta.FromSch, delta.ToSch)
 	if err != nil {
 		return nil, err
@@ -539,111 +533,123 @@ func processTableColDelta(ctx *sql.Context, ddb *doltdb.DoltDB, delta diff.Table
 	now := time.Now()
 	dp := NewDiffPartition(delta.ToTable, delta.FromTable, delta.ToName, delta.FromName, (*dtypes.Timestamp)(&now), (*dtypes.Timestamp)(&now), delta.ToSch, delta.FromSch)
 	ri := NewDiffPartitionRowIter(*dp, ddb, j)
-	colNames, err := calculateColDelta(ctx, delta, ri, diffTableSchema)
+	colNames, diffTypes, err := calculateColDelta(ctx, ri, delta.ToSch.GetAllCols(), delta.FromSch.GetAllCols(), diffTableSchema.GetAllCols())
 	if err != nil {
 		return nil, err
 	}
 
 	return &tableColChange{
-		tableName:    delta.ToName,
-		colNames:     colNames,
-		dataChange:   dataChanged,
-		schemaChange: schemaChanged,
+		tableName: delta.ToName,
+		colNames:  colNames,
+		diffTypes: diffTypes,
 	}, nil
 }
 
-func calculateColDelta(ctx *sql.Context, delta diff.TableDelta, iter sql.RowIter, diffTableSchema schema.Schema) ([]string, error) {
-	diffPkSch, err := sqlutil.FromDoltSchema("", diffTableSchema)
-	if err != nil {
-		return nil, err
+// calculateColDelta iterates through the rows of the given table delta and compares each cell in the to_ and from_
+// cells to compile a list of modified columns
+func calculateColDelta(ctx *sql.Context, iter sql.RowIter, toCols, fromCols, diffTableCols *schema.ColCollection) ([]string, []string, error) {
+	colNamesSet := make(map[string]struct{})
+	var resultColNames []string
+	var resultDiffTypes []string
+	modifiedCols, addedCols, droppedCols, _, _ := calculateColSchemaDiff(toCols, fromCols)
+
+	// add all added/dropped columns to result
+	for _, col := range addedCols {
+		resultColNames = append(resultColNames, col)
+		resultDiffTypes = append(resultDiffTypes, diffTypeAdded)
+	}
+	for _, col := range droppedCols {
+		resultColNames = append(resultColNames, col)
+		resultDiffTypes = append(resultDiffTypes, diffTypeRemoved)
 	}
-	columnsWithDiff := getColumnNamesWithDiff(delta.FromSch, delta.ToSch)
-	_, projections := getDiffSqlSchema(diffPkSch.Schema, columnsWithDiff)
-
-	columns := make(map[string]string)
-	var result []string
-	numCols := (len(projections) - 1) / 2
 
+	// check each row for diffs in modified columns
 	for {
 		r, err := iter.Next(ctx)
 		if err == io.EOF {
-			for key := range columns {
-				// returning only names of modified columns
-				result = append(result, key)
+			for col := range colNamesSet {
+				// append modified columns to result
+				resultColNames = append(resultColNames, col)
+				resultDiffTypes = append(resultDiffTypes, diffTypeModified)
 			}
-			return result, nil
+			return resultColNames, resultDiffTypes, nil
 		} else if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
-		i := 0
-		for i < numCols {
-			// compare the to_ and from_ cells, accounting for to_commit and to_commit_date columns
-			if r[i] != r[i+numCols+2] {
-				cleanedName := strings.Split(projections[i].(*expression.GetField).Name(), "_")[1]
-				columns[cleanedName] = ""
+		// only need to check modified columns
+		for _, col := range modifiedCols {
+			toColTag := diffTableCols.NameToCol["to_"+col].Tag
+			fromColTag := diffTableCols.NameToCol["from_"+col].Tag
+			toIdx := diffTableCols.TagToIdx[toColTag]
+			fromIdx := diffTableCols.TagToIdx[fromColTag]
+
+			if r[toIdx] != r[fromIdx] {
+				colNamesSet[col] = struct{}{}
 			}
-			i++
 		}
 
-		// can stop checking rows when we already have all columns in the result set
-		if len(columns) == numCols {
-			for key := range columns {
-				// returning only names of modified columns
-				result = append(result, key)
+		// can stop checking rows when we already have all modified columns in the result set
+		if len(colNamesSet) == len(modifiedCols) {
+			for col := range colNamesSet {
+				// append modified columns to result
+				resultColNames = append(resultColNames, col)
+				resultDiffTypes = append(resultDiffTypes, diffTypeModified)
 			}
-			return result, nil
+			return resultColNames, resultDiffTypes, nil
 		}
 	}
 }
 
-// getColumnNamesWithDiff attaches the to_ and from_ prefixes to all columns
-func getColumnNamesWithDiff(fromSch, toSch schema.Schema) []string {
-	var cols []string
-
-	if fromSch != nil {
-		_ = fromSch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) {
-			cols = append(cols, fmt.Sprintf("from_%s", col.Name))
-			return false, nil
-		})
+// calculateColSchemaDiff calculates which columns were modified, added, or dropped between to and from schemas and
+// returns a list of column names for each type of change, the total list of column names, and a corresponding list of
+// diff_types for each column
+func calculateColSchemaDiff(toCols *schema.ColCollection, fromCols *schema.ColCollection) ([]string, []string, []string, []string, []string) {
+	// put to/from columns into a set
+	toColTags := make(map[uint64]struct{})
+	fromColTags := make(map[uint64]struct{})
+	if toCols != nil {
+		for _, tag := range toCols.Tags {
+			toColTags[tag] = struct{}{}
+		}
 	}
-	if toSch != nil {
-		_ = toSch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) {
-			cols = append(cols, fmt.Sprintf("to_%s", col.Name))
-			return false, nil
-		})
-	}
-	return cols
-}
-
-// getDiffSqlSchema returns the schema of columns with data diff and "diff_type". This is used for diff splitter.
-// When extracting the diff schema, the ordering must follow the ordering of given columns
-func getDiffSqlSchema(diffTableSch sql.Schema, columns []string) (sql.Schema, []sql.Expression) {
-	type column struct {
-		sqlCol *sql.Column
-		idx    int
-	}
-
-	columns = append(columns, "diff_type")
-	colMap := make(map[string]*column)
-	for _, c := range columns {
-		colMap[c] = nil
-	}
-
-	var cols = make([]*sql.Column, len(columns))
-	var getFieldCols = make([]sql.Expression, len(columns))
-
-	for i, c := range diffTableSch {
-		if _, ok := colMap[c.Name]; ok {
-			colMap[c.Name] = &column{c, i}
+	if fromCols != nil {
+		for _, tag := range fromCols.Tags {
+			fromColTags[tag] = struct{}{}
 		}
 	}
 
-	for i, c := range columns {
-		col := colMap[c].sqlCol
-		cols[i] = col
-		getFieldCols[i] = expression.NewGetField(colMap[c].idx, col.Type, col.Name, col.Nullable)
+	var modifiedCols []string
+	var addedCols []string
+	var droppedCols []string
+	var allCols []string
+	var diffTypes []string
+
+	if toCols != nil {
+		for _, tag := range toCols.Tags {
+			if _, ok := fromColTags[tag]; ok {
+				// if the tag is also in fromColumnTags, this column was modified
+				modifiedCols = append(modifiedCols, toCols.TagToCol[tag].Name)
+				allCols = append(allCols, toCols.TagToCol[tag].Name)
+				diffTypes = append(diffTypes, diffTypeModified)
+				delete(fromColTags, tag)
+			} else {
+				// else if it isn't in fromColumnTags, this column was added
+				addedCols = append(addedCols, toCols.TagToCol[tag].Name)
+				allCols = append(allCols, toCols.TagToCol[tag].Name)
+				diffTypes = append(diffTypes, diffTypeAdded)
+			}
+		}
 	}
 
-	return cols, getFieldCols
+	if fromCols != nil {
+		for tag, _ := range fromColTags {
+			// all remaining tags are columns not in toColumnTags, i.e. dropped columns
+			droppedCols = append(droppedCols, fromCols.TagToCol[tag].Name)
+			allCols = append(allCols, fromCols.TagToCol[tag].Name)
+			diffTypes = append(diffTypes, diffTypeRemoved)
+		}
+	}
+
+	return modifiedCols, addedCols, droppedCols, allCols, diffTypes
 }

go/libraries/doltcore/sqle/dtables/diff_table.go

@@ -944,7 +944,7 @@ func CalculateDiffSchema(fromSch, toSch schema.Schema) (schema.Schema, error) {
 
 	j := toSch.GetAllCols().Size()
 	err = fromSch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) {
-		fromCol, err := schema.NewColumnWithTypeInfo(diff.FromColNamer(col.Name), uint64(i), col.TypeInfo, false, col.Default, false, col.Comment)
+		fromCol, err := schema.NewColumnWithTypeInfo(diff.FromColNamer(col.Name), uint64(j), col.TypeInfo, false, col.Default, false, col.Comment)
 		if err != nil {
 			return true, err
 		}

go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go

@@ -48,7 +48,7 @@ var skipPrepared bool
 // SkipPreparedsCount is used by the "ci-check-repo CI workflow
 // as a reminder to consider prepareds when adding a new
 // enginetest suite.
-const SkipPreparedsCount = 83
+const SkipPreparedsCount = 82
 
 const skipPreparedFlag = "DOLT_SKIP_PREPARED_ENGINETESTS"
 
@@ -1225,16 +1225,16 @@ func TestUnscopedDiffSystemTable(t *testing.T) {
 	}
 }
 
-func TestColumnDiffSystemTable(t *testing.T) {
-	for _, test := range ColumnDiffSystemTableScriptTests {
+func TestUnscopedDiffSystemTablePrepared(t *testing.T) {
+	for _, test := range UnscopedDiffSystemTableScriptTests {
 		t.Run(test.Name, func(t *testing.T) {
-			enginetest.TestScript(t, newDoltHarness(t), test)
+			enginetest.TestScriptPrepared(t, newDoltHarness(t), test)
 		})
 	}
 }
 
-func TestUnscopedDiffSystemTablePrepared(t *testing.T) {
-	for _, test := range UnscopedDiffSystemTableScriptTests {
+func TestColumnDiffSystemTablePrepared(t *testing.T) {
+	for _, test := range ColumnDiffSystemTableScriptTests {
 		t.Run(test.Name, func(t *testing.T) {
 			enginetest.TestScriptPrepared(t, newDoltHarness(t), test)
 		})

go/libraries/doltcore/sqle/enginetest/dolt_queries_diff.go

@@ -3681,86 +3681,421 @@ var ColumnDiffSystemTableScriptTests = []queries.ScriptTest{
 	{
 		Name: "working set changes",
 		SetUpScript: []string{
-			"create table regularTable (a int primary key, b int, c int);",
-			"create table droppedTable (a int primary key, b int, c int);",
-			"create table renamedEmptyTable (a int primary key, b int, c int);",
+			"create table regularTable (a int primary key, b int);",
+			"create table droppedTable (a int primary key, b int);",
+			"insert into droppedTable values (1, 2), (2, 3);",
+			"create table renamedTable (a int primary key, b int);",
 			"call dolt_add('.')",
-			"insert into regularTable values (1, 2, 3), (2, 3, 4);",
-			"insert into droppedTable values (1, 2, 3), (2, 3, 4);",
-			"set @Commit1 = '';",
-			"CALL DOLT_COMMIT_HASH_OUT(@Commit1, '-am', 'Creating tables x and y');",
 
-			// changeSet: STAGED; data change: false; schema change: true
-			"create table addedTable (a int primary key, b int, c int);",
-			"call DOLT_ADD('addedTable');",
-			// changeSet: STAGED; data change: true; schema change: true
+			"insert into regularTable values (1, 2), (2, 3);",
 			"drop table droppedTable;",
-			"call DOLT_ADD('droppedTable');",
-			// changeSet: WORKING; data change: false; schema change: true
-			"rename table renamedEmptyTable to newRenamedEmptyTable;",
-			// changeSet: WORKING; data change: true; schema change: false
-			"insert into regularTable values (3, 4, 5);",
+			"rename table renamedTable to newRenamedTable;",
+			"create table addedTable (a int primary key, b int);",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query: "SELECT commit_hash, table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE table_name = 'regularTable' ORDER BY commit_hash, table_name, column_name;",
+				Expected: []sql.Row{
+					{"STAGED", "regularTable", "a", "added"},
+					{"STAGED", "regularTable", "b", "added"},
+					{"WORKING", "regularTable", "a", "modified"},
+					{"WORKING", "regularTable", "b", "modified"},
+				},
+			},
+			{
+				Query: "SELECT commit_hash, table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE table_name = 'droppedTable' ORDER BY commit_hash, table_name, column_name;",
+				Expected: []sql.Row{
+					{"STAGED", "droppedTable", "a", "added"},
+					{"STAGED", "droppedTable", "b", "added"},
+					{"WORKING", "droppedTable", "a", "removed"},
+					{"WORKING", "droppedTable", "b", "removed"},
+				},
+			},
+			{
+				Query: "SELECT commit_hash, table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE table_name = 'renamedTable' OR table_name = 'newRenamedTable' ORDER BY commit_hash, table_name, column_name;",
+				Expected: []sql.Row{
+					{"STAGED", "renamedTable", "a", "added"},
+					{"STAGED", "renamedTable", "b", "added"},
+					{"WORKING", "newRenamedTable", "a", "modified"},
+					{"WORKING", "newRenamedTable", "b", "modified"},
+				},
+			},
+			{
+				Query: "SELECT commit_hash, table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE table_name = 'addedTable' ORDER BY commit_hash, table_name, column_name;",
+				Expected: []sql.Row{
+					{"WORKING", "addedTable", "a", "added"},
+					{"WORKING", "addedTable", "b", "added"},
+				},
+			},
+		},
+	},
+	{
+		Name: "add column",
+		SetUpScript: []string{
+			"create table t (pk int primary key, c int);",
+			"call dolt_add('.')",
+			"insert into t values (1, 2), (3, 4);",
+			"set @Commit1 = '';",
+			"call dolt_commit_hash_out(@Commit1, '-am', 'creating table t');",
+
+			"alter table t add column d int;",
+			"set @Commit2 = '';",
+			"update t set d = 6 where pk = 1;",
+			"call dolt_add('.')",
+			"call dolt_commit_hash_out(@Commit2, '-m', 'updating d in t');",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query:    "select count(*) from dolt_column_diff where commit_hash = @Commit1;",
+				Expected: []sql.Row{{2}},
+			},
+			{
+				Query:    "select table_name, column_name from dolt_column_diff where commit_hash = @Commit2;",
+				Expected: []sql.Row{{"t", "d"}},
+			},
+		},
+	},
+	{
+		Name: "modify column",
+		SetUpScript: []string{
+			"create table t (pk int primary key, c int);",
+			"call dolt_add('.')",
+			"insert into t values (1, 2), (3, 4);",
+			"set @Commit1 = '';",
+			"call dolt_commit_hash_out(@Commit1, '-am', 'creating table t');",
+
+			"update t set c = 5 where pk = 3;",
+			"call dolt_add('.')",
+			"set @Commit2 = '';",
+			"call dolt_commit_hash_out(@Commit2, '-am', 'updating value in t');",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query:    "select count(*) from dolt_column_diff where commit_hash = @Commit1;",
+				Expected: []sql.Row{{2}},
+			},
+			{
+				Query:    "select table_name, column_name, diff_type from dolt_column_diff where commit_hash = @Commit2;",
+				Expected: []sql.Row{{"t", "c", "modified"}},
+			},
+		},
+	},
+	{
+		Name: "drop column",
+		SetUpScript: []string{
+			"create table t (pk int primary key, c int);",
+			"call dolt_add('.')",
+			"insert into t values (1, 2), (3, 4);",
+			"set @Commit1 = '';",
+			"call dolt_commit_hash_out(@Commit1, '-am', 'creating table t');",
+
+			"alter table t drop column c;",
+			"call dolt_add('.')",
+			"set @Commit2 = '';",
+			"call dolt_commit_hash_out(@Commit2, '-am', 'dropping column c in t');",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query:    "select count(*) from dolt_column_diff where commit_hash = @Commit1;",
+				Expected: []sql.Row{{2}},
+			},
+			{
+				Query:    "select table_name, column_name, diff_type from dolt_column_diff where commit_hash = @Commit2;",
+				Expected: []sql.Row{{"t", "c", "removed"}},
+			},
+		},
+	},
+	{
+		Name: "drop column and recreate with same type",
+		SetUpScript: []string{
+			"create table t (pk int primary key, c int);",
+			"call dolt_add('.')",
+			"insert into t values (1, 2), (3, 4);",
+			"set @Commit1 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit1, '-am', 'creating table t');",
+
+			"alter table t drop column c;",
+			"set @Commit2 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit2, '-am', 'dropping column c');",
+
+			"alter table t add column c int;",
+			"insert into t values (100, 101);",
+			"set @Commit3 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit3, '-am', 'inserting into t');",
 		},
 		Assertions: []queries.ScriptTestAssertion{
 			{
 				Query:    "SELECT COUNT(*) FROM DOLT_COLUMN_DIFF;",
-				Expected: []sql.Row{{21}},
+				Expected: []sql.Row{{5}},
 			},
 			{
-				Query:    "SELECT COUNT(*) FROM DOLT_COLUMN_DIFF WHERE commit_hash = @Commit1;",
-				Expected: []sql.Row{{9}},
-			},
-			{
-				Query:    "SELECT * FROM DOLT_COLUMN_DIFF WHERE commit_hash = @Commit1 AND committer <> 'billy bob';",
-				Expected: []sql.Row{},
-			},
-			{
-				Query:    "SELECT commit_hash, committer FROM DOLT_COLUMN_DIFF WHERE commit_hash <> @Commit1 AND committer = 'billy bob' AND commit_hash NOT IN ('WORKING','STAGED');",
-				Expected: []sql.Row{},
-			},
-			{
-				Query: "SELECT commit_hash, table_name, column_name FROM DOLT_COLUMN_DIFF WHERE commit_hash <> @Commit1 AND commit_hash NOT IN ('STAGED') ORDER BY table_name;",
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit1;",
 				Expected: []sql.Row{
-					{"WORKING", "newRenamedEmptyTable", "a"},
-					{"WORKING", "newRenamedEmptyTable", "b"},
-					{"WORKING", "newRenamedEmptyTable", "c"},
-					{"WORKING", "regularTable", "a"},
-					{"WORKING", "regularTable", "b"},
-					{"WORKING", "regularTable", "c"},
+					{"t", "pk", "added"},
+					{"t", "c", "added"},
 				},
 			},
 			{
-				Query: "SELECT commit_hash, table_name, column_name FROM DOLT_COLUMN_DIFF WHERE commit_hash <> @Commit1 OR committer <> 'billy bob' ORDER BY table_name;",
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit2;",
 				Expected: []sql.Row{
-					{"STAGED", "addedTable", "a"},
-					{"STAGED", "addedTable", "b"},
-					{"STAGED", "addedTable", "c"},
-					{"STAGED", "droppedTable", "a"},
-					{"STAGED", "droppedTable", "b"},
-					{"STAGED", "droppedTable", "c"},
-					{"WORKING", "newRenamedEmptyTable", "a"},
-					{"WORKING", "newRenamedEmptyTable", "b"},
-					{"WORKING", "newRenamedEmptyTable", "c"},
-					{"WORKING", "regularTable", "a"},
-					{"WORKING", "regularTable", "b"},
-					{"WORKING", "regularTable", "c"},
+					{"t", "c", "removed"},
 				},
 			},
 			{
-				Query: "SELECT * FROM DOLT_COLUMN_DIFF WHERE COMMIT_HASH in ('WORKING', 'STAGED') ORDER BY table_name;",
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit3",
 				Expected: []sql.Row{
-					{"STAGED", "addedTable", "a", nil, nil, nil, nil, false, true},
-					{"STAGED", "addedTable", "b", nil, nil, nil, nil, false, true},
-					{"STAGED", "addedTable", "c", nil, nil, nil, nil, false, true},
-					{"STAGED", "droppedTable", "a", nil, nil, nil, nil, true, true},
-					{"STAGED", "droppedTable", "b", nil, nil, nil, nil, true, true},
-					{"STAGED", "droppedTable", "c", nil, nil, nil, nil, true, true},
-					{"WORKING", "newRenamedEmptyTable", "a", nil, nil, nil, nil, false, true},
-					{"WORKING", "newRenamedEmptyTable", "b", nil, nil, nil, nil, false, true},
-					{"WORKING", "newRenamedEmptyTable", "c", nil, nil, nil, nil, false, true},
-					{"WORKING", "regularTable", "a", nil, nil, nil, nil, true, false},
-					{"WORKING", "regularTable", "b", nil, nil, nil, nil, true, false},
-					{"WORKING", "regularTable", "c", nil, nil, nil, nil, true, false},
+					{"t", "pk", "modified"},
+					{"t", "c", "added"},
+				},
+			},
+		},
+	},
+	{
+		Name: "drop column, then rename column with same type to same name",
+		SetUpScript: []string{
+			"create table t (pk int primary key, c1 int, c2 int);",
+			"call dolt_add('.')",
+			"insert into t values (1, 2, 3), (4, 5, 6);",
+			"set @Commit1 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit1, '-am', 'creating table t');",
+
+			"alter table t drop column c1;",
+			"set @Commit2 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit2, '-am', 'dropping column c1');",
+
+			"alter table t rename column c2 to c1;",
+			"insert into t values (100, 101);",
+			"set @Commit3 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit3, '-am', 'inserting into t');",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query:    "SELECT COUNT(*) FROM DOLT_COLUMN_DIFF;",
+				Expected: []sql.Row{{6}},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit1;",
+				Expected: []sql.Row{
+					{"t", "pk", "added"},
+					{"t", "c1", "added"},
+					{"t", "c2", "added"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit2;",
+				Expected: []sql.Row{
+					{"t", "c1", "removed"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit3;",
+				Expected: []sql.Row{
+					{"t", "pk", "modified"},
+					{"t", "c1", "modified"},
+				},
+			},
+		},
+	},
+	{
+		Name: "column drop and recreate with different type that can be coerced (int -> string)",
+		SetUpScript: []string{
+			"create table t (pk int primary key, c int);",
+			"call dolt_add('.')",
+			"insert into t values (1, 2), (3, 4);",
+			"set @Commit1 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit1, '-am', 'creating table t');",
+
+			"alter table t drop column c;",
+			"set @Commit2 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit2, '-am', 'dropping column c');",
+
+			"alter table t add column c varchar(20);",
+			"insert into t values (100, '101');",
+			"set @Commit3 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit3, '-am', 're-adding column c');",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query:    "SELECT COUNT(*) FROM DOLT_COLUMN_DIFF;",
+				Expected: []sql.Row{{5}},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit1;",
+				Expected: []sql.Row{
+					{"t", "pk", "added"},
+					{"t", "c", "added"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit2;",
+				Expected: []sql.Row{
+					{"t", "c", "removed"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit3;",
+				Expected: []sql.Row{
+					{"t", "pk", "modified"},
+					{"t", "c", "added"},
+				},
+			},
+		},
+	},
+	{
+		Name: "column drop and recreate with different type that can NOT be coerced (string -> int)",
+		SetUpScript: []string{
+			"create table t (pk int primary key, c varchar(20));",
+			"call dolt_add('.')",
+			"insert into t values (1, 'two'), (3, 'four');",
+			"set @Commit1 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit1, '-am', 'creating table t');",
+
+			"alter table t drop column c;",
+			"set @Commit2 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit2, '-am', 'dropping column c');",
+
+			"alter table t add column c int;",
+			"insert into t values (100, 101);",
+			"set @Commit3 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit3, '-am', 're-adding column c');",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query:    "SELECT COUNT(*) FROM DOLT_COLUMN_DIFF;",
+				Expected: []sql.Row{{5}},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit1;",
+				Expected: []sql.Row{
+					{"t", "pk", "added"},
+					{"t", "c", "added"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit2;",
+				Expected: []sql.Row{
+					{"t", "c", "removed"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit3;",
+				Expected: []sql.Row{
+					{"t", "pk", "modified"},
+					{"t", "c", "added"},
+				},
+			},
+		},
+	},
+	{
+		Name: "multiple column renames",
+		SetUpScript: []string{
+			"create table t (pk int primary key, c1 int);",
+			"call dolt_add('.')",
+			"insert into t values (1, 2);",
+			"set @Commit1 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit1, '-am', 'creating table t');",
+
+			"alter table t rename column c1 to c2;",
+			"insert into t values (3, 4);",
+			"set @Commit2 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit2, '-am', 'renaming c1 to c2');",
+
+			"alter table t drop column c2;",
+			"set @Commit3 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit3, '-am', 'dropping column c2');",
+
+			"alter table t add column c2 int;",
+			"insert into t values (100, '101');",
+			"set @Commit4 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit4, '-am', 'recreating column c2');",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query:    "SELECT COUNT(*) FROM DOLT_COLUMN_DIFF;",
+				Expected: []sql.Row{{7}},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit1;",
+				Expected: []sql.Row{
+					{"t", "pk", "added"},
+					{"t", "c1", "added"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit2;",
+				Expected: []sql.Row{
+					{"t", "pk", "modified"},
+					{"t", "c2", "modified"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit3;",
+				Expected: []sql.Row{
+					{"t", "c2", "removed"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit4;",
+				Expected: []sql.Row{
+					{"t", "pk", "modified"},
+					{"t", "c2", "added"},
+				},
+			},
+		},
+	},
+	{
+		Name: "primary key change",
+		SetUpScript: []string{
+			"create table t (pk int primary key, c1 int);",
+			"call dolt_add('.')",
+			"insert into t values (1, 2), (3, 4);",
+			"set @Commit1 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit1, '-am', 'creating table t');",
+
+			"alter table t drop primary key;",
+			"insert into t values (5, 6);",
+			"set @Commit2 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit2, '-am', 'dropping primary key');",
+
+			"alter table t add primary key (c1);",
+			"set @Commit3 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit3, '-am', 'adding primary key');",
+
+			"insert into t values (7, 8);",
+			"set @Commit4 = '';",
+			"CALL DOLT_COMMIT_HASH_OUT(@Commit4, '-am', 'adding more data');",
+		},
+		Assertions: []queries.ScriptTestAssertion{
+			{
+				Query:    "SELECT COUNT(*) FROM DOLT_COLUMN_DIFF;",
+				Expected: []sql.Row{{8}},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit1;",
+				Expected: []sql.Row{
+					{"t", "pk", "added"},
+					{"t", "c1", "added"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit2;",
+				Expected: []sql.Row{
+					{"t", "pk", "modified"},
+					{"t", "c1", "modified"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit3;",
+				Expected: []sql.Row{
+					{"t", "pk", "modified"},
+					{"t", "c1", "modified"},
+				},
+			},
+			{
+				Query: "SELECT table_name, column_name, diff_type FROM DOLT_COLUMN_DIFF WHERE commit_hash=@Commit4;",
+				Expected: []sql.Row{
+					{"t", "pk", "modified"},
+					{"t", "c1", "modified"},
 				},
 			},
 		},

integration-tests/bats/system-tables.bats

@@ -327,10 +327,10 @@ SQL
 
     run dolt sql -r csv -q 'select * from dolt_column_diff'
     [ "$status" -eq 0 ]
-    [[ "$output" =~ "STAGED,testStaged,pk,,,,,false,true" ]] || false
-    [[ "$output" =~ "STAGED,testStaged,c1,,,,,false,true" ]] || false
-    [[ "$output" =~ "WORKING,testWorking,pk,,,,,false,true" ]] || false
-    [[ "$output" =~ "WORKING,testWorking,c1,,,,,false,true" ]] || false
+    [[ "$output" =~ "STAGED,testStaged,pk,,,,,added" ]] || false
+    [[ "$output" =~ "STAGED,testStaged,c1,,,,,added" ]] || false
+    [[ "$output" =~ "WORKING,testWorking,pk,,,,,added" ]] || false
+    [[ "$output" =~ "WORKING,testWorking,c1,,,,,added" ]] || false
 }
 
 @test "system-tables: query dolt_diff_ system table" {