Fixed some bugs in serialization / deserialization, getting started on converting virtual columns on store / read

go/libraries/doltcore/doltdb/durable/artifact_index.go

@@ -50,7 +50,7 @@ func RefFromArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, idx Ar
 		return refFromNomsValue(ctx, vrw, b)
 
 	default:
-		return types.Ref{}, errNbfUnkown
+		return types.Ref{}, errNbfUnknown
 	}
 }
 
@@ -69,7 +69,7 @@ func NewEmptyArtifactIndex(ctx context.Context, vrw types.ValueReadWriter, ns tr
 		return ArtifactIndexFromProllyMap(m), nil
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 
@@ -107,7 +107,7 @@ func artifactIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tr
 		return ArtifactIndexFromProllyMap(m), nil
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 

go/libraries/doltcore/doltdb/durable/conflict_index.go

@@ -40,7 +40,7 @@ func RefFromConflictIndex(ctx context.Context, vrw types.ValueReadWriter, idx Co
 		return types.Ref{}, fmt.Errorf("__DOLT__ conflicts should be stored in ArtifactIndex")
 
 	default:
-		return types.Ref{}, errNbfUnkown
+		return types.Ref{}, errNbfUnknown
 	}
 }
 
@@ -58,7 +58,7 @@ func NewEmptyConflictIndex(ctx context.Context, vrw types.ValueReadWriter, ns tr
 		return nil, fmt.Errorf("__DOLT__ conflicts should be stored in ArtifactIndex")
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 
@@ -91,7 +91,7 @@ func conflictIndexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tr
 		return nil, fmt.Errorf("__DOLT__ conflicts should be stored in ArtifactIndex")
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 

go/libraries/doltcore/doltdb/durable/index.go

@@ -88,7 +88,7 @@ func RefFromIndex(ctx context.Context, vrw types.ValueReadWriter, idx Index) (ty
 		return refFromNomsValue(ctx, vrw, b)
 
 	default:
-		return types.Ref{}, errNbfUnkown
+		return types.Ref{}, errNbfUnknown
 	}
 }
 
@@ -115,7 +115,7 @@ func indexFromAddr(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 		return IndexFromProllyMap(pm), nil
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 
@@ -138,7 +138,7 @@ func NewEmptyIndex(ctx context.Context, vrw types.ValueReadWriter, ns tree.NodeS
 		return IndexFromProllyMap(m), nil
 
 	default:
-		return nil, errNbfUnkown
+		return nil, errNbfUnknown
 	}
 }
 

go/libraries/doltcore/doltdb/durable/table.go

@@ -52,7 +52,7 @@ var (
 )
 
 var (
-	errNbfUnkown      = fmt.Errorf("unknown NomsBinFormat")
+	errNbfUnknown     = fmt.Errorf("unknown NomsBinFormat")
 	errNbfUnsupported = fmt.Errorf("operation unsupported for NomsBinFormat")
 )
 

go/libraries/doltcore/schema/encoding/serialization.go

@@ -304,9 +304,11 @@ func deserializeColumns(ctx context.Context, s *serial.TableSchema) ([]schema.Co
 		defVal := ""
 		generatedVal := ""
 		if c.DefaultValue() != nil {
-			defVal = string(c.DefaultValue())
-		} else {
-			generatedVal = string(c.DefaultValue())
+			if c.Generated() {
+				generatedVal = string(c.DefaultValue())
+			} else {
+				defVal = string(c.DefaultValue())
+			}
 		}
 		
 		cols[i], err = schema.NewColumnWithTypeInfo(

go/libraries/doltcore/schema/schema_impl.go

@@ -417,6 +417,10 @@ func (si *schemaImpl) GetKeyDescriptor() val.TupleDesc {
 	useCollations := false // We only use collations if a string exists
 	var collations []sql.CollationID
 	_ = si.GetPKCols().Iter(func(tag uint64, col Column) (stop bool, err error) {
+		if col.Virtual {
+			return
+		} 
+		
 		sqlType := col.TypeInfo.ToSqlType()
 		queryType := sqlType.Type()
 		var t val.Type
@@ -473,6 +477,10 @@ func (si *schemaImpl) GetValueDescriptor() val.TupleDesc {
 
 	useCollations := false // We only use collations if a string exists
 	_ = si.GetNonPKCols().Iter(func(tag uint64, col Column) (stop bool, err error) {
+		if col.Virtual {
+			return
+		}
+		
 		sqlType := col.TypeInfo.ToSqlType()
 		queryType := sqlType.Type()
 		tt = append(tt, val.Type{

go/libraries/doltcore/sqle/index/prolly_row_iter.go

@@ -32,7 +32,7 @@ type prollyRowIter struct {
 
 	keyProj []int
 	valProj []int
-	// orjProj is a concatenated list of output ordinals for |keyProj| and |valProj|
+	// ordProj is a concatenated list of output ordinals for |keyProj| and |valProj|
 	ordProj []int
 	rowLen  int
 }

go/libraries/doltcore/sqle/sqlutil/convert.go

@@ -40,21 +40,26 @@ func FromDoltSchema(dbName, tableName string, sch schema.Schema) (sql.PrimaryKey
 			extra = "auto_increment"
 		}
 
-		var deflt *sql.ColumnDefaultValue
+		var deflt, generated *sql.ColumnDefaultValue
 		if col.Default != "" {
 			deflt = sql.NewUnresolvedColumnDefaultValue(col.Default)
 		}
-
+		if col.Generated != "" {
+			generated = sql.NewUnresolvedColumnDefaultValue(col.Generated)
+		}
+		
 		cols[i] = &sql.Column{
 			Name:           col.Name,
 			Type:           sqlType,
 			Default:        deflt,
+			Generated:      generated,
 			Nullable:       col.IsNullable(),
 			DatabaseSource: dbName,
 			Source:         tableName,
 			PrimaryKey:     col.IsPartOfPK,
 			AutoIncrement:  col.AutoIncrement,
 			Comment:        col.Comment,
+			Virtual:  			col.Virtual,
 			Extra:          extra,
 		}
 		i++
@@ -137,7 +142,26 @@ func ToDoltCol(tag uint64, col *sql.Column) (schema.Column, error) {
 		return schema.Column{}, err
 	}
 
-	return schema.NewColumnWithTypeInfo(col.Name, tag, typeInfo, col.PrimaryKey, col.Default.String(), "", false, col.AutoIncrement, col.Comment, constraints...)
+	defaultVal := ""
+	generatedVal := ""
+	if col.Default != nil {
+		defaultVal = col.Default.String()
+	} else {
+		generatedVal = col.Generated.String()
+	}
+	
+	return schema.NewColumnWithTypeInfo(
+		col.Name,
+		tag,
+		typeInfo,
+		col.PrimaryKey,
+		defaultVal,
+		generatedVal,
+		col.Virtual,
+		col.AutoIncrement,
+		col.Comment,
+		constraints...,
+	)
 }
 
 // ToDoltResultSchema returns a dolt Schema from the sql schema given, suitable for use as a result set

go/libraries/doltcore/sqle/writer/prolly_index_writer_keyless.go

@@ -32,6 +32,7 @@ type prollyKeylessWriter struct {
 	keyBld *val.TupleBuilder
 	valBld *val.TupleBuilder
 	valMap val.OrdinalMapping
+	virtualCols []int
 }
 
 var _ indexWriter = prollyKeylessWriter{}
@@ -51,6 +52,7 @@ func (k prollyKeylessWriter) ValidateKeyViolations(ctx context.Context, sqlRow s
 }
 
 func (k prollyKeylessWriter) Insert(ctx context.Context, sqlRow sql.Row) error {
+	// TODO: this thing needs a function that can convert a row to it storage equivalent as necessary
 	hashId, value, err := k.tuplesFromRow(ctx, sqlRow)
 	if err != nil {
 		return err

go/libraries/doltcore/sqle/writer/prolly_table_writer.go

@@ -380,10 +380,9 @@ func makeOrdinalMapping(from sql.Schema, to *schema.ColCollection) (m val.Ordina
 	m = make(val.OrdinalMapping, len(to.GetColumns()))
 	for i := range m {
 		name := to.GetByIndex(i).Name
-		for j, col := range from {
-			if col.Name == name {
-				m[i] = j
-			}
+		colIdx := from.IndexOfColName(name)
+		if !from[colIdx].Virtual {
+			m[i] = colIdx
 		}
 	}
 	return