From 8c3b3c4ebdd39e7d8e85bf5c1ce345db5d39b4d0 Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Tue, 4 Oct 2022 17:54:16 -0700 Subject: [PATCH 01/25] It can't be this easy can it --- .../doltcore/sqle/database_provider.go | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index c8f0de9cfc..cf3dad19c7 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -430,7 +430,7 @@ func ConfigureReplicationDatabaseHook(ctx *sql.Context, p DoltDatabaseProvider, } err = newEnv.AddRemote(r) - if err != nil { + if err != env.ErrRemoteAlreadyExists && err != nil { return err } @@ -448,7 +448,11 @@ func ConfigureReplicationDatabaseHook(ctx *sql.Context, p DoltDatabaseProvider, } // CloneDatabaseFromRemote implements DoltDatabaseProvider interface -func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, branch, remoteName, remoteUrl string, remoteParams map[string]string) error { +func (p DoltDatabaseProvider) CloneDatabaseFromRemote( + ctx *sql.Context, + dbName, branch, remoteName, remoteUrl string, + remoteParams map[string]string, +) error { p.mu.Lock() defer p.mu.Unlock() @@ -459,7 +463,7 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, return fmt.Errorf("cannot create DB, file exists at %s", dbName) } - err := p.cloneDatabaseFromRemote(ctx, dbName, remoteName, branch, remoteUrl, remoteParams) + dEnv, err := p.cloneDatabaseFromRemote(ctx, dbName, remoteName, branch, remoteUrl, remoteParams) if err != nil { // Make a best effort to clean up any artifacts on disk from a failed clone // before we return the error @@ -473,7 +477,7 @@ func (p DoltDatabaseProvider) CloneDatabaseFromRemote(ctx *sql.Context, dbName, return err } - return nil + return ConfigureReplicationDatabaseHook(ctx, p, dbName, dEnv) } // cloneDatabaseFromRemote encapsulates the inner logic for cloning a database so that if any error @@ -484,26 +488,26 @@ func (p DoltDatabaseProvider) cloneDatabaseFromRemote( ctx *sql.Context, dbName, remoteName, branch, remoteUrl string, remoteParams map[string]string, -) error { +) (*env.DoltEnv, error) { if p.remoteDialer == nil { - return fmt.Errorf("unable to clone remote database; no remote dialer configured") + return nil, fmt.Errorf("unable to clone remote database; no remote dialer configured") } // TODO: params for AWS, others that need them r := env.NewRemote(remoteName, remoteUrl, nil) srcDB, err := r.GetRemoteDB(ctx, types.Format_Default, p.remoteDialer) if err != nil { - return err + return nil, err } dEnv, err := actions.EnvForClone(ctx, srcDB.ValueReadWriter().Format(), r, dbName, p.fs, "VERSION", env.GetCurrentUserHomeDir) if err != nil { - return err + return nil, err } err = actions.CloneRemote(ctx, srcDB, remoteName, branch, dEnv) if err != nil { - return err + return nil, err } err = dEnv.RepoStateWriter().UpdateBranch(dEnv.RepoState.CWBHeadRef().GetPath(), env.BranchConfig{ @@ -514,7 +518,7 @@ func (p DoltDatabaseProvider) cloneDatabaseFromRemote( sess := dsess.DSessFromSess(ctx.Session) fkChecks, err := ctx.GetSessionVariable(ctx, "foreign_key_checks") if err != nil { - return err + return nil, err } opts := editor.Options{ @@ -525,17 +529,22 @@ func (p DoltDatabaseProvider) cloneDatabaseFromRemote( db, err := NewDatabase(ctx, dbName, dEnv.DbData(), opts) if err != nil { - return err + return nil, err } p.databases[formatDbMapKeyName(db.Name())] = db dbstate, err := GetInitialDBState(ctx, db) if err != nil { - return err + return nil, err } - return sess.AddDB(ctx, dbstate) + err = sess.AddDB(ctx, dbstate) + if err != nil { + return nil, err + } + + return dEnv, nil } // DropDatabase implements the sql.MutableDatabaseProvider interface From 8562ba5343151beb88458fe2227b8f05e6e634ba Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 12 Oct 2022 23:33:59 -0700 Subject: [PATCH 02/25] calling sql methods for types --- go/store/types/multipolygon.go | 161 +++++++++++++++++++++++++++++++ go/store/types/noms_kind.go | 3 + go/store/types/read_geometry.go | 30 ++++++ go/store/types/write_geometry.go | 4 + 4 files changed, 198 insertions(+) create mode 100644 go/store/types/multipolygon.go diff --git a/go/store/types/multipolygon.go b/go/store/types/multipolygon.go new file mode 100644 index 0000000000..b0642b9d28 --- /dev/null +++ b/go/store/types/multipolygon.go @@ -0,0 +1,161 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/dolthub/dolt/go/store/hash" +) + +// MultiPolygon is a Noms Value wrapper around a string. +type MultiPolygon struct { + SRID uint32 + Polygons []Polygon +} + +// Value interface +func (v MultiPolygon) Value(ctx context.Context) (Value, error) { + return v, nil +} + +func (v MultiPolygon) Equals(other Value) bool { + // Compare types + v2, ok := other.(MultiPolygon) + if !ok { + return false + } + // Compare SRID + if v.SRID != v2.SRID { + return false + } + // Compare lengths of polygons + if len(v.Polygons) != len(v2.Polygons) { + return false + } + // Compare each polygon + for i := 0; i < len(v.Polygons); i++ { + if !v.Polygons[i].Equals(v2.Polygons[i]) { + return false + } + } + return true +} + +func (v MultiPolygon) Less(nbf *NomsBinFormat, other LesserValuable) (bool, error) { + // Compare types + v2, ok := other.(MultiPolygon) + if !ok { + return MultiPolygonKind < other.Kind(), nil + } + // Compare SRID + if v.SRID != v2.SRID { + return v.SRID < v2.SRID, nil + } + // Get shorter length + var n int + len1 := len(v.Polygons) + len2 := len(v2.Polygons) + if len1 < len2 { + n = len1 + } else { + n = len2 + } + // Compare each polygon until there is one that is less + for i := 0; i < n; i++ { + if !v.Polygons[i].Equals(v2.Polygons[i]) { + return v.Polygons[i].Less(nbf, v2.Polygons[i]) + } + } + // Determine based off length + return len1 < len2, nil +} + +func (v MultiPolygon) Hash(nbf *NomsBinFormat) (hash.Hash, error) { + return getHash(v, nbf) +} + +func (v MultiPolygon) isPrimitive() bool { + return true +} + +func (v MultiPolygon) walkRefs(nbf *NomsBinFormat, cb RefCallback) error { + return nil +} + +func (v MultiPolygon) typeOf() (*Type, error) { + return PrimitiveTypeMap[MultiPolygonKind], nil +} + +func (v MultiPolygon) Kind() NomsKind { + return MultiPolygonKind +} + +func (v MultiPolygon) valueReadWriter() ValueReadWriter { + return nil +} + +func (v MultiPolygon) writeTo(w nomsWriter, nbf *NomsBinFormat) error { + err := MultiPolygonKind.writeTo(w, nbf) + if err != nil { + return err + } + + w.writeString(string(SerializeMultiPolygon(v))) + return nil +} + +func readMultiPolygon(nbf *NomsBinFormat, b *valueDecoder) (MultiPolygon, error) { + buf := []byte(b.ReadString()) + srid, _, geomType, err := DeserializeEWKBHeader(buf) + if err != nil { + return MultiPolygon{}, err + } + if geomType != WKBMultiPolyID { + return MultiPolygon{}, errors.New("not a multipolygon") + } + buf = buf[EWKBHeaderSize:] + return DeserializeTypesMPoly(buf, false, srid), nil +} + +func (v MultiPolygon) readFrom(nbf *NomsBinFormat, b *binaryNomsReader) (Value, error) { + buf := []byte(b.ReadString()) + srid, _, geomType, err := DeserializeEWKBHeader(buf) + if err != nil { + return MultiPolygon{}, err + } + if geomType != WKBMultiPolyID { + return MultiPolygon{}, errors.New("not a multipolygon") + } + buf = buf[EWKBHeaderSize:] + return DeserializeTypesMPoly(buf, false, srid), nil +} + +func (v MultiPolygon) skip(nbf *NomsBinFormat, b *binaryNomsReader) { + b.skipString() +} + +func (v MultiPolygon) HumanReadableString() string { + polys := make([]string, len(v.Polygons)) + for i, l := range v.Polygons { + polys[i] = l.HumanReadableString() + } + s := fmt.Sprintf("SRID: %d MULTIPOLYGON(%s)", v.SRID, strings.Join(polys, ",")) + return strconv.Quote(s) +} diff --git a/go/store/types/noms_kind.go b/go/store/types/noms_kind.go index 0e7666aa8f..577074d067 100644 --- a/go/store/types/noms_kind.go +++ b/go/store/types/noms_kind.go @@ -105,6 +105,7 @@ func init() { KindToType[SerialMessageKind] = SerialMessage{} KindToType[MultiPointKind] = MultiPoint{} KindToType[MultiLineStringKind] = MultiLineString{} + KindToType[MultiPolygonKind] = MultiPolygon{} SupportedKinds[BlobKind] = true SupportedKinds[BoolKind] = true @@ -135,6 +136,7 @@ func init() { SupportedKinds[SerialMessageKind] = true SupportedKinds[MultiPointKind] = true SupportedKinds[MultiLineStringKind] = true + SupportedKinds[MultiPolygonKind] = true if serial.MessageTypesKind != int(SerialMessageKind) { panic("internal error: serial.MessageTypesKind != SerialMessageKind") @@ -174,6 +176,7 @@ var KindToString = map[NomsKind]string{ SerialMessageKind: "SerialMessage", MultiPointKind: "MultiPoint", MultiLineStringKind: "MultiLineString", + MultiPolygonKind: "MultiPolygon", } // String returns the name of the kind. diff --git a/go/store/types/read_geometry.go b/go/store/types/read_geometry.go index 32598da5cf..169eb20099 100644 --- a/go/store/types/read_geometry.go +++ b/go/store/types/read_geometry.go @@ -67,6 +67,14 @@ func ConvertTypesMultiLineStringToSQLMultiLineString(l MultiLineString) sql.Mult return sql.MultiLineString{SRID: l.SRID, Lines: lines} } +func ConvertTypesMultiPolygonToSQLMultiPolygon(p MultiPolygon) sql.MultiPolygon { + polys := make([]sql.Polygon, len(p.Polygons)) + for i, p := range p.Polygons { + polys[i] = ConvertTypesPolygonToSQLPolygon(p) + } + return sql.MultiPolygon{SRID: p.SRID, Polygons: polys} +} + func ConvertSQLGeometryToTypesGeometry(p interface{}) Value { switch inner := p.(type) { case sql.Point: @@ -79,6 +87,8 @@ func ConvertSQLGeometryToTypesGeometry(p interface{}) Value { return ConvertSQLMultiPointToTypesMultiPoint(inner) case sql.MultiLineString: return ConvertSQLMultiLineStringToTypesMultiLineString(inner) + case sql.MultiPolygon: + return ConvertSQLMultiPolygonToTypesMultiPolygon(inner) default: panic("used an invalid type sql.Geometry.Inner") } @@ -120,6 +130,14 @@ func ConvertSQLMultiLineStringToTypesMultiLineString(p sql.MultiLineString) Mult return MultiLineString{SRID: p.SRID, Lines: lines} } +func ConvertSQLMultiPolygonToTypesMultiPolygon(p sql.MultiPolygon) MultiPolygon { + polys := make([]Polygon, len(p.Polygons)) + for i, p := range p.Polygons { + polys[i] = ConvertSQLPolygonToTypesPolygon(p) + } + return MultiPolygon{SRID: p.SRID, Polygons: polys} +} + // TODO: all methods here just defer to their SQL equivalents, and assume we always receive good data func DeserializeEWKBHeader(buf []byte) (uint32, bool, uint32, error) { @@ -170,6 +188,14 @@ func DeserializeMLine(buf []byte, isBig bool, srid uint32) sql.MultiLineString { return p } +func DeserializeMPoly(buf []byte, isBig bool, srid uint32) sql.MultiPolygon { + p, err := sql.DeserializeMPoly(buf, isBig, srid) + if err != nil { + panic(err) + } + return p +} + // TODO: noms needs results to be in types func DeserializeTypesPoint(buf []byte, isBig bool, srid uint32) Point { @@ -191,3 +217,7 @@ func DeserializeTypesMPoint(buf []byte, isBig bool, srid uint32) MultiPoint { func DeserializeTypesMLine(buf []byte, isBig bool, srid uint32) MultiLineString { return ConvertSQLMultiLineStringToTypesMultiLineString(DeserializeMLine(buf, isBig, srid)) } + +func DeserializeTypesMPoly(buf []byte, isBig bool, srid uint32) MultiPolygon { + return ConvertSQLMultiPolygonToTypesMultiPolygon(DeserializeMPoly(buf, isBig, srid)) +} diff --git a/go/store/types/write_geometry.go b/go/store/types/write_geometry.go index c5abce5dc3..1f83a09f17 100644 --- a/go/store/types/write_geometry.go +++ b/go/store/types/write_geometry.go @@ -69,3 +69,7 @@ func SerializeMultiPoint(p MultiPoint) []byte { func SerializeMultiLineString(p MultiLineString) []byte { return ConvertTypesMultiLineStringToSQLMultiLineString(p).Serialize() } + +func SerializeMultiPolygon(p MultiPolygon) []byte { + return ConvertTypesMultiPolygonToSQLMultiPolygon(p).Serialize() +} From 9e3848d6f1850a0ebbf5f1e93b6902c827dd6064 Mon Sep 17 00:00:00 2001 From: James Cor Date: Wed, 12 Oct 2022 23:49:18 -0700 Subject: [PATCH 03/25] migrate --- go/libraries/doltcore/migrate/tuples.go | 4 ++++ go/libraries/doltcore/migrate/validation.go | 1 + 2 files changed, 5 insertions(+) diff --git a/go/libraries/doltcore/migrate/tuples.go b/go/libraries/doltcore/migrate/tuples.go index ff6f036e6a..ab921161e2 100644 --- a/go/libraries/doltcore/migrate/tuples.go +++ b/go/libraries/doltcore/migrate/tuples.go @@ -284,6 +284,10 @@ func translateGeometryField(value types.Value, idx int, b *val.TupleBuilder) { l := types.ConvertTypesMultiLineStringToSQLMultiLineString(value.(types.MultiLineString)) b.PutGeometry(idx, l.Serialize()) + case types.MultiPolygonKind: + p := types.ConvertTypesMultiPolygonToSQLMultiPolygon(value.(types.MultiPolygon)) + b.PutGeometry(idx, p.Serialize()) + default: panic(fmt.Sprintf("unexpected NomsKind for geometry (%d)", nk)) } diff --git a/go/libraries/doltcore/migrate/validation.go b/go/libraries/doltcore/migrate/validation.go index d82c460452..407c004c59 100644 --- a/go/libraries/doltcore/migrate/validation.go +++ b/go/libraries/doltcore/migrate/validation.go @@ -240,6 +240,7 @@ func nomsKindsFromQueryTypes(qt query.Type) []types.NomsKind { types.PolygonKind, types.MultiPointKind, types.MultiLineStringKind, + types.MultiPolygonKind, } case query.Type_JSON: From cf48fb9bf111f27b0ccb8cb446752dd2a808cf7e Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 13 Oct 2022 00:19:41 -0700 Subject: [PATCH 04/25] progress --- .../doltcore/schema/typeinfo/multipolygon.go | 229 ++++++++++++++++++ .../doltcore/schema/typeinfo/typeinfo.go | 6 + .../doltcore/sqle/index/prolly_fields.go | 2 + go/store/types/value_decoder.go | 31 ++- 4 files changed, 267 insertions(+), 1 deletion(-) create mode 100644 go/libraries/doltcore/schema/typeinfo/multipolygon.go diff --git a/go/libraries/doltcore/schema/typeinfo/multipolygon.go b/go/libraries/doltcore/schema/typeinfo/multipolygon.go new file mode 100644 index 0000000000..5f3535bbc0 --- /dev/null +++ b/go/libraries/doltcore/schema/typeinfo/multipolygon.go @@ -0,0 +1,229 @@ +// Copyright 2020 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package typeinfo + +import ( + "context" + "fmt" + "strconv" + + "github.com/dolthub/go-mysql-server/sql" + + "github.com/dolthub/dolt/go/store/types" +) + +// This is a dolt implementation of the MySQL type Point, thus most of the functionality +// within is directly reliant on the go-mysql-server implementation. +type multipolygonType struct { + sqlMultiPolygonType sql.MultiPolygonType +} + +var _ TypeInfo = (*multipolygonType)(nil) + +var MultiPolygonType = &multipolygonType{sql.MultiPolygonType{}} + +// ConvertNomsValueToValue implements TypeInfo interface. +func (ti *multipolygonType) ConvertNomsValueToValue(v types.Value) (interface{}, error) { + // Check for null + if _, ok := v.(types.Null); ok || v == nil { + return nil, nil + } + // Expect a types.MultiPolygon, return a sql.MultiPolygon + if val, ok := v.(types.MultiPolygon); ok { + return types.ConvertTypesMultiPolygonToSQLMultiPolygon(val), nil + } + + return nil, fmt.Errorf(`"%v" cannot convert NomsKind "%v" to a value`, ti.String(), v.Kind()) +} + +// ReadFrom reads a go value from a noms types.CodecReader directly +func (ti *multipolygonType) ReadFrom(nbf *types.NomsBinFormat, reader types.CodecReader) (interface{}, error) { + k := reader.ReadKind() + switch k { + case types.MultiPolygonKind: + p, err := reader.ReadMultiPolygon() + if err != nil { + return nil, err + } + return ti.ConvertNomsValueToValue(p) + case types.NullKind: + return nil, nil + } + + return nil, fmt.Errorf(`"%v" cannot convert NomsKind "%v" to a value`, ti.String(), k) +} + +// ConvertValueToNomsValue implements TypeInfo interface. +func (ti *multipolygonType) ConvertValueToNomsValue(ctx context.Context, vrw types.ValueReadWriter, v interface{}) (types.Value, error) { + // Check for null + if v == nil { + return types.NullValue, nil + } + + // Convert to sql.MultiPolygon + mpoly, err := ti.sqlMultiPolygonType.Convert(v) + if err != nil { + return nil, err + } + + return types.ConvertSQLMultiPolygonToTypesMultiPolygon(mpoly.(sql.MultiPolygon)), nil +} + +// Equals implements TypeInfo interface. +func (ti *multipolygonType) Equals(other TypeInfo) bool { + if other == nil { + return false + } + if o, ok := other.(*multipolygonType); ok { + // if either ti or other has defined SRID, then check SRID value; otherwise, + return (!ti.sqlMultiPolygonType.DefinedSRID && !o.sqlMultiPolygonType.DefinedSRID) || ti.sqlMultiPolygonType.SRID == o.sqlMultiPolygonType.SRID + } + return false +} + +// FormatValue implements TypeInfo interface. +func (ti *multipolygonType) FormatValue(v types.Value) (*string, error) { + if val, ok := v.(types.MultiPolygon); ok { + resStr := string(types.SerializeMultiPolygon(val)) + return &resStr, nil + } + if _, ok := v.(types.Null); ok || v == nil { + return nil, nil + } + + return nil, fmt.Errorf(`"%v" has unexpectedly encountered a value of type "%T" from embedded type`, ti.String(), v.Kind()) +} + +// GetTypeIdentifier implements TypeInfo interface. +func (ti *multipolygonType) GetTypeIdentifier() Identifier { + return MultiPolygonTypeIdentifier +} + +// GetTypeParams implements TypeInfo interface. +func (ti *multipolygonType) GetTypeParams() map[string]string { + return map[string]string{"SRID": strconv.FormatUint(uint64(ti.sqlMultiPolygonType.SRID), 10), + "DefinedSRID": strconv.FormatBool(ti.sqlMultiPolygonType.DefinedSRID)} +} + +// IsValid implements TypeInfo interface. +func (ti *multipolygonType) IsValid(v types.Value) bool { + if _, ok := v.(types.MultiPolygon); ok { + return true + } + if _, ok := v.(types.Null); ok || v == nil { + return true + } + return false +} + +// NomsKind implements TypeInfo interface. +func (ti *multipolygonType) NomsKind() types.NomsKind { + return types.MultiPolygonKind +} + +// Promote implements TypeInfo interface. +func (ti *multipolygonType) Promote() TypeInfo { + return &multipolygonType{ti.sqlMultiPolygonType.Promote().(sql.MultiPolygonType)} +} + +// String implements TypeInfo interface. +func (ti *multipolygonType) String() string { + return "MultiPolygon" +} + +// ToSqlType implements TypeInfo interface. +func (ti *multipolygonType) ToSqlType() sql.Type { + return ti.sqlMultiPolygonType +} + +// multipolygonTypeConverter is an internal function for GetTypeConverter that handles the specific type as the source TypeInfo. +func multipolygonTypeConverter(ctx context.Context, src *multipolygonType, destTi TypeInfo) (tc TypeConverter, needsConversion bool, err error) { + switch dest := destTi.(type) { + case *bitType: + return func(ctx context.Context, vrw types.ValueReadWriter, v types.Value) (types.Value, error) { + return types.Uint(0), nil + }, true, nil + case *blobStringType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *boolType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *datetimeType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *decimalType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *enumType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *floatType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *geometryType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *inlineBlobType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *intType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *jsonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *linestringType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multilinestringType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipointType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *pointType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *polygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *setType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *timeType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *uintType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *uuidType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *varBinaryType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *varStringType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *yearType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + default: + return nil, false, UnhandledTypeConversion.New(src.String(), destTi.String()) + } +} + +func CreateMultiPolygonTypeFromParams(params map[string]string) (TypeInfo, error) { + var ( + err error + sridVal uint64 + def bool + ) + if s, ok := params["SRID"]; ok { + sridVal, err = strconv.ParseUint(s, 10, 32) + if err != nil { + return nil, err + } + } + if d, ok := params["DefinedSRID"]; ok { + def, err = strconv.ParseBool(d) + if err != nil { + return nil, err + } + } + + return &multipolygonType{sqlMultiPolygonType: sql.MultiPolygonType{SRID: uint32(sridVal), DefinedSRID: def}}, nil +} diff --git a/go/libraries/doltcore/schema/typeinfo/typeinfo.go b/go/libraries/doltcore/schema/typeinfo/typeinfo.go index be878126f3..595d7647d8 100644 --- a/go/libraries/doltcore/schema/typeinfo/typeinfo.go +++ b/go/libraries/doltcore/schema/typeinfo/typeinfo.go @@ -53,6 +53,7 @@ const ( PolygonTypeIdentifier Identifier = "polygon" MultiPointTypeIdentifier Identifier = "multipoint" MultiLineStringTypeIdentifier Identifier = "multilinestring" + MultiPolygonTypeIdentifier Identifier = "multipolygon" ) var Identifiers = map[Identifier]struct{}{ @@ -81,6 +82,7 @@ var Identifiers = map[Identifier]struct{}{ PolygonTypeIdentifier: {}, MultiPointTypeIdentifier: {}, MultiLineStringTypeIdentifier: {}, + MultiPolygonTypeIdentifier: {}, } // TypeInfo is an interface used for encoding type information. @@ -179,6 +181,8 @@ func FromSqlType(sqlType sql.Type) (TypeInfo, error) { return &multipointType{}, nil case sql.MultiLineStringType{}.String(): return &multilinestringType{}, nil + case sql.MultiPolygonType{}.String(): + return &multipolygonType{}, nil case sql.GeometryType{}.String(): return &geometryType{sqlGeometryType: sqlType.(sql.GeometryType)}, nil default: @@ -290,6 +294,8 @@ func FromTypeParams(id Identifier, params map[string]string) (TypeInfo, error) { return CreateMultiPointTypeFromParams(params) case MultiLineStringTypeIdentifier: return CreateMultiLineStringTypeFromParams(params) + case MultiPolygonTypeIdentifier: + return CreateMultiPolygonTypeFromParams(params) case SetTypeIdentifier: return CreateSetTypeFromParams(params) case TimeTypeIdentifier: diff --git a/go/libraries/doltcore/sqle/index/prolly_fields.go b/go/libraries/doltcore/sqle/index/prolly_fields.go index b1f95afa9a..55b9af0d14 100644 --- a/go/libraries/doltcore/sqle/index/prolly_fields.go +++ b/go/libraries/doltcore/sqle/index/prolly_fields.go @@ -286,6 +286,8 @@ func deserializeGeometry(buf []byte) (v interface{}) { v, _ = sql.DeserializeMPoint(buf, false, srid) case sql.WKBMultiLineID: v, _ = sql.DeserializeMLine(buf, false, srid) + case sql.WKBMultiPolyID: + v, _ = sql.DeserializeMPoly(buf, false, srid) default: panic(fmt.Sprintf("unknown geometry type %d", typ)) } diff --git a/go/store/types/value_decoder.go b/go/store/types/value_decoder.go index a1b870448c..db1a27865f 100644 --- a/go/store/types/value_decoder.go +++ b/go/store/types/value_decoder.go @@ -52,6 +52,7 @@ type CodecReader interface { ReadPolygon() (Polygon, error) ReadMultiPoint() (MultiPoint, error) ReadMultiLineString() (MultiLineString, error) + ReadMultiPolygon() (MultiPolygon, error) ReadBlob() (Blob, error) ReadJSON() (JSON, error) } @@ -110,6 +111,10 @@ func (r *valueDecoder) ReadMultiLineString() (MultiLineString, error) { return readMultiLineString(nil, r) } +func (r *valueDecoder) ReadMultiPolygon() (MultiPolygon, error) { + return readMultiPolygon(nil, r) +} + func (r *valueDecoder) ReadJSON() (JSON, error) { return readJSON(r.vrw.Format(), r) } @@ -449,11 +454,35 @@ func (r *valueDecoder) readValue(nbf *NomsBinFormat) (Value, error) { if err != nil { return nil, err } - if geomType != WKBPolyID { + if geomType != WKBMultiPointID { return nil, ErrUnknownType } buf = buf[EWKBHeaderSize:] return DeserializeTypesMPoint(buf, false, srid), nil + case MultiLineStringKind: + r.skipKind() + buf := []byte(r.ReadString()) + srid, _, geomType, err := DeserializeEWKBHeader(buf) + if err != nil { + return nil, err + } + if geomType != WKBMultiLineID { + return nil, ErrUnknownType + } + buf = buf[EWKBHeaderSize:] + return DeserializeTypesMLine(buf, false, srid), nil + case MultiPolygonKind: + r.skipKind() + buf := []byte(r.ReadString()) + srid, _, geomType, err := DeserializeEWKBHeader(buf) + if err != nil { + return nil, err + } + if geomType != WKBMultiPolyID { + return nil, ErrUnknownType + } + buf = buf[EWKBHeaderSize:] + return DeserializeTypesMPoly(buf, false, srid), nil case TypeKind: r.skipKind() return r.readType() From e03049cfbca44a8de465fb655b8083467b00a5f5 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 13 Oct 2022 00:20:28 -0700 Subject: [PATCH 05/25] asd --- go/go.mod | 2 ++ go/store/types/value_decoder.go | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/go/go.mod b/go/go.mod index a2c4141dea..bebdf0ecbb 100644 --- a/go/go.mod +++ b/go/go.mod @@ -140,6 +140,8 @@ replace ( github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi => ./gen/proto/dolt/services/eventsapi github.com/google/flatbuffers => github.com/dolthub/flatbuffers v1.13.0-dh.1 github.com/oliveagle/jsonpath => github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 + github.com/dolthub/go-mysql-server => ../../go-mysql-server + github.com/dolthub/vitess => ../../vitess ) go 1.19 diff --git a/go/store/types/value_decoder.go b/go/store/types/value_decoder.go index db1a27865f..1e990b7af7 100644 --- a/go/store/types/value_decoder.go +++ b/go/store/types/value_decoder.go @@ -544,9 +544,15 @@ func (r *valueDecoder) SkipValue(nbf *NomsBinFormat) error { case PolygonKind: r.skipKind() r.skipString() + case MultiLineStringKind: + r.skipKind() + r.skipString() case MultiPointKind: r.skipKind() r.skipString() + case MultiPolygonKind: + r.skipKind() + r.skipString() case ListKind: err := r.skipList(nbf) if err != nil { From 4f2515785ba23c97f380f1c6e0816cb27ec0b985 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 13 Oct 2022 00:28:04 -0700 Subject: [PATCH 06/25] type convert functions --- go/libraries/doltcore/schema/typeinfo/bit.go | 2 ++ go/libraries/doltcore/schema/typeinfo/blobstring.go | 2 ++ go/libraries/doltcore/schema/typeinfo/bool.go | 2 ++ go/libraries/doltcore/schema/typeinfo/datetime.go | 2 ++ go/libraries/doltcore/schema/typeinfo/decimal.go | 2 ++ go/libraries/doltcore/schema/typeinfo/enum.go | 2 ++ go/libraries/doltcore/schema/typeinfo/float.go | 2 ++ go/libraries/doltcore/schema/typeinfo/geometry.go | 2 ++ go/libraries/doltcore/schema/typeinfo/inlineblob.go | 2 ++ go/libraries/doltcore/schema/typeinfo/int.go | 2 ++ go/libraries/doltcore/schema/typeinfo/json.go | 2 ++ go/libraries/doltcore/schema/typeinfo/linestring.go | 2 ++ go/libraries/doltcore/schema/typeinfo/multilinestring.go | 2 ++ go/libraries/doltcore/schema/typeinfo/multipoint.go | 2 ++ go/libraries/doltcore/schema/typeinfo/point.go | 2 ++ go/libraries/doltcore/schema/typeinfo/polygon.go | 2 ++ go/libraries/doltcore/schema/typeinfo/set.go | 2 ++ go/libraries/doltcore/schema/typeinfo/time.go | 2 ++ go/libraries/doltcore/schema/typeinfo/typeconverter.go | 2 ++ go/libraries/doltcore/schema/typeinfo/uint.go | 2 ++ go/libraries/doltcore/schema/typeinfo/uuid.go | 2 ++ go/libraries/doltcore/schema/typeinfo/varbinary.go | 2 ++ go/libraries/doltcore/schema/typeinfo/varstring.go | 2 ++ go/libraries/doltcore/schema/typeinfo/year.go | 2 ++ 24 files changed, 48 insertions(+) diff --git a/go/libraries/doltcore/schema/typeinfo/bit.go b/go/libraries/doltcore/schema/typeinfo/bit.go index 5849767926..3a02a82f45 100644 --- a/go/libraries/doltcore/schema/typeinfo/bit.go +++ b/go/libraries/doltcore/schema/typeinfo/bit.go @@ -222,6 +222,8 @@ func bitTypeConverter(ctx context.Context, src *bitType, destTi TypeInfo) (tc Ty return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/blobstring.go b/go/libraries/doltcore/schema/typeinfo/blobstring.go index 443e196b81..15ad5aa097 100644 --- a/go/libraries/doltcore/schema/typeinfo/blobstring.go +++ b/go/libraries/doltcore/schema/typeinfo/blobstring.go @@ -252,6 +252,8 @@ func blobStringTypeConverter(ctx context.Context, src *blobStringType, destTi Ty return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/bool.go b/go/libraries/doltcore/schema/typeinfo/bool.go index 33b1b97fdb..9c16851bc9 100644 --- a/go/libraries/doltcore/schema/typeinfo/bool.go +++ b/go/libraries/doltcore/schema/typeinfo/bool.go @@ -239,6 +239,8 @@ func boolTypeConverter(ctx context.Context, src *boolType, destTi TypeInfo) (tc return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/datetime.go b/go/libraries/doltcore/schema/typeinfo/datetime.go index 0b78e5e644..76525989cf 100644 --- a/go/libraries/doltcore/schema/typeinfo/datetime.go +++ b/go/libraries/doltcore/schema/typeinfo/datetime.go @@ -233,6 +233,8 @@ func datetimeTypeConverter(ctx context.Context, src *datetimeType, destTi TypeIn return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/decimal.go b/go/libraries/doltcore/schema/typeinfo/decimal.go index 63a26894b0..aa82501731 100644 --- a/go/libraries/doltcore/schema/typeinfo/decimal.go +++ b/go/libraries/doltcore/schema/typeinfo/decimal.go @@ -272,6 +272,8 @@ func decimalTypeConverter(ctx context.Context, src *decimalType, destTi TypeInfo return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/enum.go b/go/libraries/doltcore/schema/typeinfo/enum.go index e0a2124c17..aebb75ae41 100644 --- a/go/libraries/doltcore/schema/typeinfo/enum.go +++ b/go/libraries/doltcore/schema/typeinfo/enum.go @@ -239,6 +239,8 @@ func enumTypeConverter(ctx context.Context, src *enumType, destTi TypeInfo) (tc return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/float.go b/go/libraries/doltcore/schema/typeinfo/float.go index 0cf93d7ff1..390fdf149c 100644 --- a/go/libraries/doltcore/schema/typeinfo/float.go +++ b/go/libraries/doltcore/schema/typeinfo/float.go @@ -263,6 +263,8 @@ func floatTypeConverter(ctx context.Context, src *floatType, destTi TypeInfo) (t return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/geometry.go b/go/libraries/doltcore/schema/typeinfo/geometry.go index ba8da784db..5d488a5a19 100644 --- a/go/libraries/doltcore/schema/typeinfo/geometry.go +++ b/go/libraries/doltcore/schema/typeinfo/geometry.go @@ -249,6 +249,8 @@ func geometryTypeConverter(ctx context.Context, src *geometryType, destTi TypeIn return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/inlineblob.go b/go/libraries/doltcore/schema/typeinfo/inlineblob.go index 9b516d61c0..5838da0c22 100644 --- a/go/libraries/doltcore/schema/typeinfo/inlineblob.go +++ b/go/libraries/doltcore/schema/typeinfo/inlineblob.go @@ -253,6 +253,8 @@ func inlineBlobTypeConverter(ctx context.Context, src *inlineBlobType, destTi Ty return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/int.go b/go/libraries/doltcore/schema/typeinfo/int.go index f956a7c666..3f44dc225c 100644 --- a/go/libraries/doltcore/schema/typeinfo/int.go +++ b/go/libraries/doltcore/schema/typeinfo/int.go @@ -291,6 +291,8 @@ func intTypeConverter(ctx context.Context, src *intType, destTi TypeInfo) (tc Ty return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/json.go b/go/libraries/doltcore/schema/typeinfo/json.go index b9493d7a0b..56fa35c05f 100644 --- a/go/libraries/doltcore/schema/typeinfo/json.go +++ b/go/libraries/doltcore/schema/typeinfo/json.go @@ -181,6 +181,8 @@ func jsonTypeConverter(ctx context.Context, src *jsonType, destTi TypeInfo) (tc return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/linestring.go b/go/libraries/doltcore/schema/typeinfo/linestring.go index ad4d253936..63ea809417 100644 --- a/go/libraries/doltcore/schema/typeinfo/linestring.go +++ b/go/libraries/doltcore/schema/typeinfo/linestring.go @@ -181,6 +181,8 @@ func linestringTypeConverter(ctx context.Context, src *linestringType, destTi Ty return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/multilinestring.go b/go/libraries/doltcore/schema/typeinfo/multilinestring.go index b153274922..816e7e76ea 100644 --- a/go/libraries/doltcore/schema/typeinfo/multilinestring.go +++ b/go/libraries/doltcore/schema/typeinfo/multilinestring.go @@ -181,6 +181,8 @@ func multilinestringTypeConverter(ctx context.Context, src *multilinestringType, return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/multipoint.go b/go/libraries/doltcore/schema/typeinfo/multipoint.go index e9860ea0b8..b74c5db19c 100644 --- a/go/libraries/doltcore/schema/typeinfo/multipoint.go +++ b/go/libraries/doltcore/schema/typeinfo/multipoint.go @@ -181,6 +181,8 @@ func multipointTypeConverter(ctx context.Context, src *multipointType, destTi Ty return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/point.go b/go/libraries/doltcore/schema/typeinfo/point.go index f8faf9da29..f3eb7cac67 100644 --- a/go/libraries/doltcore/schema/typeinfo/point.go +++ b/go/libraries/doltcore/schema/typeinfo/point.go @@ -181,6 +181,8 @@ func pointTypeConverter(ctx context.Context, src *pointType, destTi TypeInfo) (t return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/polygon.go b/go/libraries/doltcore/schema/typeinfo/polygon.go index 88c8d341dd..62e6a3175b 100644 --- a/go/libraries/doltcore/schema/typeinfo/polygon.go +++ b/go/libraries/doltcore/schema/typeinfo/polygon.go @@ -181,6 +181,8 @@ func polygonTypeConverter(ctx context.Context, src *polygonType, destTi TypeInfo return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/set.go b/go/libraries/doltcore/schema/typeinfo/set.go index 826b05541a..b5dd28c854 100644 --- a/go/libraries/doltcore/schema/typeinfo/set.go +++ b/go/libraries/doltcore/schema/typeinfo/set.go @@ -222,6 +222,8 @@ func setTypeConverter(ctx context.Context, src *setType, destTi TypeInfo) (tc Ty return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/time.go b/go/libraries/doltcore/schema/typeinfo/time.go index 2fa8f75358..270a280940 100644 --- a/go/libraries/doltcore/schema/typeinfo/time.go +++ b/go/libraries/doltcore/schema/typeinfo/time.go @@ -164,6 +164,8 @@ func timeTypeConverter(ctx context.Context, src *timeType, destTi TypeInfo) (tc return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/typeconverter.go b/go/libraries/doltcore/schema/typeinfo/typeconverter.go index 276cea2632..3a14c6ea0b 100644 --- a/go/libraries/doltcore/schema/typeinfo/typeconverter.go +++ b/go/libraries/doltcore/schema/typeinfo/typeconverter.go @@ -79,6 +79,8 @@ func GetTypeConverter(ctx context.Context, srcTi TypeInfo, destTi TypeInfo) (tc return multilinestringTypeConverter(ctx, src, destTi) case *multipointType: return multipointTypeConverter(ctx, src, destTi) + case *multipolygonType: + return multipolygonTypeConverter(ctx, src, destTi) case *pointType: return pointTypeConverter(ctx, src, destTi) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/uint.go b/go/libraries/doltcore/schema/typeinfo/uint.go index 3049553ce6..af8254170d 100644 --- a/go/libraries/doltcore/schema/typeinfo/uint.go +++ b/go/libraries/doltcore/schema/typeinfo/uint.go @@ -291,6 +291,8 @@ func uintTypeConverter(ctx context.Context, src *uintType, destTi TypeInfo) (tc return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/uuid.go b/go/libraries/doltcore/schema/typeinfo/uuid.go index e0e9ae56d3..eea0c8bf5a 100644 --- a/go/libraries/doltcore/schema/typeinfo/uuid.go +++ b/go/libraries/doltcore/schema/typeinfo/uuid.go @@ -169,6 +169,8 @@ func uuidTypeConverter(ctx context.Context, src *uuidType, destTi TypeInfo) (tc return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/varbinary.go b/go/libraries/doltcore/schema/typeinfo/varbinary.go index 56b394284c..9f92a9485a 100644 --- a/go/libraries/doltcore/schema/typeinfo/varbinary.go +++ b/go/libraries/doltcore/schema/typeinfo/varbinary.go @@ -281,6 +281,8 @@ func varBinaryTypeConverter(ctx context.Context, src *varBinaryType, destTi Type return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/varstring.go b/go/libraries/doltcore/schema/typeinfo/varstring.go index c3ddc2d2fb..763885e4f4 100644 --- a/go/libraries/doltcore/schema/typeinfo/varstring.go +++ b/go/libraries/doltcore/schema/typeinfo/varstring.go @@ -304,6 +304,8 @@ func varStringTypeConverter(ctx context.Context, src *varStringType, destTi Type return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: diff --git a/go/libraries/doltcore/schema/typeinfo/year.go b/go/libraries/doltcore/schema/typeinfo/year.go index 565a0ddb42..1f43510ce2 100644 --- a/go/libraries/doltcore/schema/typeinfo/year.go +++ b/go/libraries/doltcore/schema/typeinfo/year.go @@ -180,6 +180,8 @@ func yearTypeConverter(ctx context.Context, src *yearType, destTi TypeInfo) (tc return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *multipointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) + case *multipolygonType: + return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *pointType: return wrapConvertValueToNomsValue(dest.ConvertValueToNomsValue) case *polygonType: From 1ef70fe72d27abdd58607d2a975e8a855a5ea9ee Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 13 Oct 2022 00:37:43 -0700 Subject: [PATCH 07/25] type info tests --- go/libraries/doltcore/schema/typeinfo/geometry.go | 15 ++++++++++++++- .../doltcore/schema/typeinfo/typeinfo_test.go | 8 +++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/go/libraries/doltcore/schema/typeinfo/geometry.go b/go/libraries/doltcore/schema/typeinfo/geometry.go index 5d488a5a19..c6a9decd96 100644 --- a/go/libraries/doltcore/schema/typeinfo/geometry.go +++ b/go/libraries/doltcore/schema/typeinfo/geometry.go @@ -55,6 +55,8 @@ func (ti *geometryType) ConvertNomsValueToValue(v types.Value) (interface{}, err return types.ConvertTypesMultiPointToSQLMultiPoint(val), nil case types.MultiLineString: return types.ConvertTypesMultiLineStringToSQLMultiLineString(val), nil + case types.MultiPolygon: + return types.ConvertTypesMultiPolygonToSQLMultiPolygon(val), nil default: return nil, fmt.Errorf(`"%v" cannot convert NomsKind "%v" to a value`, ti.String(), v.Kind()) } @@ -87,6 +89,10 @@ func (ti *geometryType) ReadFrom(nbf *types.NomsBinFormat, reader types.CodecRea if val, err = reader.ReadMultiLineString(); err != nil { return nil, err } + case types.MultiPolygonKind: + if val, err = reader.ReadMultiPolygon(); err != nil { + return nil, err + } case types.GeometryKind: // Note: GeometryKind is no longer written // included here for backward compatibility @@ -148,6 +154,8 @@ func (ti *geometryType) FormatValue(v types.Value) (*string, error) { return MultiPointType.FormatValue(val) case types.MultiLineString: return MultiLineStringType.FormatValue(val) + case types.MultiPolygon: + return MultiPolygonType.FormatValue(val) case types.Geometry: switch inner := val.Inner.(type) { case types.Point: @@ -160,6 +168,8 @@ func (ti *geometryType) FormatValue(v types.Value) (*string, error) { return MultiPointType.FormatValue(inner) case types.MultiLineString: return MultiLineStringType.FormatValue(inner) + case types.MultiPolygon: + return MultiPolygonType.FormatValue(val) default: return nil, fmt.Errorf(`"%v" has unexpectedly encountered a value of type "%T" from embedded type`, ti.String(), v.Kind()) } @@ -189,7 +199,10 @@ func (ti *geometryType) IsValid(v types.Value) bool { case types.Geometry, types.Point, types.LineString, - types.Polygon: + types.Polygon, + types.MultiPoint, + types.MultiLineString, + types.MultiPolygon: return true default: return false diff --git a/go/libraries/doltcore/schema/typeinfo/typeinfo_test.go b/go/libraries/doltcore/schema/typeinfo/typeinfo_test.go index 0091f0438f..c777a82746 100644 --- a/go/libraries/doltcore/schema/typeinfo/typeinfo_test.go +++ b/go/libraries/doltcore/schema/typeinfo/typeinfo_test.go @@ -359,6 +359,7 @@ func generateTypeInfoArrays(t *testing.T) ([][]TypeInfo, [][]types.Value) { {PolygonType}, {MultiPointType}, {MultiLineStringType}, + {MultiPolygonType}, {GeometryType}, generateSetTypes(t, 16), {TimeType}, @@ -394,9 +395,10 @@ func generateTypeInfoArrays(t *testing.T) ([][]TypeInfo, [][]types.Value) { json.MustTypesJSON(`false`), json.MustTypesJSON(`{"a": 1, "b": []}`)}, //JSON {types.LineString{SRID: 0, Points: []types.Point{{SRID: 0, X: 1, Y: 2}, {SRID: 0, X: 3, Y: 4}}}}, // LineString {types.Point{SRID: 0, X: 1, Y: 2}}, // Point - {types.Polygon{SRID: 0, Lines: []types.LineString{{SRID: 0, Points: []types.Point{{SRID: 0, X: 0, Y: 0}, {SRID: 0, X: 0, Y: 1}, {SRID: 0, X: 1, Y: 1}, {SRID: 0, X: 0, Y: 0}}}}}}, // Polygon - {types.MultiPoint{SRID: 0, Points: []types.Point{{SRID: 0, X: 1, Y: 2}, {SRID: 0, X: 3, Y: 4}}}}, // MultiPoint - {types.MultiLineString{SRID: 0, Lines: []types.LineString{{SRID: 0, Points: []types.Point{{SRID: 0, X: 0, Y: 0}, {SRID: 0, X: 0, Y: 1}, {SRID: 0, X: 1, Y: 1}, {SRID: 0, X: 0, Y: 0}}}}}}, // Polygon + {types.Polygon{SRID: 0, Lines: []types.LineString{{SRID: 0, Points: []types.Point{{SRID: 0, X: 0, Y: 0}, {SRID: 0, X: 0, Y: 1}, {SRID: 0, X: 1, Y: 1}, {SRID: 0, X: 0, Y: 0}}}}}}, // Polygon + {types.MultiPoint{SRID: 0, Points: []types.Point{{SRID: 0, X: 1, Y: 2}, {SRID: 0, X: 3, Y: 4}}}}, // MultiPoint + {types.MultiLineString{SRID: 0, Lines: []types.LineString{{SRID: 0, Points: []types.Point{{SRID: 0, X: 0, Y: 0}, {SRID: 0, X: 0, Y: 1}, {SRID: 0, X: 1, Y: 1}, {SRID: 0, X: 0, Y: 0}}}}}}, // MultiLineString + {types.MultiPolygon{SRID: 0, Polygons: []types.Polygon{{SRID: 0, Lines: []types.LineString{{SRID: 0, Points: []types.Point{{SRID: 0, X: 0, Y: 0}, {SRID: 0, X: 0, Y: 1}, {SRID: 0, X: 1, Y: 1}, {SRID: 0, X: 0, Y: 0}}}}}}}}, // MultiPolygon {types.Geometry{Inner: types.Point{SRID: 0, X: 1, Y: 2}}}, // Geometry holding a Point {types.Uint(1), types.Uint(5), types.Uint(64), types.Uint(42), types.Uint(192)}, //Set {types.Int(0), types.Int(1000000 /*"00:00:01"*/), types.Int(113000000 /*"00:01:53"*/), types.Int(247019000000 /*"68:36:59"*/), types.Int(458830485214 /*"127:27:10.485214"*/)}, //Time From b45b29cf02f0f0e94c522a98c94c29793e903079 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 13 Oct 2022 00:42:41 -0700 Subject: [PATCH 08/25] aaaaaaaaaaa --- go/go.mod | 4 +--- go/go.sum | 4 ++-- go/libraries/doltcore/sqle/sqlfmt/row_fmt.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go/go.mod b/go/go.mod index bebdf0ecbb..007068d7ec 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/dolthub/go-mysql-server v0.12.1-0.20221012045247-2eb37b2d80e0 + github.com/dolthub/go-mysql-server v0.12.1-0.20221012234900-959225d12a05 github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 @@ -140,8 +140,6 @@ replace ( github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi => ./gen/proto/dolt/services/eventsapi github.com/google/flatbuffers => github.com/dolthub/flatbuffers v1.13.0-dh.1 github.com/oliveagle/jsonpath => github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 - github.com/dolthub/go-mysql-server => ../../go-mysql-server - github.com/dolthub/vitess => ../../vitess ) go 1.19 diff --git a/go/go.sum b/go/go.sum index 695dae96ef..99cbcce9a1 100644 --- a/go/go.sum +++ b/go/go.sum @@ -178,8 +178,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20221012045247-2eb37b2d80e0 h1:lmmFbgh51CHtwM+skbDuzN3j7fdgkLsn0omS1C0TZxA= -github.com/dolthub/go-mysql-server v0.12.1-0.20221012045247-2eb37b2d80e0/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= +github.com/dolthub/go-mysql-server v0.12.1-0.20221012234900-959225d12a05 h1:RShCM0/n2KRc62LS6RAD3ByneDxBE36epAFmDcD/vPc= +github.com/dolthub/go-mysql-server v0.12.1-0.20221012234900-959225d12a05/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= diff --git a/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go b/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go index 458556647f..c95ec501ef 100644 --- a/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go +++ b/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go @@ -534,7 +534,7 @@ func interfaceValueAsSqlString(ti typeinfo.TypeInfo, value interface{}) (string, return "", fmt.Errorf("typeinfo.VarStringTypeIdentifier is not types.String") } return quoteAndEscapeString(string(s)), nil - case typeinfo.GeometryTypeIdentifier, typeinfo.PointTypeIdentifier, typeinfo.LineStringTypeIdentifier, typeinfo.PolygonTypeIdentifier, typeinfo.MultiPointTypeIdentifier, typeinfo.MultiLineStringTypeIdentifier: + case typeinfo.GeometryTypeIdentifier, typeinfo.PointTypeIdentifier, typeinfo.LineStringTypeIdentifier, typeinfo.PolygonTypeIdentifier, typeinfo.MultiPointTypeIdentifier, typeinfo.MultiLineStringTypeIdentifier, typeinfo.MultiPolygonTypeIdentifier: return singleQuote + str + singleQuote, nil default: return str, nil From fe1e31f8226a8f5b75736dc02783fc79c1d6d5c3 Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Thu, 13 Oct 2022 18:28:11 +0000 Subject: [PATCH 09/25] [ga-bump-dep] Bump dependency in Dolt by JCOR11599 --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index a2c4141dea..deb2051813 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/dolthub/go-mysql-server v0.12.1-0.20221012045247-2eb37b2d80e0 + github.com/dolthub/go-mysql-server v0.12.1-0.20221013182632-4d8b14427bf8 github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index 695dae96ef..20635345b9 100644 --- a/go/go.sum +++ b/go/go.sum @@ -178,8 +178,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20221012045247-2eb37b2d80e0 h1:lmmFbgh51CHtwM+skbDuzN3j7fdgkLsn0omS1C0TZxA= -github.com/dolthub/go-mysql-server v0.12.1-0.20221012045247-2eb37b2d80e0/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= +github.com/dolthub/go-mysql-server v0.12.1-0.20221013182632-4d8b14427bf8 h1:f2CIMnbR7mfa8GiSUMDrej7LS3suau8gez7I8wI3pQw= +github.com/dolthub/go-mysql-server v0.12.1-0.20221013182632-4d8b14427bf8/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From d696173acdd59b3d13ea54e1853679d415538cfc Mon Sep 17 00:00:00 2001 From: Zach Musgrave Date: Thu, 13 Oct 2022 12:45:15 -0700 Subject: [PATCH 10/25] Test of pushing newly cloned DB from primary --- .../bats/replication-multidb.bats | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/integration-tests/bats/replication-multidb.bats b/integration-tests/bats/replication-multidb.bats index abf930855d..fc588029af 100644 --- a/integration-tests/bats/replication-multidb.bats +++ b/integration-tests/bats/replication-multidb.bats @@ -119,8 +119,8 @@ SQL dolt clone "file://./rem1/newdb" "dbs2/newdb" # this is a hack: we have to change our persisted global server - # vars for the sql command to work on the replica TODO: fix this - # mess + # vars for the sql command to work on the replica + # TODO: fix this mess dolt config --global --unset sqlserver.global.dolt_replicate_to_remote run dolt sql --data-dir=dbs2 -q "use newdb; show tables" -r csv @@ -129,6 +129,53 @@ SQL [[ "$output" =~ "new_table" ]] || false } +@test "replication-multidb: push newly cloned database" { + pushd . + + mkdir -p "${TMPDIRS}/rem2" + # push all the dbs to remote1 + for i in {1..3}; do + cd "${TMPDIRS}/dbs1/repo${i}" + dolt push remote1 main + # also create a new remote2 for each DB but don't push to it + dolt remote add remote2 "file://../../rem2/repo${i}" + done + + popd + + dolt config --global --add sqlserver.global.dolt_replicate_to_remote remote2 + dolt sql -q "set @@persist.dolt_replication_remote_url_template = 'file://$TMPDIRS/rem2/{database}'" + + mkdir -p "${TMPDIRS}/dbs2" + dolt sql --data-dir=dbs2 < Date: Thu, 13 Oct 2022 19:50:09 +0000 Subject: [PATCH 11/25] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/sqle/database_provider.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/libraries/doltcore/sqle/database_provider.go b/go/libraries/doltcore/sqle/database_provider.go index 90048c8bb6..f478cfab7f 100644 --- a/go/libraries/doltcore/sqle/database_provider.go +++ b/go/libraries/doltcore/sqle/database_provider.go @@ -430,7 +430,7 @@ func ConfigureReplicationDatabaseHook(ctx *sql.Context, p DoltDatabaseProvider, } err = newEnv.AddRemote(r) - if err != env.ErrRemoteAlreadyExists && err != nil { + if err != env.ErrRemoteAlreadyExists && err != nil { return err } @@ -449,9 +449,9 @@ func ConfigureReplicationDatabaseHook(ctx *sql.Context, p DoltDatabaseProvider, // CloneDatabaseFromRemote implements DoltDatabaseProvider interface func (p DoltDatabaseProvider) CloneDatabaseFromRemote( - ctx *sql.Context, - dbName, branch, remoteName, remoteUrl string, - remoteParams map[string]string, + ctx *sql.Context, + dbName, branch, remoteName, remoteUrl string, + remoteParams map[string]string, ) error { p.mu.Lock() defer p.mu.Unlock() From 22b65a833033df4cd19b0dacb08034f0a318d07d Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Thu, 13 Oct 2022 23:18:57 +0000 Subject: [PATCH 12/25] [ga-bump-dep] Bump dependency in Dolt by JCOR11599 --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index deb2051813..5132028397 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/dolthub/go-mysql-server v0.12.1-0.20221013182632-4d8b14427bf8 + github.com/dolthub/go-mysql-server v0.12.1-0.20221013231634-f198cd068a53 github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index 20635345b9..133cf203b6 100644 --- a/go/go.sum +++ b/go/go.sum @@ -178,8 +178,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20221013182632-4d8b14427bf8 h1:f2CIMnbR7mfa8GiSUMDrej7LS3suau8gez7I8wI3pQw= -github.com/dolthub/go-mysql-server v0.12.1-0.20221013182632-4d8b14427bf8/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= +github.com/dolthub/go-mysql-server v0.12.1-0.20221013231634-f198cd068a53 h1:aii+cgN4GBOpbMGKhqB0gZALC01jz0lAkU8ULD+kbEg= +github.com/dolthub/go-mysql-server v0.12.1-0.20221013231634-f198cd068a53/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From f719353aec9e7f66dc0e6d4a272e2c474d8866c3 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 14 Oct 2022 01:13:29 +0000 Subject: [PATCH 13/25] [ga-bump-release] Update Dolt version to 0.50.4 and release v0.50.4 --- go/cmd/dolt/dolt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/cmd/dolt/dolt.go b/go/cmd/dolt/dolt.go index 7314460581..f65d819699 100644 --- a/go/cmd/dolt/dolt.go +++ b/go/cmd/dolt/dolt.go @@ -57,7 +57,7 @@ import ( ) const ( - Version = "0.50.3" + Version = "0.50.4" ) var dumpDocsCommand = &commands.DumpDocsCmd{} From 08226c2e2d4e4b25d35b3317caae70e1b7167586 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 13 Oct 2022 21:30:30 -0700 Subject: [PATCH 14/25] bump --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index 007068d7ec..0ced36df22 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/dolthub/go-mysql-server v0.12.1-0.20221012234900-959225d12a05 + github.com/dolthub/go-mysql-server v0.12.1-0.20221014042841-158f3fe18d21 github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index 99cbcce9a1..a97fc4bcd5 100644 --- a/go/go.sum +++ b/go/go.sum @@ -178,8 +178,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20221012234900-959225d12a05 h1:RShCM0/n2KRc62LS6RAD3ByneDxBE36epAFmDcD/vPc= -github.com/dolthub/go-mysql-server v0.12.1-0.20221012234900-959225d12a05/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= +github.com/dolthub/go-mysql-server v0.12.1-0.20221014042841-158f3fe18d21 h1:ZNpesB9TQy/qHqIe1KvCXauw6koo3pvGNvKWpqEcRg8= +github.com/dolthub/go-mysql-server v0.12.1-0.20221014042841-158f3fe18d21/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From 64fe3f07ccca5945dc9b50e733c2a2845293d687 Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Fri, 14 Oct 2022 05:30:40 +0000 Subject: [PATCH 15/25] [ga-bump-dep] Bump dependency in Dolt by JCOR11599 --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index 0ced36df22..51d8dc8d7d 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/dolthub/go-mysql-server v0.12.1-0.20221014042841-158f3fe18d21 + github.com/dolthub/go-mysql-server v0.12.1-0.20221014051913-cafab36d6df4 github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index a97fc4bcd5..03ea46e9e9 100644 --- a/go/go.sum +++ b/go/go.sum @@ -178,8 +178,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20221014042841-158f3fe18d21 h1:ZNpesB9TQy/qHqIe1KvCXauw6koo3pvGNvKWpqEcRg8= -github.com/dolthub/go-mysql-server v0.12.1-0.20221014042841-158f3fe18d21/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= +github.com/dolthub/go-mysql-server v0.12.1-0.20221014051913-cafab36d6df4 h1:pVk14bKIAJuRVG8GNOI62UuQquC6VkbEjJooKYwr/p8= +github.com/dolthub/go-mysql-server v0.12.1-0.20221014051913-cafab36d6df4/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From 148370e209a533adff2811705e7827842b94f74c Mon Sep 17 00:00:00 2001 From: jennifersp Date: Fri, 14 Oct 2022 11:15:34 -0700 Subject: [PATCH 16/25] update Dockerfile on release --- .github/workflows/cd-push-docker-image.yaml | 34 --------------------- .github/workflows/cd-release.yaml | 18 ++++------- docker/Dockerfile | 2 +- 3 files changed, 7 insertions(+), 47 deletions(-) delete mode 100644 .github/workflows/cd-push-docker-image.yaml diff --git a/.github/workflows/cd-push-docker-image.yaml b/.github/workflows/cd-push-docker-image.yaml deleted file mode 100644 index 563d41636f..0000000000 --- a/.github/workflows/cd-push-docker-image.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Push Docker Image to DockerHub - -on: - workflow_dispatch: - inputs: - version: - description: 'SemVer format release tag, i.e. 0.24.5' - required: true - repository_dispatch: - types: [ push-docker-image ] - -jobs: - docker-image-push: - name: Push Docker Image - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - file: ./docker/Dockerfile - push: true - tags: ${{ secrets.DOCKER_HUB_USERNAME }}/dolt:${{ github.event.inputs.version || github.event.client_payload.version }} , ${{ secrets.DOCKER_HUB_USERNAME }}/dolt:latest - build-args: | - DOLT_VERSION=${{ github.event.inputs.version || github.event.client_payload.version }} diff --git a/.github/workflows/cd-release.yaml b/.github/workflows/cd-release.yaml index 49bc0371b8..1341eab1ad 100644 --- a/.github/workflows/cd-release.yaml +++ b/.github/workflows/cd-release.yaml @@ -37,10 +37,15 @@ jobs: env: FILE: ${{ format('{0}/go/cmd/dolt/dolt.go', github.workspace) }} NEW_VERSION: ${{ needs.format-version.outputs.version }} + - name: Update Dockerfile + run: sed -i -e 's/ARG DOLT_VERSION=.*/ARG DOLT_VERSION='"$NEW_VERSION"'/' "$FILE" + env: + FILE: ${{ format('{0}/docker/Dockerfile', github.workspace) }} + NEW_VERSION: ${{ needs.format-version.outputs.version }} - uses: EndBug/add-and-commit@v7 with: message: ${{ format('[ga-bump-release] Update Dolt version to {0} and release v{0}', needs.format-version.outputs.version) }} - add: ${{ format('{0}/go/cmd/dolt/dolt.go', github.workspace) }} + add: ${{ format('[{0}/go/cmd/dolt/dolt.go, {0}/docker/Dockerfile]', github.workspace) }} cwd: "." - name: Build Binaries id: build_binaries @@ -164,14 +169,3 @@ jobs: token: ${{ secrets.REPO_ACCESS_TOKEN }} event-type: release-dolt client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "actor": "${{ github.actor }}"}' - - docker-image-push: - needs: [ format-version, create-release ] - runs-on: ubuntu-22.04 - steps: - - name: Trigger Push Docker Image - uses: peter-evans/repository-dispatch@v1 - with: - token: ${{ secrets.REPO_ACCESS_TOKEN }} - event-type: push-docker-image - client-payload: '{"version": "${{ needs.format-version.outputs.version }}"}' diff --git a/docker/Dockerfile b/docker/Dockerfile index e48cbc5b99..acdf587298 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.3-labs FROM --platform=linux/amd64 ubuntu:22.04 -ARG DOLT_VERSION +ARG DOLT_VERSION=0.50.4 ADD https://github.com/dolthub/dolt/releases/download/v${DOLT_VERSION}/dolt-linux-amd64.tar.gz dolt-linux-amd64.tar.gz RUN tar zxvf dolt-linux-amd64.tar.gz && \ From 75241c0fbb4e84ca04680a7b9b32c0fd62a0f3f2 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 14 Oct 2022 11:33:06 -0700 Subject: [PATCH 17/25] remote schema.CollCollection.GetAtIndex() in favor of GetByIndex --- .../doltcore/rowconv/field_mapping.go | 11 +++--- go/libraries/doltcore/schema/col_coll.go | 34 ++----------------- go/libraries/doltcore/schema/schema.go | 4 +-- .../doltcore/sqle/index/dolt_index.go | 2 +- go/libraries/doltcore/sqle/sqlfmt/row_fmt.go | 2 +- .../doltcore/sqle/sqlfmt/schema_fmt.go | 4 +-- go/libraries/doltcore/sqle/sqlutil/sql_row.go | 2 +- go/libraries/doltcore/sqle/tables.go | 4 +-- .../sqle/writer/prolly_table_writer.go | 2 +- .../doltcore/table/table_iterator_test.go | 1 - .../doltcore/table/untyped/csv/writer.go | 4 +-- 11 files changed, 19 insertions(+), 51 deletions(-) diff --git a/go/libraries/doltcore/rowconv/field_mapping.go b/go/libraries/doltcore/rowconv/field_mapping.go index 6a12a403da..ae9d5899c2 100644 --- a/go/libraries/doltcore/rowconv/field_mapping.go +++ b/go/libraries/doltcore/rowconv/field_mapping.go @@ -29,9 +29,6 @@ import ( // ErrMappingFileRead is an error returned when a mapping file cannot be read var ErrMappingFileRead = errors.New("error reading mapping file") -// ErrUnmarshallingMapping is an error used when a mapping file cannot be converted from json -var ErrUnmarshallingMapping = errors.New("error unmarshalling mapping") - // ErrEmptyMapping is an error returned when the mapping is empty (No src columns, no destination columns) var ErrEmptyMapping = errors.New("empty mapping error") @@ -276,8 +273,8 @@ func TagMappingByTagAndName(srcSch, destSch schema.Schema) (*FieldMapping, error if j == -1 { continue } - srcTag := srcSch.GetPKCols().GetAtIndex(i).Tag - dstTag := destSch.GetPKCols().GetAtIndex(j).Tag + srcTag := srcSch.GetPKCols().GetByIndex(i).Tag + dstTag := destSch.GetPKCols().GetByIndex(j).Tag srcToDest[srcTag] = dstTag successes++ } @@ -285,8 +282,8 @@ func TagMappingByTagAndName(srcSch, destSch schema.Schema) (*FieldMapping, error if j == -1 { continue } - srcTag := srcSch.GetNonPKCols().GetAtIndex(i).Tag - dstTag := destSch.GetNonPKCols().GetAtIndex(j).Tag + srcTag := srcSch.GetNonPKCols().GetByIndex(i).Tag + dstTag := destSch.GetNonPKCols().GetByIndex(j).Tag srcToDest[srcTag] = dstTag successes++ } diff --git a/go/libraries/doltcore/schema/col_coll.go b/go/libraries/doltcore/schema/col_coll.go index a318b2a93e..7c16aa7534 100644 --- a/go/libraries/doltcore/schema/col_coll.go +++ b/go/libraries/doltcore/schema/col_coll.go @@ -118,10 +118,6 @@ func (cc *ColCollection) GetColumns() []Column { return colsCopy } -func (cc *ColCollection) GetAtIndex(i int) Column { - return cc.cols[i] -} - // GetColumnNames returns a list of names of the columns. func (cc *ColCollection) GetColumnNames() []string { names := make([]string, len(cc.cols)) @@ -177,7 +173,7 @@ func (cc *ColCollection) Iter(cb func(tag uint64, col Column) (stop bool, err er return nil } -// IterInSortOrder iterates over all the columns from lowest tag to highest tag. +// IterInSortedOrder iterates over all the columns from lowest tag to highest tag. func (cc *ColCollection) IterInSortedOrder(cb func(tag uint64, col Column) (stop bool)) { for _, tag := range cc.SortedTags { val := cc.TagToCol[tag] @@ -199,7 +195,7 @@ func (cc *ColCollection) GetByName(name string) (Column, bool) { return InvalidCol, false } -// GetByNameCaseInensitive takes the name of a column and returns the column and true if there is a column with that +// GetByNameCaseInsensitive takes the name of a column and returns the column and true if there is a column with that // name ignoring case. Otherwise InvalidCol and false are returned. If multiple columns have the same case-insensitive // name, the first declared one is returned. func (cc *ColCollection) GetByNameCaseInsensitive(name string) (Column, bool) { @@ -245,39 +241,15 @@ func ColCollsAreEqual(cc1, cc2 *ColCollection) bool { if cc1.Size() != cc2.Size() { return false } - // Pks Cols need to be in the same order and equivalent. for i := 0; i < cc1.Size(); i++ { - if !cc1.GetAtIndex(i).Equals(cc2.GetAtIndex(i)) { + if !cc1.cols[i].Equals(cc2.cols[i]) { return false } } - return true } -// ColCollsAreCompatible determines whether two ColCollections are compatible with each other. Compatible columns have -// the same tags and storage types, but may have different names, constraints or SQL type parameters. -func ColCollsAreCompatible(cc1, cc2 *ColCollection) bool { - if cc1.Size() != cc2.Size() { - return false - } - - areCompatible := true - _ = cc1.Iter(func(tag uint64, col1 Column) (stop bool, err error) { - col2, ok := cc2.GetByTag(tag) - - if !ok || !col1.Compatible(col2) { - areCompatible = false - return true, nil - } - - return false, nil - }) - - return areCompatible -} - // MapColCollection applies a function to each column in a ColCollection and creates a new ColCollection from the results. func MapColCollection(cc *ColCollection, cb func(col Column) Column) *ColCollection { mapped := make([]Column, cc.Size()) diff --git a/go/libraries/doltcore/schema/schema.go b/go/libraries/doltcore/schema/schema.go index 1bd7c8a708..01afd3316c 100644 --- a/go/libraries/doltcore/schema/schema.go +++ b/go/libraries/doltcore/schema/schema.go @@ -209,8 +209,8 @@ func ArePrimaryKeySetsDiffable(format *types.NomsBinFormat, fromSch, toSch Schem } for i := 0; i < cc1.Size(); i++ { - c1 := cc1.GetAtIndex(i) - c2 := cc2.GetAtIndex(i) + c1 := cc1.GetByIndex(i) + c2 := cc2.GetByIndex(i) if (c1.Tag != c2.Tag) || (c1.IsPartOfPK != c2.IsPartOfPK) { return false } diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index bf5f9ce9b2..e1f934d773 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -288,7 +288,7 @@ func (s *durableIndexState) coversAllColumns(i *doltIndex) bool { } covers := true for i := 0; i < cols.Size(); i++ { - col := cols.GetAtIndex(i) + col := cols.GetByIndex(i) if _, ok := idxCols.GetByNameCaseInsensitive(col.Name); !ok { covers = false break diff --git a/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go b/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go index c95ec501ef..9f0fa55b04 100644 --- a/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go +++ b/go/libraries/doltcore/sqle/sqlfmt/row_fmt.go @@ -334,7 +334,7 @@ func SqlRowAsTupleString(r sql.Row, tableSch schema.Schema) (string, error) { if seenOne { b.WriteRune(',') } - col := tableSch.GetAllCols().GetAtIndex(i) + col := tableSch.GetAllCols().GetByIndex(i) str := "NULL" if val != nil { str, err = interfaceValueAsSqlString(col.TypeInfo, val) diff --git a/go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go b/go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go index b59b59539f..89c9105c88 100644 --- a/go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go +++ b/go/libraries/doltcore/sqle/sqlfmt/schema_fmt.go @@ -223,9 +223,9 @@ func AlterTableAddPrimaryKeys(tableName string, pks *schema.ColCollection) strin for i := 0; i < pks.Size(); i++ { if i == 0 { - b.WriteString(pks.GetAtIndex(i).Name) + b.WriteString(pks.GetByIndex(i).Name) } else { - b.WriteString("," + pks.GetAtIndex(i).Name) + b.WriteString("," + pks.GetByIndex(i).Name) } } b.WriteRune(')') diff --git a/go/libraries/doltcore/sqle/sqlutil/sql_row.go b/go/libraries/doltcore/sqle/sqlutil/sql_row.go index a2ba4c8354..b3e4f61f84 100644 --- a/go/libraries/doltcore/sqle/sqlutil/sql_row.go +++ b/go/libraries/doltcore/sqle/sqlutil/sql_row.go @@ -142,7 +142,7 @@ func DoltKeyAndMappingFromSqlRow(ctx context.Context, vrw types.ValueReadWriter, } for i := 0; i < numCols; i++ { - schCol := allCols.GetAtIndex(i) + schCol := allCols.GetByIndex(i) val := r[i] if val == nil { continue diff --git a/go/libraries/doltcore/sqle/tables.go b/go/libraries/doltcore/sqle/tables.go index 5fa2534c3e..124b37e6a1 100644 --- a/go/libraries/doltcore/sqle/tables.go +++ b/go/libraries/doltcore/sqle/tables.go @@ -2110,7 +2110,7 @@ func (t *AlterableDoltTable) AddForeignKey(ctx *sql.Context, sqlFk sql.ForeignKe //TODO: use the primary key as-is var refPkTags []uint64 for _, i := range refSch.GetPkOrdinals() { - refPkTags = append(refPkTags, refSch.GetAllCols().GetAtIndex(i).Tag) + refPkTags = append(refPkTags, refSch.GetAllCols().GetByIndex(i).Tag) } var colNames []string @@ -2328,7 +2328,7 @@ func (t *AlterableDoltTable) UpdateForeignKey(ctx *sql.Context, fkName string, s //TODO: use the primary key as-is var refPkTags []uint64 for _, i := range refSch.GetPkOrdinals() { - refPkTags = append(refPkTags, refSch.GetAllCols().GetAtIndex(i).Tag) + refPkTags = append(refPkTags, refSch.GetAllCols().GetByIndex(i).Tag) } var colNames []string diff --git a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go index 448935ae9a..012378f573 100755 --- a/go/libraries/doltcore/sqle/writer/prolly_table_writer.go +++ b/go/libraries/doltcore/sqle/writer/prolly_table_writer.go @@ -368,7 +368,7 @@ func ordinalMappingsFromSchema(from sql.Schema, to schema.Schema) (km, vm val.Or func makeOrdinalMapping(from sql.Schema, to *schema.ColCollection) (m val.OrdinalMapping) { m = make(val.OrdinalMapping, len(to.GetColumns())) for i := range m { - name := to.GetAtIndex(i).Name + name := to.GetByIndex(i).Name for j, col := range from { if col.Name == name { m[i] = j diff --git a/go/libraries/doltcore/table/table_iterator_test.go b/go/libraries/doltcore/table/table_iterator_test.go index ad4f5f451a..6ed9e4e034 100644 --- a/go/libraries/doltcore/table/table_iterator_test.go +++ b/go/libraries/doltcore/table/table_iterator_test.go @@ -61,7 +61,6 @@ func testIterator(t *testing.T, iter RowIter, expected []sql.Row) { require.Equal(t, io.EOF, err) } -var colSqlType = sch.GetAllCols().GetAtIndex(0).TypeInfo.ToSqlType() var sch = schema.MustSchemaFromCols(schema.NewColCollection( schema.NewColumn("pk", 0, types.UintKind, true), schema.NewColumn("col1", 1, types.UintKind, false))) diff --git a/go/libraries/doltcore/table/untyped/csv/writer.go b/go/libraries/doltcore/table/untyped/csv/writer.go index 1c3d87c84c..1861bdb487 100644 --- a/go/libraries/doltcore/table/untyped/csv/writer.go +++ b/go/libraries/doltcore/table/untyped/csv/writer.go @@ -95,7 +95,7 @@ func (csvw *CSVWriter) WriteRow(ctx context.Context, r row.Row) error { if val == nil { colValStrs[i] = nil } else { - v, err := sqlutil.SqlColToStr(csvw.sch.GetAllCols().GetAtIndex(i).TypeInfo.ToSqlType(), val) + v, err := sqlutil.SqlColToStr(csvw.sch.GetAllCols().GetByIndex(i).TypeInfo.ToSqlType(), val) if err != nil { return err } @@ -114,7 +114,7 @@ func (csvw *CSVWriter) WriteSqlRow(ctx context.Context, r sql.Row) error { } else { var v string var err error - colType := csvw.sch.GetAllCols().GetAtIndex(i).TypeInfo.ToSqlType() + colType := csvw.sch.GetAllCols().GetByIndex(i).TypeInfo.ToSqlType() // Due to BIT's unique output, we special-case writing the integer specifically for CSV if _, ok := colType.(sql.BitType); ok { v = strconv.FormatUint(val.(uint64), 10) From 7cb535e3d94a4dedcb5d143a44e07cedda6b2793 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 14 Oct 2022 11:46:40 -0700 Subject: [PATCH 18/25] one more usage --- go/libraries/doltcore/sqle/enginetest/validation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/enginetest/validation.go b/go/libraries/doltcore/sqle/enginetest/validation.go index a55b5b942a..4f2404b048 100644 --- a/go/libraries/doltcore/sqle/enginetest/validation.go +++ b/go/libraries/doltcore/sqle/enginetest/validation.go @@ -240,7 +240,7 @@ func ordinalMappingsForSecondaryIndex(sch schema.Schema, def schema.Index) (ord ord = make(val.OrdinalMapping, secondary.Size()) for i := range ord { - name := secondary.GetAtIndex(i).Name + name := secondary.GetByIndex(i).Name ord[i] = -1 pks := sch.GetPKCols().GetColumns() From 24108f14a2413567d28e8e32c133bf5b277fd2d7 Mon Sep 17 00:00:00 2001 From: jennifersp <44716627+jennifersp@users.noreply.github.com> Date: Fri, 14 Oct 2022 14:07:35 -0700 Subject: [PATCH 19/25] get unused port number and stop running server only (#4550) --- .../bats/helper/query-server-common.bash | 41 +++++++++++++------ integration-tests/bats/sql-jwt-auth.bats | 2 +- integration-tests/bats/sql-privs.bats | 8 ++-- integration-tests/bats/sql-server.bats | 29 +++++++------ .../mysql-client-tests.bats | 16 +++++++- integration-tests/orm-tests/orm-tests.bats | 2 +- 6 files changed, 67 insertions(+), 31 deletions(-) diff --git a/integration-tests/bats/helper/query-server-common.bash b/integration-tests/bats/helper/query-server-common.bash index 7a4e0a175e..0175c20962 100644 --- a/integration-tests/bats/helper/query-server-common.bash +++ b/integration-tests/bats/helper/query-server-common.bash @@ -113,7 +113,7 @@ wait_for_connection(port=int(port_str), timeout_ms=int(timeout_ms), database=dat start_sql_server() { DEFAULT_DB="$1" - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) dolt sql-server --host 0.0.0.0 --port=$PORT --user dolt & SERVER_PID=$! wait_for_connection $PORT 5000 @@ -124,7 +124,7 @@ start_sql_server() { # this func) start_sql_server_with_args() { DEFAULT_DB="" - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) dolt sql-server "$@" --port=$PORT & SERVER_PID=$! wait_for_connection $PORT 5000 @@ -132,7 +132,7 @@ start_sql_server_with_args() { start_sql_server_with_config() { DEFAULT_DB="$1" - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) echo " log_level: debug @@ -155,7 +155,7 @@ behavior: start_sql_multi_user_server() { DEFAULT_DB="$1" - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) echo " log_level: debug @@ -177,7 +177,7 @@ behavior: start_multi_db_server() { DEFAULT_DB="$1" - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) dolt sql-server --host 0.0.0.0 --port=$PORT --user dolt --data-dir ./ & SERVER_PID=$! wait_for_connection $PORT 5000 @@ -186,15 +186,19 @@ start_multi_db_server() { # stop_sql_server stops the SQL server. For cases where it's important # to wait for the process to exit after the kill signal (e.g. waiting # for an async replication push), pass 1. +# kill the process if it's still running stop_sql_server() { wait=$1 if [ ! -z "$SERVER_PID" ]; then - kill $SERVER_PID - if [ $wait ]; then - while ps -p $SERVER_PID > /dev/null; do - sleep .1; - done - fi; + serverpidinuse=$(lsof -i -P -n | grep LISTEN | grep $SERVER_PID | wc -l) + if [ $serverpidinuse -gt 0 ]; then + kill $SERVER_PID + if [ $wait ]; then + while ps -p $SERVER_PID > /dev/null; do + sleep .1; + done + fi; + fi fi SERVER_PID= } @@ -233,7 +237,7 @@ stop_sql_server() { # * param7: Expected exception value of 1. Mutually exclusive with param6. # server_query() { - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) server_query_with_port "$PORT" "$@" } @@ -246,3 +250,16 @@ server_query_with_port() { echo Executing server_query python3 -u -c "$PYTHON_QUERY_SCRIPT" -- "$PYTEST_DIR" "$1" "$PORT" "$2" "$3" "$4" "$5" "$6" "$7" } + +definePORT() { + getPORT="" + for i in {0..9} + do + let getPORT="($$ + $i) % (65536-1024) + 1024" + portinuse=$(lsof -i -P -n | grep LISTEN | grep $attemptedPORT | wc -l) + if [ $portinuse -eq 0 ]; then + echo "$getPORT" + break + fi + done +} diff --git a/integration-tests/bats/sql-jwt-auth.bats b/integration-tests/bats/sql-jwt-auth.bats index 4049f7998c..58284e5a4a 100644 --- a/integration-tests/bats/sql-jwt-auth.bats +++ b/integration-tests/bats/sql-jwt-auth.bats @@ -29,7 +29,7 @@ teardown() { cp "$BATS_TEST_DIRNAME"/../../go/cmd/dolt/commands/sqlserver/testdata/chain_key.pem . cp "$BATS_TEST_DIRNAME"/../../go/cmd/dolt/commands/sqlserver/testdata/chain_cert.pem . cp "$BATS_TEST_DIRNAME"/../../go/libraries/utils/jwtauth/gen_keys/test_jwks.json . - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) TOKEN="`cat $BATS_TEST_DIRNAME/../../go/libraries/utils/jwtauth/gen_keys/token.jwt`" cat >config.yml < log.txt 2>&1 & SERVER_PID=$! sleep 5 @@ -684,7 +684,7 @@ SQL skiponwindows "Missing dependencies" cd repo1 dolt sql -q 'create table test (id int primary key)' - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) cat >config.yml <config.yml <> log.txt 2>&1 & SERVER_PID=$! @@ -1228,7 +1228,7 @@ databases: skiponwindows "unix socket is not available on Windows" cd repo2 DEFAULT_DB="repo2" - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) dolt sql-server --port $PORT --user dolt --socket > log.txt 2>&1 & SERVER_PID=$! @@ -1249,7 +1249,7 @@ databases: run pwd REPO_NAME=$output - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) dolt sql-server --port=$PORT --socket="$REPO_NAME/mysql.sock" --user dolt > log.txt 2>&1 & SERVER_PID=$! run wait_for_connection $PORT 5000 @@ -1264,7 +1264,7 @@ databases: skiponwindows "unix socket is not available on Windows" cd repo2 DEFAULT_DB="repo2" - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) echo " log_level: debug @@ -1382,7 +1382,7 @@ s.close() run dolt init --new-format [ $status -eq 0 ] - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) dolt sql-server --host 0.0.0.0 --port=$PORT --user dolt & SERVER_PID=$! # will get killed by teardown_common sleep 5 # not using python wait so this works on windows @@ -1420,6 +1420,11 @@ s.close() run grep "failed to access 'mydb2' database: can no longer find .dolt dir on disk" server_log.txt [ "${#lines[@]}" -eq 1 ] + + # this tests fails sometimes as the server is stopped from the above error + # but stop_sql_server in teardown tries to kill process that is not running anymore, + # so start the server again, and it will be stopped in teardown + start_sql_server } @test "sql-server: dropping database that the server is running in should drop only the db itself not its nested dbs" { diff --git a/integration-tests/mysql-client-tests/mysql-client-tests.bats b/integration-tests/mysql-client-tests/mysql-client-tests.bats index 992f73b1b1..5c76c8dd07 100644 --- a/integration-tests/mysql-client-tests/mysql-client-tests.bats +++ b/integration-tests/mysql-client-tests/mysql-client-tests.bats @@ -19,7 +19,7 @@ setup() { dolt sql -q "CREATE TABLE warehouse(warehouse_id int primary key, warehouse_name longtext)" dolt sql -q "INSERT into warehouse VALUES (1, 'UPS'), (2, 'TV'), (3, 'Table');" - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) USER="dolt" dolt sql-server --host 0.0.0.0 --port=$PORT --user=$USER --loglevel=trace & SERVER_PID=$! @@ -167,3 +167,17 @@ EOF" -m "postgres" @test "R RMariaDB client" { Rscript $BATS_TEST_DIRNAME/r/rmariadb-test.r $USER $PORT $REPO_NAME } + +definePORT() { + getPORT="" + for i in {0..9} + do + let getPORT="($$ + $i) % (65536-1024) + 1024" + portinuse=$(lsof -i -P -n | grep LISTEN | grep $attemptedPORT | wc -l) + if [ $portinuse -eq 0 ] + then + echo "$getPORT" + break + fi + done +} diff --git a/integration-tests/orm-tests/orm-tests.bats b/integration-tests/orm-tests/orm-tests.bats index f27958775c..dc21f4ccb9 100644 --- a/integration-tests/orm-tests/orm-tests.bats +++ b/integration-tests/orm-tests/orm-tests.bats @@ -7,7 +7,7 @@ setup() { dolt init - let PORT="$$ % (65536-1024) + 1024" + PORT=$( definePORT ) USER="dolt" dolt sql-server --host 0.0.0.0 --port=$PORT --user=$USER --loglevel=trace & SERVER_PID=$! From 201151cd1fecb02b72363c8546b7cca4db9942fa Mon Sep 17 00:00:00 2001 From: JCOR11599 Date: Fri, 14 Oct 2022 21:48:24 +0000 Subject: [PATCH 20/25] [ga-bump-dep] Bump dependency in Dolt by JCOR11599 --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index 51d8dc8d7d..c1b1eaa8d3 100644 --- a/go/go.mod +++ b/go/go.mod @@ -57,7 +57,7 @@ require ( require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible github.com/cenkalti/backoff/v4 v4.1.3 - github.com/dolthub/go-mysql-server v0.12.1-0.20221014051913-cafab36d6df4 + github.com/dolthub/go-mysql-server v0.12.1-0.20221014214651-c2bf09248ff9 github.com/google/flatbuffers v2.0.6+incompatible github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 github.com/mitchellh/go-ps v1.0.0 diff --git a/go/go.sum b/go/go.sum index 03ea46e9e9..87c19d39fb 100644 --- a/go/go.sum +++ b/go/go.sum @@ -178,8 +178,8 @@ github.com/dolthub/flatbuffers v1.13.0-dh.1 h1:OWJdaPep22N52O/0xsUevxJ6Qfw1M2txC github.com/dolthub/flatbuffers v1.13.0-dh.1/go.mod h1:CorYGaDmXjHz1Z7i50PYXG1Ricn31GcA2wNOTFIQAKE= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.12.1-0.20221014051913-cafab36d6df4 h1:pVk14bKIAJuRVG8GNOI62UuQquC6VkbEjJooKYwr/p8= -github.com/dolthub/go-mysql-server v0.12.1-0.20221014051913-cafab36d6df4/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= +github.com/dolthub/go-mysql-server v0.12.1-0.20221014214651-c2bf09248ff9 h1:xGTHflXQ6gWZFvgXIyHHrBtmIEWoLEc70dSY6SGPl/g= +github.com/dolthub/go-mysql-server v0.12.1-0.20221014214651-c2bf09248ff9/go.mod h1:9Q9FhWO82GrV4he13V2ZuDE0T/eDZbPVMOWLcZluOvg= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From 1490a68095c38d550484c76038728aaed3d95a3a Mon Sep 17 00:00:00 2001 From: jennifersp <44716627+jennifersp@users.noreply.github.com> Date: Fri, 14 Oct 2022 15:52:46 -0700 Subject: [PATCH 21/25] add list of `dolt config` variables to command description (#4538) --- go/cmd/dolt/commands/config.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/go/cmd/dolt/commands/config.go b/go/cmd/dolt/commands/config.go index 714dcb928f..77df592c45 100644 --- a/go/cmd/dolt/commands/config.go +++ b/go/cmd/dolt/commands/config.go @@ -40,9 +40,21 @@ var cfgDocs = cli.CommandDocumentationContent{ ShortDesc: `Get and set repository or global options`, LongDesc: `You can query/set/replace/unset options with this command. - When reading, the values are read from the global and repository local configuration files, and options {{.LessThan}}--global{{.GreaterThan}}, and {{.LessThan}}--local{{.GreaterThan}} can be used to tell the command to read from only that location. - - When writing, the new value is written to the repository local configuration file by default, and options {{.LessThan}}--global{{.GreaterThan}}, can be used to tell the command to write to that location (you can say {{.LessThan}}--local{{.GreaterThan}} but that is the default). +When reading, the values are read from the global and repository local configuration files, and options {{.LessThan}}--global{{.GreaterThan}}, and {{.LessThan}}--local{{.GreaterThan}} can be used to tell the command to read from only that location. + +When writing, the new value is written to the repository local configuration file by default, and options {{.LessThan}}--global{{.GreaterThan}}, can be used to tell the command to write to that location (you can say {{.LessThan}}--local{{.GreaterThan}} but that is the default). + +Valid configuration variables: + - core.editor - lets you edit 'commit' or 'tag' messages by launching the set editor. + - creds.add_url - sets the endpoint used to authenticate a client for 'dolt login'. + - doltlab.insecure - boolean flag used to authenticate a client against DoltLab. + - init.defaultbranch - allows overriding the default branch name e.g. when initializing a new repository. + - metrics.disabled - boolean flag disables sending metrics when true. + - user.creds - sets user keypairs for authenticating with doltremoteapi + - user.email - sets name used in the author and committer field of commit objects + - user.name - sets email used in the author and committer field of commit objects + - remotes.default_host - sets default host for authenticating eith doltremoteapi + - remotes.default_port - sets default port for authenticating eith doltremoteapi `, Synopsis: []string{ From b2e3373e3356365a0b5782bfea5bd3916acb6528 Mon Sep 17 00:00:00 2001 From: andrew-wm-arthur Date: Mon, 17 Oct 2022 15:18:16 +0000 Subject: [PATCH 22/25] [ga-bump-release] Update Dolt version to 0.50.5 and release v0.50.5 --- docker/Dockerfile | 2 +- go/cmd/dolt/dolt.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index acdf587298..c993d86318 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.3-labs FROM --platform=linux/amd64 ubuntu:22.04 -ARG DOLT_VERSION=0.50.4 +ARG DOLT_VERSION=0.50.5 ADD https://github.com/dolthub/dolt/releases/download/v${DOLT_VERSION}/dolt-linux-amd64.tar.gz dolt-linux-amd64.tar.gz RUN tar zxvf dolt-linux-amd64.tar.gz && \ diff --git a/go/cmd/dolt/dolt.go b/go/cmd/dolt/dolt.go index f65d819699..577b11da35 100644 --- a/go/cmd/dolt/dolt.go +++ b/go/cmd/dolt/dolt.go @@ -57,7 +57,7 @@ import ( ) const ( - Version = "0.50.4" + Version = "0.50.5" ) var dumpDocsCommand = &commands.DumpDocsCmd{} From e72052eadf1ec450eba63da4e83e4b88e9401b0b Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 17 Oct 2022 10:23:54 -0700 Subject: [PATCH 23/25] go/doltcore/rowconv: cleaup pkg rowconv --- .../doltcore/doltdb/foreign_key_coll.go | 128 ------------------ .../doltcore/rowconv/field_mapping.go | 106 --------------- .../doltcore/rowconv/field_mapping_test.go | 4 - .../doltcore/rowconv/row_converter.go | 6 - .../doltcore/rowconv/row_converter_test.go | 51 ------- 5 files changed, 295 deletions(-) diff --git a/go/libraries/doltcore/doltdb/foreign_key_coll.go b/go/libraries/doltcore/doltdb/foreign_key_coll.go index 90e26a559c..daff2dae1e 100644 --- a/go/libraries/doltcore/doltdb/foreign_key_coll.go +++ b/go/libraries/doltcore/doltdb/foreign_key_coll.go @@ -20,13 +20,9 @@ import ( "encoding/binary" "errors" "fmt" - "io" "sort" "strings" - "github.com/dolthub/dolt/go/libraries/doltcore/rowconv" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/utils/set" @@ -677,130 +673,6 @@ func (refOp ForeignKeyReferentialAction) ReducedString() string { } } -// ValidateData ensures that the foreign key is valid by comparing the index data from the given table -// against the index data from the referenced table. Returns an error for each violation. -func (fk ForeignKey) ValidateData( - ctx context.Context, - childSch schema.Schema, - childRowData, childIdxData, parentIdxData types.Map, - childDef, parentDef schema.Index, - vrw types.ValueReadWriter, -) error { - // Any unresolved foreign key does not yet reference any indexes, therefore there's no need to error here. - // Data will be validated whenever the foreign key is resolved. - if !fk.IsResolved() { - return nil - } - if fk.ReferencedTableIndex != parentDef.Name() { - return fmt.Errorf("cannot validate data as wrong referenced index was given: expected `%s` but received `%s`", - fk.ReferencedTableIndex, parentDef.Name()) - } - - tagMap := make(map[uint64]uint64, len(fk.TableColumns)) - for i, childTag := range fk.TableColumns { - tagMap[childTag] = fk.ReferencedTableColumns[i] - } - - // FieldMappings ignore columns not in the tagMap - fm, err := rowconv.NewFieldMapping(childDef.Schema(), parentDef.Schema(), tagMap) - if err != nil { - return err - } - - rc, err := rowconv.NewRowConverter(ctx, vrw, fm) - if err != nil { - return err - } - - rdr, err := noms.NewNomsMapReader(ctx, childIdxData, childDef.Schema()) - if err != nil { - return err - } - - var violatingRows []row.Row - for { - childIdxRow, err := rdr.ReadRow(ctx) - if err == io.EOF { - break - } - if err != nil { - return err - } - - // Check if there are any NULL values, as they should be skipped - hasNulls := false - _, err = childIdxRow.IterSchema(childDef.Schema(), func(tag uint64, val types.Value) (stop bool, err error) { - if types.IsNull(val) { - hasNulls = true - return true, nil - } - return false, nil - }) - if err != nil { - return err - } - if hasNulls { - continue - } - - parentIdxRow, err := rc.Convert(childIdxRow) - if err != nil { - return err - } - if row.IsEmpty(parentIdxRow) { - continue - } - - partial, err := row.ReduceToIndexPartialKey(parentDef, parentIdxRow) - if err != nil { - return err - } - - indexIter := noms.NewNomsRangeReader(parentDef.Schema(), parentIdxData, - []*noms.ReadRange{{Start: partial, Inclusive: true, Reverse: false, Check: noms.InRangeCheckPartial(partial)}}, - ) - - switch _, err = indexIter.ReadRow(ctx); err { - case nil: - continue // parent table contains child key - case io.EOF: - childFullKey, err := childIdxRow.NomsMapKey(childDef.Schema()).Value(ctx) - if err != nil { - return err - } - childKey, err := childDef.ToTableTuple(ctx, childFullKey.(types.Tuple), childIdxRow.Format()) - if err != nil { - return err - } - childVal, ok, err := childRowData.MaybeGetTuple(ctx, childKey) - if err != nil { - return err - } - if !ok { - childKeyStr, _ := types.EncodedValue(ctx, childKey) - return fmt.Errorf("could not find row value for key %s on table `%s`", childKeyStr, fk.TableName) - } - childRow, err := row.FromNoms(childSch, childKey, childVal) - if err != nil { - return err - } - violatingRows = append(violatingRows, childRow) - default: - return err - } - } - - if len(violatingRows) == 0 { - return nil - } else { - return &ForeignKeyViolationError{ - ForeignKey: fk, - Schema: childSch, - ViolationRows: violatingRows, - } - } -} - // ColumnHasFkRelationship returns a foreign key that uses this tag. Returns n func (fkc *ForeignKeyCollection) ColumnHasFkRelationship(tag uint64) (ForeignKey, bool) { for _, key := range fkc.AllKeys() { diff --git a/go/libraries/doltcore/rowconv/field_mapping.go b/go/libraries/doltcore/rowconv/field_mapping.go index ae9d5899c2..0ac5bf81d9 100644 --- a/go/libraries/doltcore/rowconv/field_mapping.go +++ b/go/libraries/doltcore/rowconv/field_mapping.go @@ -21,9 +21,7 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/errhand" "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/table/untyped" "github.com/dolthub/dolt/go/libraries/utils/filesys" - "github.com/dolthub/dolt/go/libraries/utils/set" ) // ErrMappingFileRead is an error returned when a mapping file cannot be read @@ -43,12 +41,6 @@ func (err *BadMappingErr) Error() string { return fmt.Sprintf("Mapping file attempted to map %s to %s, but one or both of those fields are unknown.", err.srcField, err.destField) } -// IsBadMappingErr returns true if the error is a BadMappingErr -func IsBadMappingErr(err error) bool { - _, ok := err.(*BadMappingErr) - return ok -} - // NameMapper is a simple interface for mapping a string to another string type NameMapper map[string]string @@ -83,34 +75,6 @@ type FieldMapping struct { SrcToDest map[uint64]uint64 } -// MapsAllDestPKs checks that each PK column in DestSch has a corresponding column in SrcSch -func (fm *FieldMapping) MapsAllDestPKs() bool { - ds := set.NewUint64Set(nil) - for _, v := range fm.SrcToDest { - ds.Add(v) - } - for _, tag := range fm.DestSch.GetPKCols().Tags { - if !ds.Contains(tag) { - return false - } - } - return true -} - -func InvertMapping(fm *FieldMapping) *FieldMapping { - invertedMap := make(map[uint64]uint64) - - for k, v := range fm.SrcToDest { - invertedMap[v] = k - } - - return &FieldMapping{ - SrcSch: fm.DestSch, - DestSch: fm.SrcSch, - SrcToDest: invertedMap, - } -} - // NewFieldMapping creates a FieldMapping from a source schema, a destination schema, and a map from tags in the source // schema to tags in the dest schema. func NewFieldMapping(srcSch, destSch schema.Schema, srcTagToDestTag map[uint64]uint64) (*FieldMapping, error) { @@ -131,15 +95,6 @@ func NewFieldMapping(srcSch, destSch schema.Schema, srcTagToDestTag map[uint64]u return &FieldMapping{srcSch, destSch, srcTagToDestTag}, nil } -// Returns the identity mapping for the schema given. -func IdentityMapping(sch schema.Schema) *FieldMapping { - fieldMapping, err := TagMapping(sch, sch) - if err != nil { - panic("Error creating identity mapping") - } - return fieldMapping -} - // TagMapping takes a source schema and a destination schema and maps all columns which have a matching tag in the // source and destination schemas. func TagMapping(srcSch, destSch schema.Schema) (*FieldMapping, error) { @@ -223,41 +178,6 @@ func NameMapperFromFile(mappingFile string, FS filesys.ReadableFS) (NameMapper, return nm, nil } -// TagMappingWithNameFallback takes a source schema and a destination schema and maps columns -// by matching tags first, then attempts to match by column name for any columns that didn't -// match with an exact tag. -func TagMappingWithNameFallback(srcSch, destSch schema.Schema) (*FieldMapping, error) { - successes := 0 - srcCols := srcSch.GetAllCols() - destCols := destSch.GetAllCols() - - srcToDest := make(map[uint64]uint64, destCols.Size()) - err := destCols.Iter(func(tag uint64, col schema.Column) (stop bool, err error) { - srcCol, ok := srcCols.GetByTag(tag) - if !ok { - srcCol, ok = srcCols.GetByName(col.Name) - - if !ok { - return false, nil - } - } - - srcToDest[srcCol.Tag] = col.Tag - successes++ - - return false, nil - }) - if err != nil { - return nil, err - } - - if successes == 0 { - return nil, ErrEmptyMapping - } - - return NewFieldMapping(srcSch, destSch, srcToDest) -} - // TagMappingByTagAndName takes a source schema and a destination schema and maps // pks by tag and non-pks by name. func TagMappingByTagAndName(srcSch, destSch schema.Schema) (*FieldMapping, error) { @@ -294,29 +214,3 @@ func TagMappingByTagAndName(srcSch, destSch schema.Schema) (*FieldMapping, error return NewFieldMapping(srcSch, destSch, srcToDest) } - -// TypedToUntypedMapping takes a schema and creates a mapping to an untyped schema with all the same columns. -func TypedToUntypedMapping(sch schema.Schema) (*FieldMapping, error) { - untypedSch, err := untyped.UntypeSchema(sch) - if err != nil { - return nil, err - } - - identityMap := make(map[uint64]uint64) - err = sch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) { - identityMap[tag] = tag - return false, nil - }) - - if err != nil { - return nil, err - } - - mapping, err := NewFieldMapping(sch, untypedSch, identityMap) - - if err != nil { - panic(err) - } - - return mapping, nil -} diff --git a/go/libraries/doltcore/rowconv/field_mapping_test.go b/go/libraries/doltcore/rowconv/field_mapping_test.go index 6483dbd223..87df1744a4 100644 --- a/go/libraries/doltcore/rowconv/field_mapping_test.go +++ b/go/libraries/doltcore/rowconv/field_mapping_test.go @@ -105,10 +105,6 @@ func TestFieldMapping(t *testing.T) { if !reflect.DeepEqual(mapping.SrcToDest, test.expected) { t.Error("Mapping does not match expected. Expected:", test.expected, "Actual:", mapping.SrcToDest) } - - //if test.identity != mapping.IsIdentityMapping() { - // t.Error("identity expected", test.identity, "actual:", !test.identity) - //} } } } diff --git a/go/libraries/doltcore/rowconv/row_converter.go b/go/libraries/doltcore/rowconv/row_converter.go index 430dffa5ac..ae977b30bc 100644 --- a/go/libraries/doltcore/rowconv/row_converter.go +++ b/go/libraries/doltcore/rowconv/row_converter.go @@ -100,12 +100,6 @@ func (rc *RowConverter) ConvertWithWarnings(inRow row.Row, warnFn WarnFunction) return rc.convert(inRow, warnFn) } -// Convert takes an input row, maps its columns to destination columns, and performs any type conversion needed to -// create a row of the expected destination schema. -func (rc *RowConverter) Convert(inRow row.Row) (row.Row, error) { - return rc.convert(inRow, nil) -} - // convert takes a row and maps its columns to their destination columns, automatically performing any type conversion // needed, and using the optional WarnFunction to let callers log warnings on any type conversion errors. func (rc *RowConverter) convert(inRow row.Row, warnFn WarnFunction) (row.Row, error) { diff --git a/go/libraries/doltcore/rowconv/row_converter_test.go b/go/libraries/doltcore/rowconv/row_converter_test.go index c8c6ff8692..6a3bc940e9 100644 --- a/go/libraries/doltcore/rowconv/row_converter_test.go +++ b/go/libraries/doltcore/rowconv/row_converter_test.go @@ -17,12 +17,7 @@ package rowconv import ( "context" "testing" - "time" - "github.com/google/uuid" - "github.com/stretchr/testify/require" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/store/types" ) @@ -39,52 +34,6 @@ var srcCols = schema.NewColCollection( var srcSch = schema.MustSchemaFromCols(srcCols) -func TestRowConverter(t *testing.T) { - mapping, err := TypedToUntypedMapping(srcSch) - - require.NoError(t, err) - - vrw := types.NewMemoryValueStore() - rConv, err := NewRowConverter(context.Background(), vrw, mapping) - - if err != nil { - t.Fatal("Error creating row converter") - } - - id, _ := uuid.NewRandom() - tt := types.Timestamp(time.Now()) - inRow, err := row.New(vrw.Format(), srcSch, row.TaggedValues{ - 0: types.UUID(id), - 1: types.Float(1.25), - 2: types.Uint(12345678), - 3: types.Bool(true), - 4: types.Int(-1234), - 5: types.String("string string string"), - 6: tt, - }) - - require.NoError(t, err) - outData, err := rConv.Convert(inRow) - require.NoError(t, err) - - destSch := mapping.DestSch - expected, err := row.New(vrw.Format(), destSch, row.TaggedValues{ - 0: types.String(id.String()), - 1: types.String("1.25"), - 2: types.String("12345678"), - 3: types.String("1"), - 4: types.String("-1234"), - 5: types.String("string string string"), - 6: types.String(tt.String()), - }) - - require.NoError(t, err) - - if !row.AreEqual(outData, expected, destSch) { - t.Error("\n", row.Fmt(context.Background(), expected, destSch), "!=\n", row.Fmt(context.Background(), outData, destSch)) - } -} - func TestUnneccessaryConversion(t *testing.T) { mapping, err := TagMapping(srcSch, srcSch) if err != nil { From 0d4b3cce3ffd4a2bcab3c79295860ffce991d0f8 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 17 Oct 2022 11:50:45 -0700 Subject: [PATCH 24/25] remove dead, deprecated code from table packages --- go/cmd/dolt/commands/engine/sql_print.go | 3 +- go/libraries/doltcore/merge/merge_rows.go | 26 --- .../doltcore/merge/violations_fk_prolly.go | 4 - .../doltcore/table/composite_table_reader.go | 92 ---------- .../table/composite_table_reader_test.go | 93 ----------- .../table/editor/keyless_table_editor.go | 45 ++++- go/libraries/doltcore/table/inmem_table.go | 25 --- .../doltcore/table/inmem_table_test.go | 43 +---- go/libraries/doltcore/table/io.go | 6 +- go/libraries/doltcore/table/keyless_reader.go | 157 ------------------ .../doltcore/table/map_point_reader.go | 82 --------- go/libraries/doltcore/table/pk_reader.go | 153 ----------------- .../doltcore/table/read_ahead_table_reader.go | 71 -------- go/libraries/doltcore/table/table_writer.go | 16 +- .../doltcore/table/typed/json/writer.go | 73 -------- .../doltcore/table/typed/parquet/writer.go | 10 -- .../table/typed/parquet/writer_test.go | 38 +---- .../doltcore/table/untyped/csv/line.go | 4 - .../doltcore/table/untyped/csv/writer.go | 26 --- .../doltcore/table/untyped/csv/writer_test.go | 33 ++-- .../untyped/sqlexport/batch_sqlwriter.go | 59 ------- .../table/untyped/sqlexport/sqlwriter.go | 16 -- .../table/untyped/sqlexport/sqlwriter_test.go | 15 +- .../untyped/tabular/fixedwidth_tablewriter.go | 5 - go/store/types/edits/disk_backed.go | 8 - 25 files changed, 75 insertions(+), 1028 deletions(-) delete mode 100644 go/libraries/doltcore/table/composite_table_reader.go delete mode 100644 go/libraries/doltcore/table/composite_table_reader_test.go delete mode 100644 go/libraries/doltcore/table/keyless_reader.go delete mode 100644 go/libraries/doltcore/table/map_point_reader.go delete mode 100644 go/libraries/doltcore/table/pk_reader.go delete mode 100644 go/libraries/doltcore/table/read_ahead_table_reader.go diff --git a/go/cmd/dolt/commands/engine/sql_print.go b/go/cmd/dolt/commands/engine/sql_print.go index fdd8879007..1d07b431cc 100644 --- a/go/cmd/dolt/commands/engine/sql_print.go +++ b/go/cmd/dolt/commands/engine/sql_print.go @@ -187,9 +187,8 @@ func secondsSince(start time.Time) float64 { // nullWriter is a no-op SqlRowWriter implementation type nullWriter struct{} -func (n nullWriter) WriteRow(ctx context.Context, r row.Row) error { return nil } -func (n nullWriter) Close(ctx context.Context) error { return nil } func (n nullWriter) WriteSqlRow(ctx context.Context, r sql.Row) error { return nil } +func (n nullWriter) Close(ctx context.Context) error { return nil } func printEmptySetResult(start time.Time) { seconds := secondsSince(start) diff --git a/go/libraries/doltcore/merge/merge_rows.go b/go/libraries/doltcore/merge/merge_rows.go index dbda9aef5d..84591706c1 100644 --- a/go/libraries/doltcore/merge/merge_rows.go +++ b/go/libraries/doltcore/merge/merge_rows.go @@ -379,32 +379,6 @@ func setConflicts(ctx context.Context, cons durable.ConflictIndex, tbl, mergeTbl return tableToUpdate, nil } -func getTableInfoFromRoot(ctx context.Context, tblName string, root *doltdb.RootValue) ( - ok bool, - table *doltdb.Table, - sch schema.Schema, - h hash.Hash, - err error, -) { - table, ok, err = root.GetTable(ctx, tblName) - if err != nil { - return false, nil, nil, hash.Hash{}, err - } - - if ok { - h, err = table.HashOf() - if err != nil { - return false, nil, nil, hash.Hash{}, err - } - sch, err = table.GetSchema(ctx) - if err != nil { - return false, nil, nil, hash.Hash{}, err - } - } - - return ok, table, sch, h, nil -} - func calcTableMergeStats(ctx context.Context, tbl *doltdb.Table, mergeTbl *doltdb.Table) (MergeStats, error) { ms := MergeStats{Operation: TableModified} diff --git a/go/libraries/doltcore/merge/violations_fk_prolly.go b/go/libraries/doltcore/merge/violations_fk_prolly.go index 141af3da30..2dc29aea68 100644 --- a/go/libraries/doltcore/merge/violations_fk_prolly.go +++ b/go/libraries/doltcore/merge/violations_fk_prolly.go @@ -289,10 +289,6 @@ func createCVsForPartialKeyMatches( return createdViolation, nil } -func makePartialDescriptor(desc val.TupleDesc, n int) val.TupleDesc { - return val.NewTupleDescriptor(desc.Types[:n]...) -} - func makePartialKey(kb *val.TupleBuilder, idxSch schema.Index, tblSch schema.Schema, k, v val.Tuple, pool pool.BuffPool) (val.Tuple, bool) { for i, tag := range idxSch.IndexedColumnTags() { if j, ok := tblSch.GetPKCols().TagToIdx[tag]; ok { diff --git a/go/libraries/doltcore/table/composite_table_reader.go b/go/libraries/doltcore/table/composite_table_reader.go deleted file mode 100644 index 4345e348ac..0000000000 --- a/go/libraries/doltcore/table/composite_table_reader.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package table - -import ( - "context" - "io" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" -) - -// CompositeTableReader is a TableReader implementation which will concatenate the results -// of multiple TableReader instances into a single set of results. -type CompositeTableReader struct { - sch schema.Schema - readers []ReadCloser - idx int -} - -// NewCompositeTableReader creates a new CompositeTableReader instance from a slice of TableReadClosers. -func NewCompositeTableReader(readers []ReadCloser) (*CompositeTableReader, error) { - if len(readers) == 0 { - panic("nothing to iterate") - } - - sch := readers[0].GetSchema() - for i := 1; i < len(readers); i++ { - otherSch := readers[i].GetSchema() - if !schema.SchemasAreEqual(sch, otherSch) { - panic("readers must have the same schema") - } - } - - return &CompositeTableReader{sch: sch, readers: readers, idx: 0}, nil -} - -// GetSchema gets the schema of the rows that this reader will return -func (rd *CompositeTableReader) GetSchema() schema.Schema { - return rd.sch -} - -// ReadRow reads a row from a table. If there is a bad row the returned error will be non nil, and calling -// IsBadRow(err) will be return true. This is a potentially non-fatal error and callers can decide if they want to -// continue on a bad row, or fail. -func (rd *CompositeTableReader) ReadRow(ctx context.Context) (row.Row, error) { - for rd.idx < len(rd.readers) { - r, err := rd.readers[rd.idx].ReadRow(ctx) - - if err == io.EOF { - rd.idx++ - continue - } else if err != nil { - return nil, err - } - - return r, nil - } - - return nil, io.EOF -} - -// VerifySchema checks that the incoming schema matches the schema from the existing table -func (rd *CompositeTableReader) VerifySchema(outSch schema.Schema) (bool, error) { - return schema.VerifyInSchema(rd.sch, outSch) -} - -// Close should release resources being held -func (rd *CompositeTableReader) Close(ctx context.Context) error { - var firstErr error - for _, rdr := range rd.readers { - err := rdr.Close(ctx) - - if firstErr == nil { - firstErr = err - } - } - - return firstErr -} diff --git a/go/libraries/doltcore/table/composite_table_reader_test.go b/go/libraries/doltcore/table/composite_table_reader_test.go deleted file mode 100644 index cae3643c83..0000000000 --- a/go/libraries/doltcore/table/composite_table_reader_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package table - -import ( - "context" - "io" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/store/types" -) - -func TestCompositeTableReader(t *testing.T) { - const ( - numReaders = 7 - numItemsPerReader = 7 - ) - - ctx := context.Background() - - coll := schema.NewColCollection( - schema.NewColumn("id", 0, types.UintKind, true, schema.NotNullConstraint{}), - schema.NewColumn("val", 1, types.IntKind, false), - ) - sch, err := schema.SchemaFromCols(coll) - require.NoError(t, err) - - var readers []ReadCloser - var expectedKeys []uint64 - var expectedVals []int64 - for i := 0; i < numReaders; i++ { - var rows []row.Row - for j := 0; j < numItemsPerReader; j++ { - idx := j + (i * numItemsPerReader) - expectedKeys = append(expectedKeys, uint64(idx)) - expectedVals = append(expectedVals, int64(idx)) - r, err := row.New(types.Format_Default, sch, row.TaggedValues{ - 0: types.Uint(uint64(idx)), - 1: types.Int(idx), - }) - require.NoError(t, err) - rows = append(rows, r) - } - imt := NewInMemTableWithData(sch, rows) - rd := NewInMemTableReader(imt) - readers = append(readers, rd) - } - - compositeRd, err := NewCompositeTableReader(readers) - require.NoError(t, err) - - var keys []uint64 - var vals []int64 - for { - r, err := compositeRd.ReadRow(ctx) - - if err == io.EOF { - break - } - - assert.NoError(t, err) - val0, ok := r.GetColVal(0) - assert.True(t, ok) - val1, ok := r.GetColVal(1) - assert.True(t, ok) - - keys = append(keys, uint64(val0.(types.Uint))) - vals = append(vals, int64(val1.(types.Int))) - } - - assert.Equal(t, expectedKeys, keys) - assert.Equal(t, expectedVals, vals) - - err = compositeRd.Close(ctx) - assert.NoError(t, err) -} diff --git a/go/libraries/doltcore/table/editor/keyless_table_editor.go b/go/libraries/doltcore/table/editor/keyless_table_editor.go index 6fb8480d20..198b4b02f7 100644 --- a/go/libraries/doltcore/table/editor/keyless_table_editor.go +++ b/go/libraries/doltcore/table/editor/keyless_table_editor.go @@ -25,7 +25,6 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/table" "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" "github.com/dolthub/dolt/go/store/hash" "github.com/dolthub/dolt/go/store/types" @@ -437,14 +436,14 @@ func (kte *keylessTableEditor) flush(ctx context.Context) error { } kte.eg.Go(func() (err error) { - kte.tbl, err = applyEdits(ctx, tbl, acc, kte.indexEds, nil) + kte.tbl, err = applyEdits(ctx, tbl, acc, kte.indexEds) return err }) return nil } -func applyEdits(ctx context.Context, tbl *doltdb.Table, acc keylessEditAcc, indexEds []*IndexEditor, errFunc PKDuplicateCb) (_ *doltdb.Table, retErr error) { +func applyEdits(ctx context.Context, tbl *doltdb.Table, acc keylessEditAcc, indexEds []*IndexEditor) (_ *doltdb.Table, retErr error) { rowData, err := tbl.GetNomsRowData(ctx) if err != nil { return nil, err @@ -463,7 +462,7 @@ func applyEdits(ctx context.Context, tbl *doltdb.Table, acc keylessEditAcc, inde } ed := rowData.Edit() - iter := table.NewMapPointReader(rowData, keys...) + iter := newMapPointReader(rowData, keys) var ok bool for { @@ -568,6 +567,44 @@ func applyEdits(ctx context.Context, tbl *doltdb.Table, acc keylessEditAcc, inde return tbl.UpdateNomsRows(ctx, rowData) } +type pointReader struct { + m types.Map + keys []types.Tuple + emptyTuple types.Tuple + idx int +} + +func newMapPointReader(m types.Map, keys []types.Tuple) *pointReader { + return &pointReader{ + m: m, + keys: keys, + emptyTuple: types.EmptyTuple(m.Format()), + } +} + +// NextTuple implements types.MapIterator. +func (pr *pointReader) NextTuple(ctx context.Context) (k, v types.Tuple, err error) { + if pr.idx >= len(pr.keys) { + return types.Tuple{}, types.Tuple{}, io.EOF + } + + k = pr.keys[pr.idx] + v = pr.emptyTuple + + var ok bool + // todo: optimize by implementing MapIterator.Seek() + v, ok, err = pr.m.MaybeGetTuple(ctx, k) + pr.idx++ + + if err != nil { + return types.Tuple{}, types.Tuple{}, err + } else if !ok { + return k, pr.emptyTuple, nil + } + + return k, v, nil +} + // for deletes (cardinality < 1): |ok| is set false func initializeCardinality(val types.Tuple, card int64) (v types.Tuple, ok bool, err error) { if card < 1 { diff --git a/go/libraries/doltcore/table/inmem_table.go b/go/libraries/doltcore/table/inmem_table.go index 65382b29d6..3bb4665e71 100644 --- a/go/libraries/doltcore/table/inmem_table.go +++ b/go/libraries/doltcore/table/inmem_table.go @@ -162,28 +162,3 @@ func (rd *InMemTableReader) GetSchema() schema.Schema { func (rd *InMemTableReader) VerifySchema(outSch schema.Schema) (bool, error) { return schema.VerifyInSchema(rd.tt.sch, outSch) } - -// InMemTableWriter is an implementation of a RowWriter for an InMemTable -type InMemTableWriter struct { - tt *InMemTable -} - -// NewInMemTableWriter creates an instance of a RowWriter from an InMemTable -func NewInMemTableWriter(imt *InMemTable) *InMemTableWriter { - return &InMemTableWriter{imt} -} - -// WriteRow will write a row to a table -func (w *InMemTableWriter) WriteRow(ctx context.Context, r row.Row) error { - return w.tt.AppendRow(r) -} - -// Close should flush all writes, release resources being held -func (w *InMemTableWriter) Close(ctx context.Context) error { - return nil -} - -// GetSchema gets the schema of the rows that this writer writes -func (w *InMemTableWriter) GetSchema() schema.Schema { - return w.tt.sch -} diff --git a/go/libraries/doltcore/table/inmem_table_test.go b/go/libraries/doltcore/table/inmem_table_test.go index 7903adc903..57fc9a7134 100644 --- a/go/libraries/doltcore/table/inmem_table_test.go +++ b/go/libraries/doltcore/table/inmem_table_test.go @@ -76,12 +76,8 @@ func TestInMemTable(t *testing.T) { imt := NewInMemTable(rowSch) func() { - var wr TableWriteCloser - wr = NewInMemTableWriter(imt) - defer wr.Close(context.Background()) - - for _, row := range rows { - err := wr.WriteRow(context.Background(), row) + for _, r := range rows { + err := imt.AppendRow(r) if err != nil { t.Fatal("Failed to write row") @@ -112,41 +108,6 @@ func TestInMemTable(t *testing.T) { }() } -func TestPipeRows(t *testing.T) { - imt := NewInMemTableWithData(rowSch, rows) - imtt2 := NewInMemTable(rowSch) - - var err error - func() { - rd := NewInMemTableReader(imt) - defer rd.Close(context.Background()) - wr := NewInMemTableWriter(imtt2) - defer wr.Close(context.Background()) - _, _, err = PipeRows(context.Background(), rd, wr, false) - }() - - if err != nil { - t.Error("Error piping rows from reader to writer", err) - } - - if imt.NumRows() != imtt2.NumRows() { - t.Error("Row counts should match") - } - - for i := 0; i < imt.NumRows(); i++ { - r1, err1 := imt.GetRow(i) - r2, err2 := imtt2.GetRow(i) - - if err1 != nil || err2 != nil { - t.Fatal("Couldn't Get row.") - } - - if !row.AreEqual(r1, r2, rowSch) { - t.Error("Rows should be the same.", row.Fmt(context.Background(), r1, rowSch), "!=", row.Fmt(context.Background(), r2, rowSch)) - } - } -} - func TestReadAllRows(t *testing.T) { imt := NewInMemTableWithData(rowSch, rows) diff --git a/go/libraries/doltcore/table/io.go b/go/libraries/doltcore/table/io.go index d3543987b7..a60f6c3acc 100644 --- a/go/libraries/doltcore/table/io.go +++ b/go/libraries/doltcore/table/io.go @@ -29,10 +29,10 @@ import ( // // Returns a tuple: (number of rows written, number of errors ignored, error). In the case that err is non-nil, the // row counter fields in the tuple will be set to -1. -func PipeRows(ctx context.Context, rd Reader, wr RowWriter, contOnBadRow bool) (int, int, error) { +func PipeRows(ctx context.Context, rd SqlRowReader, wr SqlRowWriter, contOnBadRow bool) (int, int, error) { var numBad, numGood int for { - r, err := rd.ReadRow(ctx) + r, err := rd.ReadSqlRow(ctx) if err != nil && err != io.EOF { if IsBadRow(err) && contOnBadRow { @@ -48,7 +48,7 @@ func PipeRows(ctx context.Context, rd Reader, wr RowWriter, contOnBadRow bool) ( return -1, -1, errors.New("reader returned nil row with err==nil") } - err = wr.WriteRow(ctx, r) + err = wr.WriteSqlRow(ctx, r) if err != nil { return -1, -1, err diff --git a/go/libraries/doltcore/table/keyless_reader.go b/go/libraries/doltcore/table/keyless_reader.go deleted file mode 100644 index 0f94be3094..0000000000 --- a/go/libraries/doltcore/table/keyless_reader.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package table - -import ( - "context" - "io" - - "github.com/dolthub/go-mysql-server/sql" - - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" - "github.com/dolthub/dolt/go/store/types" -) - -type keylessTableReader struct { - iter types.MapIterator - sch schema.Schema - - row row.Row - remainingCopies uint64 - bounded bool - remainingEntries uint64 -} - -var _ SqlTableReader = &keylessTableReader{} -var _ ReadCloser = &keylessTableReader{} - -// GetSchema implements the TableReader interface. -func (rdr *keylessTableReader) GetSchema() schema.Schema { - return rdr.sch -} - -// ReadSqlRow implements the SqlTableReader interface. -func (rdr *keylessTableReader) ReadRow(ctx context.Context) (row.Row, error) { - if rdr.remainingCopies <= 0 { - if rdr.bounded && rdr.remainingEntries == 0 { - return nil, io.EOF - } - - key, val, err := rdr.iter.Next(ctx) - if err != nil { - return nil, err - } else if key == nil { - return nil, io.EOF - } - - rdr.row, rdr.remainingCopies, err = row.KeylessRowsFromTuples(key.(types.Tuple), val.(types.Tuple)) - if err != nil { - return nil, err - } - - if rdr.remainingEntries > 0 { - rdr.remainingEntries -= 1 - } - - if rdr.remainingCopies == 0 { - return nil, row.ErrZeroCardinality - } - } - - rdr.remainingCopies -= 1 - - return rdr.row, nil -} - -// ReadSqlRow implements the SqlTableReader interface. -func (rdr *keylessTableReader) ReadSqlRow(ctx context.Context) (sql.Row, error) { - r, err := rdr.ReadRow(ctx) - if err != nil { - return nil, err - } - - return sqlutil.DoltRowToSqlRow(r, rdr.sch) -} - -// Close implements the TableReadCloser interface. -func (rdr *keylessTableReader) Close(_ context.Context) error { - return nil -} - -func newKeylessTableReader(ctx context.Context, tbl *doltdb.Table, sch schema.Schema, buffered bool) (*keylessTableReader, error) { - rows, err := tbl.GetNomsRowData(ctx) - if err != nil { - return nil, err - } - - return newKeylessTableReaderForRows(ctx, rows, sch, buffered) -} - -func newKeylessTableReaderForRows(ctx context.Context, rows types.Map, sch schema.Schema, buffered bool) (*keylessTableReader, error) { - var err error - var iter types.MapIterator - if buffered { - iter, err = rows.Iterator(ctx) - } else { - iter, err = rows.BufferedIterator(ctx) - } - if err != nil { - return nil, err - } - - return &keylessTableReader{ - iter: iter, - sch: sch, - }, nil -} - -func newKeylessTableReaderForPartition(ctx context.Context, tbl *doltdb.Table, sch schema.Schema, start, end uint64) (SqlTableReader, error) { - rows, err := tbl.GetNomsRowData(ctx) - if err != nil { - return nil, err - } - - iter, err := rows.BufferedIteratorAt(ctx, start) - if err != nil { - return nil, err - } - - return &keylessTableReader{ - iter: iter, - sch: sch, - remainingEntries: end - start, - bounded: true, - }, nil -} - -func newKeylessTableReaderFrom(ctx context.Context, tbl *doltdb.Table, sch schema.Schema, val types.Value) (SqlTableReader, error) { - rows, err := tbl.GetNomsRowData(ctx) - if err != nil { - return nil, err - } - - iter, err := rows.IteratorFrom(ctx, val) - if err != nil { - return nil, err - } - - return &keylessTableReader{ - iter: iter, - sch: sch, - }, nil -} diff --git a/go/libraries/doltcore/table/map_point_reader.go b/go/libraries/doltcore/table/map_point_reader.go deleted file mode 100644 index 240d36348d..0000000000 --- a/go/libraries/doltcore/table/map_point_reader.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package table - -import ( - "context" - "io" - - "github.com/dolthub/dolt/go/store/types" -) - -type PointReader struct { - m types.Map - emptyTuple types.Tuple - keys []types.Tuple - idx int -} - -var _ types.MapIterator = &PointReader{} - -// read the map values for a set of map keys -func NewMapPointReader(m types.Map, keys ...types.Tuple) types.MapIterator { - return &PointReader{ - m: m, - emptyTuple: types.EmptyTuple(m.Format()), - keys: keys, - } -} - -// Next implements types.MapIterator. -func (pr *PointReader) Next(ctx context.Context) (k, v types.Value, err error) { - kt, vt, err := pr.NextTuple(ctx) - - if err != nil { - return nil, nil, err - } - - if !kt.Empty() { - k = kt - } - - if !vt.Empty() { - v = vt - } - - return k, v, nil -} - -// NextTuple implements types.MapIterator. -func (pr *PointReader) NextTuple(ctx context.Context) (k, v types.Tuple, err error) { - if pr.idx >= len(pr.keys) { - return types.Tuple{}, types.Tuple{}, io.EOF - } - - k = pr.keys[pr.idx] - v = pr.emptyTuple - - var ok bool - // todo: optimize by implementing MapIterator.Seek() - v, ok, err = pr.m.MaybeGetTuple(ctx, k) - pr.idx++ - - if err != nil { - return types.Tuple{}, types.Tuple{}, err - } else if !ok { - return k, pr.emptyTuple, nil - } - - return k, v, nil -} diff --git a/go/libraries/doltcore/table/pk_reader.go b/go/libraries/doltcore/table/pk_reader.go deleted file mode 100644 index 9bdbd1eaf6..0000000000 --- a/go/libraries/doltcore/table/pk_reader.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package table - -import ( - "context" - "io" - - "github.com/dolthub/go-mysql-server/sql" - - "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms" - "github.com/dolthub/dolt/go/store/types" -) - -type pkTableReader struct { - iter types.MapIterator - sch schema.Schema -} - -var _ SqlTableReader = pkTableReader{} -var _ ReadCloser = pkTableReader{} - -// GetSchema implements the TableReader interface. -func (rdr pkTableReader) GetSchema() schema.Schema { - return rdr.sch -} - -// ReadRow implements the TableReader interface. -func (rdr pkTableReader) ReadRow(ctx context.Context) (row.Row, error) { - key, val, err := rdr.iter.Next(ctx) - - if err != nil { - return nil, err - } else if key == nil { - return nil, io.EOF - } - - return row.FromNoms(rdr.sch, key.(types.Tuple), val.(types.Tuple)) -} - -// ReadSqlRow implements the SqlTableReader interface. -func (rdr pkTableReader) ReadSqlRow(ctx context.Context) (sql.Row, error) { - key, val, err := rdr.iter.Next(ctx) - - if err != nil { - return nil, err - } else if key == nil { - return nil, io.EOF - } - - return noms.SqlRowFromTuples(rdr.sch, key.(types.Tuple), val.(types.Tuple)) -} - -// Close implements the TableReadCloser interface. -func (rdr pkTableReader) Close(_ context.Context) error { - return nil -} - -func newPkTableReader(ctx context.Context, tbl *doltdb.Table, sch schema.Schema, buffered bool) (pkTableReader, error) { - rows, err := tbl.GetNomsRowData(ctx) - if err != nil { - return pkTableReader{}, err - } - - return newPkTableReaderForRows(ctx, rows, sch, buffered) -} - -func newPkTableReaderForRows(ctx context.Context, rows types.Map, sch schema.Schema, buffered bool) (pkTableReader, error) { - var err error - var iter types.MapIterator - if buffered { - iter, err = rows.Iterator(ctx) - } else { - iter, err = rows.BufferedIterator(ctx) - } - if err != nil { - return pkTableReader{}, err - } - - return pkTableReader{ - iter: iter, - sch: sch, - }, nil -} - -func newPkTableReaderFrom(ctx context.Context, tbl *doltdb.Table, sch schema.Schema, val types.Value) (SqlTableReader, error) { - rows, err := tbl.GetNomsRowData(ctx) - if err != nil { - return nil, err - } - - iter, err := rows.IteratorFrom(ctx, val) - if err != nil { - return nil, err - } - - return pkTableReader{ - iter: iter, - sch: sch, - }, nil -} - -type partitionTableReader struct { - SqlTableReader - remaining uint64 -} - -var _ SqlTableReader = &partitionTableReader{} - -func newPkTableReaderForPartition(ctx context.Context, tbl *doltdb.Table, sch schema.Schema, start, end uint64) (SqlTableReader, error) { - rows, err := tbl.GetNomsRowData(ctx) - if err != nil { - return nil, err - } - - iter, err := rows.BufferedIteratorAt(ctx, start) - if err != nil { - return nil, err - } - - return &partitionTableReader{ - SqlTableReader: pkTableReader{ - iter: iter, - sch: sch, - }, - remaining: end - start, - }, nil -} - -// ReadSqlRow implements the SqlTableReader interface. -func (rdr *partitionTableReader) ReadSqlRow(ctx context.Context) (sql.Row, error) { - if rdr.remaining == 0 { - return nil, io.EOF - } - rdr.remaining -= 1 - - return rdr.SqlTableReader.ReadSqlRow(ctx) -} diff --git a/go/libraries/doltcore/table/read_ahead_table_reader.go b/go/libraries/doltcore/table/read_ahead_table_reader.go deleted file mode 100644 index 125e5a94ed..0000000000 --- a/go/libraries/doltcore/table/read_ahead_table_reader.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2020 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package table - -import ( - "context" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" - "github.com/dolthub/dolt/go/libraries/doltcore/schema" - "github.com/dolthub/dolt/go/libraries/utils/async" -) - -var _ ReadCloser = (*AsyncReadAheadTableReader)(nil) - -// AsyncReadAheadTableReader is a TableReadCloser implementation that spins up a go routine to keep reading data into -// a buffered channel so that it is ready when the caller wants it. -type AsyncReadAheadTableReader struct { - backingReader ReadCloser - reader *async.AsyncReader -} - -// NewAsyncReadAheadTableReader creates a new AsyncReadAheadTableReader -func NewAsyncReadAheadTableReader(tr ReadCloser, bufferSize int) *AsyncReadAheadTableReader { - read := func(ctx context.Context) (interface{}, error) { - return tr.ReadRow(ctx) - } - - reader := async.NewAsyncReader(read, bufferSize) - return &AsyncReadAheadTableReader{tr, reader} -} - -// Start the worker routine reading rows to the channel -func (tr *AsyncReadAheadTableReader) Start(ctx context.Context) error { - return tr.reader.Start(ctx) -} - -// GetSchema gets the schema of the rows that this reader will return -func (tr *AsyncReadAheadTableReader) GetSchema() schema.Schema { - return tr.backingReader.GetSchema() -} - -// ReadRow reads a row from a table. If there is a bad row the returned error will be non nil, and calling -// IsBadRow(err) will be return true. This is a potentially non-fatal error and callers can decide if they want to -// continue on a bad row, or fail. -func (tr *AsyncReadAheadTableReader) ReadRow(ctx context.Context) (row.Row, error) { - obj, err := tr.reader.Read() - - if err != nil { - return nil, err - } - - return obj.(row.Row), err -} - -// Close releases resources being held -func (tr *AsyncReadAheadTableReader) Close(ctx context.Context) error { - _ = tr.reader.Close() - return tr.backingReader.Close(ctx) -} diff --git a/go/libraries/doltcore/table/table_writer.go b/go/libraries/doltcore/table/table_writer.go index 9bdb049cfd..18cb349887 100644 --- a/go/libraries/doltcore/table/table_writer.go +++ b/go/libraries/doltcore/table/table_writer.go @@ -18,23 +18,9 @@ import ( "context" "github.com/dolthub/go-mysql-server/sql" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" ) -// RowWriter knows how to write table rows to some destination -type RowWriter interface { - // WriteRow writes a row to the destination of this writer - WriteRow(ctx context.Context, r row.Row) error -} - -// TableWriteCloser is an interface for writing rows to a table, that can be closed -type TableWriteCloser interface { - RowWriter - Closer -} - type SqlRowWriter interface { - TableWriteCloser + Closer WriteSqlRow(ctx context.Context, r sql.Row) error } diff --git a/go/libraries/doltcore/table/typed/json/writer.go b/go/libraries/doltcore/table/typed/json/writer.go index 76f705d122..bbe2dfb566 100644 --- a/go/libraries/doltcore/table/typed/json/writer.go +++ b/go/libraries/doltcore/table/typed/json/writer.go @@ -24,12 +24,10 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/dolthub/vitess/go/sqltypes" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/libraries/doltcore/table" "github.com/dolthub/dolt/go/libraries/utils/iohelp" - "github.com/dolthub/dolt/go/store/types" ) const jsonHeader = `{"rows": [` @@ -72,77 +70,6 @@ func (j *RowWriter) GetSchema() schema.Schema { return j.sch } -// WriteRow encodes the row given into JSON format and writes it, returning any error -func (j *RowWriter) WriteRow(ctx context.Context, r row.Row) error { - if j.rowsWritten == 0 { - err := iohelp.WriteAll(j.bWr, []byte(j.header)) - if err != nil { - return err - } - } - - allCols := j.sch.GetAllCols() - colValMap := make(map[string]interface{}, allCols.Size()) - if err := allCols.Iter(func(tag uint64, col schema.Column) (stop bool, err error) { - val, ok := r.GetColVal(tag) - if !ok || types.IsNull(val) { - return false, nil - } - - switch col.TypeInfo.GetTypeIdentifier() { - case typeinfo.DatetimeTypeIdentifier, - typeinfo.DecimalTypeIdentifier, - typeinfo.EnumTypeIdentifier, - typeinfo.InlineBlobTypeIdentifier, - typeinfo.SetTypeIdentifier, - typeinfo.TimeTypeIdentifier, - typeinfo.TupleTypeIdentifier, - typeinfo.UuidTypeIdentifier, - typeinfo.VarBinaryTypeIdentifier, - typeinfo.YearTypeIdentifier: - v, err := col.TypeInfo.FormatValue(val) - if err != nil { - return true, err - } - val = types.String(*v) - - case typeinfo.BitTypeIdentifier, - typeinfo.BoolTypeIdentifier, - typeinfo.VarStringTypeIdentifier, - typeinfo.UintTypeIdentifier, - typeinfo.IntTypeIdentifier, - typeinfo.FloatTypeIdentifier: - // use primitive type - } - - colValMap[col.Name] = val - - return false, nil - }); err != nil { - return err - } - - data, err := marshalToJson(colValMap) - if err != nil { - return errors.New("marshaling did not work") - } - - if j.rowsWritten != 0 { - _, err := j.bWr.WriteString(j.separator) - if err != nil { - return err - } - } - - newErr := iohelp.WriteAll(j.bWr, data) - if newErr != nil { - return newErr - } - j.rowsWritten++ - - return nil -} - func (j *RowWriter) WriteSqlRow(ctx context.Context, row sql.Row) error { // The Type.SQL() call takes in a SQL context to determine the output character set for types that use a collation. // The context given is not a SQL context, so we force the `utf8mb4` character set to be used, as it is the most diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go index c2393c80cc..5945c8063e 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer.go +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -24,7 +24,6 @@ import ( "github.com/xitongsys/parquet-go/source" "github.com/xitongsys/parquet-go/writer" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" @@ -96,15 +95,6 @@ func (pwr *ParquetWriter) GetSchema() schema.Schema { return pwr.sch } -// WriteRow will write a row to a table -func (pwr *ParquetWriter) WriteRow(ctx context.Context, r row.Row) error { - sqlRow, err := sqlutil.DoltRowToSqlRow(r, pwr.GetSchema()) - if err != nil { - return err - } - return pwr.WriteSqlRow(ctx, sqlRow) -} - func (pwr *ParquetWriter) WriteSqlRow(ctx context.Context, r sql.Row) error { colValStrs := make([]*string, pwr.sch.GetAllCols().Size()) diff --git a/go/libraries/doltcore/table/typed/parquet/writer_test.go b/go/libraries/doltcore/table/typed/parquet/writer_test.go index 91f5d1560f..8ec6abb426 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer_test.go +++ b/go/libraries/doltcore/table/typed/parquet/writer_test.go @@ -21,11 +21,11 @@ import ( "os" "testing" + "github.com/dolthub/go-mysql-server/sql" "github.com/stretchr/testify/assert" "github.com/xitongsys/parquet-go-source/local" "github.com/xitongsys/parquet-go/reader" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/store/types" @@ -54,41 +54,21 @@ type Person struct { Title string `parquet:"name=title, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"` } -func mustRow(r row.Row, err error) row.Row { - if err != nil { - panic(err) - } - - return r -} - -func getSampleRows() (rows []row.Row) { - return []row.Row{ - mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ - nameColTag: types.String("Bill Billerson"), - ageColTag: types.Uint(32), - titleColTag: types.String("Senior Dufus")})), - mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ - nameColTag: types.String("Rob Robertson"), - ageColTag: types.Uint(25), - titleColTag: types.String("Dufus")})), - mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ - nameColTag: types.String("John Johnson"), - ageColTag: types.Uint(21), - titleColTag: types.String("")})), - mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ - nameColTag: types.String("Andy Anderson"), - ageColTag: types.Uint(27), - /* title = NULL */})), +func getSampleRows() []sql.Row { + return []sql.Row{ + {"Bill Billerson", 32, "Senior Dufus"}, + {"Rob Robertson", 25, "Dufus"}, + {"John Johnson", 21, ""}, + {"Andy Anderson", 27, nil}, } } -func writeToParquet(pWr *ParquetWriter, rows []row.Row, t *testing.T) { +func writeToParquet(pWr *ParquetWriter, rows []sql.Row, t *testing.T) { func() { defer pWr.Close(context.Background()) for _, row := range rows { - err := pWr.WriteRow(context.Background(), row) + err := pWr.WriteSqlRow(context.Background(), row) if err != nil { t.Fatal("Failed to write row") } diff --git a/go/libraries/doltcore/table/untyped/csv/line.go b/go/libraries/doltcore/table/untyped/csv/line.go index ba5dfee70e..869ad61b57 100644 --- a/go/libraries/doltcore/table/untyped/csv/line.go +++ b/go/libraries/doltcore/table/untyped/csv/line.go @@ -20,10 +20,6 @@ import ( "strings" ) -func csvSplitLineRuneDelim(str string, delim rune, escapedQuotes bool) ([]*string, error) { - return csvSplitLine(str, string(delim), escapedQuotes) -} - func csvSplitLine(str string, delim string, escapedQuotes bool) ([]*string, error) { if strings.IndexRune(delim, '"') != -1 { panic("delims cannot contain quotes") diff --git a/go/libraries/doltcore/table/untyped/csv/writer.go b/go/libraries/doltcore/table/untyped/csv/writer.go index 1861bdb487..7f0493cda2 100644 --- a/go/libraries/doltcore/table/untyped/csv/writer.go +++ b/go/libraries/doltcore/table/untyped/csv/writer.go @@ -26,7 +26,6 @@ import ( "github.com/dolthub/go-mysql-server/sql" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/libraries/doltcore/table" @@ -81,31 +80,6 @@ func NewCSVWriter(wr io.WriteCloser, outSch schema.Schema, info *CSVFileInfo) (* return csvw, nil } -// WriteRow will write a row to a table -func (csvw *CSVWriter) WriteRow(ctx context.Context, r row.Row) error { - allCols := csvw.sch.GetAllCols() - colValStrs := make([]*string, allCols.Size()) - - sqlRow, err := sqlutil.DoltRowToSqlRow(r, csvw.sch) - if err != nil { - return err - } - - for i, val := range sqlRow { - if val == nil { - colValStrs[i] = nil - } else { - v, err := sqlutil.SqlColToStr(csvw.sch.GetAllCols().GetByIndex(i).TypeInfo.ToSqlType(), val) - if err != nil { - return err - } - colValStrs[i] = &v - } - } - - return csvw.write(colValStrs) -} - func (csvw *CSVWriter) WriteSqlRow(ctx context.Context, r sql.Row) error { colValStrs := make([]*string, csvw.sch.GetAllCols().Size()) for i, val := range r { diff --git a/go/libraries/doltcore/table/untyped/csv/writer_test.go b/go/libraries/doltcore/table/untyped/csv/writer_test.go index 4461a3ce10..747bc08d37 100644 --- a/go/libraries/doltcore/table/untyped/csv/writer_test.go +++ b/go/libraries/doltcore/table/untyped/csv/writer_test.go @@ -19,7 +19,8 @@ import ( "os" "testing" - "github.com/dolthub/dolt/go/libraries/doltcore/row" + "github.com/dolthub/go-mysql-server/sql" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/libraries/utils/filesys" @@ -43,33 +44,21 @@ var inCols = []schema.Column{ var colColl = schema.NewColCollection(inCols...) var rowSch = schema.MustSchemaFromCols(colColl) -func getSampleRows() (rows []row.Row) { - return []row.Row{ - mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ - nameColTag: types.String("Bill Billerson"), - ageColTag: types.Uint(32), - titleColTag: types.String("Senior Dufus")})), - mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ - nameColTag: types.String("Rob Robertson"), - ageColTag: types.Uint(25), - titleColTag: types.String("Dufus")})), - mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ - nameColTag: types.String("John Johnson"), - ageColTag: types.Uint(21), - titleColTag: types.String("")})), - mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ - nameColTag: types.String("Andy Anderson"), - ageColTag: types.Uint(27), - /* title = NULL */})), +func getSampleRows() []sql.Row { + return []sql.Row{ + {"Bill Billerson", 32, "Senior Dufus"}, + {"Rob Robertson", 25, "Dufus"}, + {"John Johnson", 21, ""}, + {"Andy Anderson", 27, nil}, } } -func writeToCSV(csvWr *CSVWriter, rows []row.Row, t *testing.T) { +func writeToCSV(csvWr *CSVWriter, rows []sql.Row, t *testing.T) { func() { defer csvWr.Close(context.Background()) - for _, row := range rows { - err := csvWr.WriteRow(context.Background(), row) + for _, r := range rows { + err := csvWr.WriteSqlRow(context.Background(), r) if err != nil { t.Fatal("Failed to write row") diff --git a/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go b/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go index 94d8e40fd2..94cde1394b 100644 --- a/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go +++ b/go/libraries/doltcore/table/untyped/sqlexport/batch_sqlwriter.go @@ -23,7 +23,6 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/errhand" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlfmt" @@ -81,64 +80,6 @@ func (w *BatchSqlExportWriter) GetSchema() schema.Schema { return w.sch } -// WriteRow will write a row to a table -func (w *BatchSqlExportWriter) WriteRow(ctx context.Context, r row.Row) error { - if err := w.maybeWriteDropCreate(ctx); err != nil { - return err - } - - // Previous write was last insert - if w.numInserts > 0 && r == nil { - return iohelp.WriteLine(w.wr, ";") - } - - // Reached max number of inserts on one line - if w.numInserts == batchSize { - // Reset count - w.numInserts = 0 - - // End line - err := iohelp.WriteLine(w.wr, ";") - if err != nil { - return err - } - } - - // Append insert values as tuples - var stmt string - if w.numInserts == 0 { - // Get insert prefix string - prefix, err := sqlfmt.InsertStatementPrefix(w.tableName, w.sch) - if err != nil { - return nil - } - // Write prefix - err = iohelp.WriteWithoutNewLine(w.wr, prefix) - if err != nil { - return nil - } - } else { - stmt = ", " - } - - // Get insert tuple string - tuple, err := sqlfmt.RowAsTupleString(r, w.sch) - if err != nil { - return err - } - - // Write insert tuple - err = iohelp.WriteWithoutNewLine(w.wr, stmt+tuple) - if err != nil { - return nil - } - - // Increase count of inserts written on this line - w.numInserts++ - - return err -} - func (w *BatchSqlExportWriter) WriteSqlRow(ctx context.Context, r sql.Row) error { if err := w.maybeWriteDropCreate(ctx); err != nil { return err diff --git a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go index f5f8f60626..6d4d38d24e 100644 --- a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go +++ b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go @@ -23,7 +23,6 @@ import ( "github.com/dolthub/dolt/go/cmd/dolt/errhand" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlfmt" @@ -77,21 +76,6 @@ func (w *SqlExportWriter) GetSchema() schema.Schema { return w.sch } -// WriteRow will write a row to a table -func (w *SqlExportWriter) WriteRow(ctx context.Context, r row.Row) error { - if err := w.maybeWriteDropCreate(ctx); err != nil { - return err - } - - stmt, err := sqlfmt.RowAsInsertStmt(r, w.tableName, w.sch) - - if err != nil { - return err - } - - return iohelp.WriteLine(w.wr, stmt) -} - func (w *SqlExportWriter) WriteSqlRow(ctx context.Context, r sql.Row) error { if r == nil { return nil diff --git a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go index 3576eca8c7..35fa921646 100644 --- a/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go +++ b/go/libraries/doltcore/table/untyped/sqlexport/sqlwriter_test.go @@ -19,7 +19,7 @@ import ( "strings" "testing" - "github.com/google/uuid" + "github.com/dolthub/go-mysql-server/sql" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -40,9 +40,7 @@ func (*StringBuilderCloser) Close() error { } func TestEndToEnd(t *testing.T) { - id := uuid.MustParse("00000000-0000-0000-0000-000000000000") tableName := "people" - dropCreateStatement := sqlfmt.DropTableIfExistsStmt(tableName) + "\n" + "CREATE TABLE `people` (\n" + " `id` varchar(16383) NOT NULL,\n" + @@ -57,7 +55,7 @@ func TestEndToEnd(t *testing.T) { type test struct { name string - rows []row.Row + rows []sql.Row sch schema.Schema expectedOutput string } @@ -65,9 +63,10 @@ func TestEndToEnd(t *testing.T) { tests := []test{ { name: "two rows", - rows: rs( - dtestutils.NewTypedRow(id, "some guy", 100, false, strPointer("normie")), - dtestutils.NewTypedRow(id, "guy personson", 0, true, strPointer("officially a person"))), + rows: []sql.Row{ + {"00000000-0000-0000-0000-000000000000", "some guy", 100, 0, "normie"}, + {"00000000-0000-0000-0000-000000000000", "guy personson", 0, 1, "officially a person"}, + }, sch: dtestutils.TypedSchema, expectedOutput: dropCreateStatement + "\n" + "INSERT INTO `people` (`id`,`name`,`age`,`is_married`,`title`) " + @@ -112,7 +111,7 @@ func TestEndToEnd(t *testing.T) { } for _, r := range tt.rows { - assert.NoError(t, w.WriteRow(ctx, r)) + assert.NoError(t, w.WriteSqlRow(ctx, r)) } assert.NoError(t, w.Close(ctx)) diff --git a/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go b/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go index 00123dc516..d1e66d2516 100644 --- a/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go +++ b/go/libraries/doltcore/table/untyped/tabular/fixedwidth_tablewriter.go @@ -25,7 +25,6 @@ import ( "github.com/fatih/color" "github.com/dolthub/dolt/go/libraries/doltcore/diff" - "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" "github.com/dolthub/dolt/go/libraries/doltcore/table" "github.com/dolthub/dolt/go/libraries/utils/iohelp" @@ -168,10 +167,6 @@ func (w *FixedWidthTableWriter) WriteColoredRow(ctx context.Context, r sql.Row, return nil } -func (w *FixedWidthTableWriter) WriteRow(ctx context.Context, r row.Row) error { - panic("unimplemented") -} - func (w *FixedWidthTableWriter) sampleRow(r sql.Row, colors []*color.Color) (tableRow, error) { row := tableRow{ columns: make([]string, len(r)), diff --git a/go/store/types/edits/disk_backed.go b/go/store/types/edits/disk_backed.go index f8108beb5f..80f5babea4 100644 --- a/go/store/types/edits/disk_backed.go +++ b/go/store/types/edits/disk_backed.go @@ -16,7 +16,6 @@ package edits import ( "context" - "os" "github.com/dolthub/dolt/go/store/types" ) @@ -111,10 +110,3 @@ func (dbea *DiskBackedEditAcc) Close(ctx context.Context) error { return nil } - -// best effort deletion ignores errors -func tryDeleteFiles(paths []string) { - for _, path := range paths { - _ = os.Remove(path) - } -} From 6bd964babebd278a90301ae4fa1b7487651ccd53 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Mon, 17 Oct 2022 12:03:51 -0700 Subject: [PATCH 25/25] a bit more cleanup in pkg table --- .../doltcore/table/inmem_table_test.go | 76 --------------- go/libraries/doltcore/table/io.go | 94 ------------------- go/libraries/doltcore/table/table_reader.go | 40 ++++++++ .../doltcore/table/untyped/untyped_rows.go | 52 ---------- 4 files changed, 40 insertions(+), 222 deletions(-) delete mode 100644 go/libraries/doltcore/table/io.go diff --git a/go/libraries/doltcore/table/inmem_table_test.go b/go/libraries/doltcore/table/inmem_table_test.go index 57fc9a7134..6c573e1ca0 100644 --- a/go/libraries/doltcore/table/inmem_table_test.go +++ b/go/libraries/doltcore/table/inmem_table_test.go @@ -107,79 +107,3 @@ func TestInMemTable(t *testing.T) { } }() } - -func TestReadAllRows(t *testing.T) { - imt := NewInMemTableWithData(rowSch, rows) - - var err error - var numBad int - var results []row.Row - func() { - rd := NewInMemTableReader(imt) - defer rd.Close(context.Background()) - results, numBad, err = ReadAllRows(context.Background(), rd, true) - }() - - if err != nil { - t.Fatal("Error reading rows") - } - - if len(rows) != len(results) { - t.Error("Unexpected count.") - } - - if numBad != 0 { - t.Error("Unexpected BadRow Count") - } - - for i := 0; i < len(rows); i++ { - if !row.AreEqual(rows[i], results[i], rowSch) { - t.Error(row.Fmt(context.Background(), rows[i], rowSch), "!=", row.Fmt(context.Background(), results[i], rowSch)) - } - } -} - -/* -func TestReadAllRowsToMap(t *testing.T) { - imt := NewInMemTableWithData(rowSch, rows) - greatIndex := rowSch.GetFieldIndex("is_great") - - var err error - var numBad int - var results map[types.Value][]row.Row - func() { - rd := NewInMemTableReader(imt) - defer rd.Close() - results, numBad, err = ReadAllRowsToMap(rd, greatIndex, true) - }() - - if err != nil { - t.Fatal("Error reading rows") - } - - if numBad != 0 { - t.Error("Unexpected BadRow Count") - } - - if len(results) != 2 { - t.Error("Unexpected count.") - } - - if len(results[types.Bool(true)]) != 2 || len(results[types.Bool(false)]) != 1 { - t.Error("Unexpected count for one or more values of is_great") - } - - for _, great := range []types.Bool{types.Bool(true), types.Bool(false)} { - for i, j := 0, 0; i < len(rows); i++ { - rowIsGreat, _ := rows[i].CurrData().GetField(greatIndex) - - if rowIsGreat == great { - if !RowsEqualIgnoringSchema(rows[i], results[great][j]) { - t.Error(RowFmt(rows[i]), "!=", RowFmt(results[great][j])) - } - j++ - } - } - } -} -*/ diff --git a/go/libraries/doltcore/table/io.go b/go/libraries/doltcore/table/io.go deleted file mode 100644 index a60f6c3acc..0000000000 --- a/go/libraries/doltcore/table/io.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2019 Dolthub, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package table - -import ( - "context" - "errors" - "io" - - "github.com/dolthub/dolt/go/libraries/doltcore/row" -) - -// PipeRows will read a row from given TableReader and write it to the provided RowWriter. It will do this -// for every row until the TableReader's ReadRow method returns io.EOF or encounters an error in either reading -// or writing. The caller will need to handle closing the tables as necessary. If contOnBadRow is true, errors reading -// or writing will be ignored and the pipe operation will continue. -// -// Returns a tuple: (number of rows written, number of errors ignored, error). In the case that err is non-nil, the -// row counter fields in the tuple will be set to -1. -func PipeRows(ctx context.Context, rd SqlRowReader, wr SqlRowWriter, contOnBadRow bool) (int, int, error) { - var numBad, numGood int - for { - r, err := rd.ReadSqlRow(ctx) - - if err != nil && err != io.EOF { - if IsBadRow(err) && contOnBadRow { - numBad++ - continue - } - - return -1, -1, err - } else if err == io.EOF && r == nil { - break - } else if r == nil { - // row equal to nil should - return -1, -1, errors.New("reader returned nil row with err==nil") - } - - err = wr.WriteSqlRow(ctx, r) - - if err != nil { - return -1, -1, err - } else { - numGood++ - } - } - - return numGood, numBad, nil -} - -// ReadAllRows reads all rows from a TableReader and returns a slice containing those rows. Usually this is used -// for testing, or with very small data sets. -func ReadAllRows(ctx context.Context, rd Reader, contOnBadRow bool) ([]row.Row, int, error) { - var rows []row.Row - var err error - - badRowCount := 0 - for { - var r row.Row - r, err = rd.ReadRow(ctx) - - if err != nil && err != io.EOF || r == nil { - if IsBadRow(err) { - badRowCount++ - - if contOnBadRow { - continue - } - } - - break - } - - rows = append(rows, r) - } - - if err == nil || err == io.EOF { - return rows, badRowCount, nil - } - - return nil, badRowCount, err -} diff --git a/go/libraries/doltcore/table/table_reader.go b/go/libraries/doltcore/table/table_reader.go index dff02763f4..2936f63963 100644 --- a/go/libraries/doltcore/table/table_reader.go +++ b/go/libraries/doltcore/table/table_reader.go @@ -16,6 +16,8 @@ package table import ( "context" + "errors" + "io" "github.com/dolthub/go-mysql-server/sql" @@ -60,3 +62,41 @@ type SqlTableReader interface { // ReadSqlRow reads a row from a table as go-mysql-server sql.Row. ReadSqlRow(ctx context.Context) (sql.Row, error) } + +// PipeRows will read a row from given TableReader and write it to the provided RowWriter. It will do this +// for every row until the TableReader's ReadRow method returns io.EOF or encounters an error in either reading +// or writing. The caller will need to handle closing the tables as necessary. If contOnBadRow is true, errors reading +// or writing will be ignored and the pipe operation will continue. +// +// Returns a tuple: (number of rows written, number of errors ignored, error). In the case that err is non-nil, the +// row counter fields in the tuple will be set to -1. +func PipeRows(ctx context.Context, rd SqlRowReader, wr SqlRowWriter, contOnBadRow bool) (int, int, error) { + var numBad, numGood int + for { + r, err := rd.ReadSqlRow(ctx) + + if err != nil && err != io.EOF { + if IsBadRow(err) && contOnBadRow { + numBad++ + continue + } + + return -1, -1, err + } else if err == io.EOF && r == nil { + break + } else if r == nil { + // row equal to nil should + return -1, -1, errors.New("reader returned nil row with err==nil") + } + + err = wr.WriteSqlRow(ctx, r) + + if err != nil { + return -1, -1, err + } else { + numGood++ + } + } + + return numGood, numBad, nil +} diff --git a/go/libraries/doltcore/table/untyped/untyped_rows.go b/go/libraries/doltcore/table/untyped/untyped_rows.go index 37c13a0a09..5b60766dad 100644 --- a/go/libraries/doltcore/table/untyped/untyped_rows.go +++ b/go/libraries/doltcore/table/untyped/untyped_rows.go @@ -66,16 +66,6 @@ func NewRowFromStrings(nbf *types.NomsBinFormat, sch schema.Schema, valStrs []st return row.New(nbf, sch, taggedVals) } -// NewRowFromTaggedStrings takes an untyped schema and a map of column tag to string value and returns a row -func NewRowFromTaggedStrings(nbf *types.NomsBinFormat, sch schema.Schema, taggedStrs map[uint64]string) (row.Row, error) { - taggedVals := make(row.TaggedValues) - for tag, valStr := range taggedStrs { - taggedVals[tag] = types.String(valStr) - } - - return row.New(nbf, sch, taggedVals) -} - // UntypeSchema takes a schema and returns a schema with the same columns, but with the types of each of those columns // as types.StringKind func UntypeSchema(sch schema.Schema) (schema.Schema, error) { @@ -102,48 +92,6 @@ func UntypeSchema(sch schema.Schema) (schema.Schema, error) { return newSch, nil } -// UnkeySchema takes a schema and returns a schema with the same columns and types, but stripped of constraints and -// primary keys. Meant for use in result sets. -func UnkeySchema(sch schema.Schema) (schema.Schema, error) { - var cols []schema.Column - err := sch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) { - col.IsPartOfPK = false - col.Constraints = nil - cols = append(cols, col) - return false, nil - }) - - if err != nil { - return nil, err - } - - colColl := schema.NewColCollection(cols...) - - return schema.UnkeyedSchemaFromCols(colColl), nil -} - -// UntypeUnkeySchema takes a schema and returns a schema with the same columns, but stripped of constraints and primary -// keys and using only string types. Meant for displaying output and tests. -func UntypeUnkeySchema(sch schema.Schema) (schema.Schema, error) { - var cols []schema.Column - err := sch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) { - col.Kind = types.StringKind - col.IsPartOfPK = false - col.Constraints = nil - col.TypeInfo = typeinfo.StringDefaultType - cols = append(cols, col) - return false, nil - }) - - if err != nil { - return nil, err - } - - colColl := schema.NewColCollection(cols...) - - return schema.UnkeyedSchemaFromCols(colColl), nil -} - // UntypedSchemaUnion takes an arbitrary number of schemas and provides the union of all of their key and non-key columns. // The columns will all be of type types.StringKind and and IsPartOfPK will be false for every column, and all of the // columns will be in the schemas non-key ColumnCollection. Columns that share tags must have compatible types.