mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-03-17 23:56:33 -05:00
Code Issues Packages Projects Releases 147 Wiki Activity

removed noms map creator

Browse Source
This commit is contained in:
Andy Arthur
2021-12-13 15:23:51 -08:00
parent 0cbd7e780c
commit 3e800180af
13 changed files with 79 additions and 416 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go/libraries/doltcore/doltdocs/docs.go
View File

@@ -35,7 +35,7 @@ var doltDocsColumns = schema.NewColCollection(
schema.NewColumn(doltdb.DocPkColumnName, schema.DocNameTag, types.StringKind, true, schema.NotNullConstraint{}),
schema.NewColumn(doltdb.DocTextColumnName, schema.DocTextTag, types.StringKind, false),
)
var Schema = schema.MustSchemaFromCols(doltDocsColumns)
var DocsSchema = schema.MustSchemaFromCols(doltDocsColumns)
type Doc struct {
Text []byte

72
go/libraries/doltcore/doltdocs/docs_table.go
View File

@@ -24,7 +24,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding"
"github.com/dolthub/dolt/go/libraries/doltcore/table"
"github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms"
"github.com/dolthub/dolt/go/store/types"
)
@@ -81,59 +80,58 @@ func updateDocsTable(ctx context.Context, docTbl *doltdb.Table, docs Docs) (*dol
// createDocsTable creates a new in memory table that stores the given doc details.
func createDocsTable(ctx context.Context, vrw types.ValueReadWriter, docs Docs) (*doltdb.Table, error) {
imt := table.NewInMemTable(Schema)
rows := make([]row.Row, 0, len(docs))
// Determines if the table needs to be created at all and initializes a schema if it does.
createTable := false
for _, doc := range docs {
if doc.Text != nil {
createTable = true
docTaggedVals := row.TaggedValues{
schema.DocNameTag: types.String(doc.DocPk),
schema.DocTextTag: types.String(doc.Text),
}
docRow, err := row.New(types.Format_Default, Schema, docTaggedVals)
if err != nil {
return nil, err
}
err = imt.AppendRow(docRow)
r, err := row.New(types.Format_Default, DocsSchema, docTaggedVals)
if err != nil {
return nil, err
}
rows = append(rows, r)
}
}
if createTable {
rd := table.NewInMemTableReader(imt)
wr := noms.NewNomsMapCreator(context.Background(), vrw, Schema)
_, _, err := table.PipeRows(context.Background(), rd, wr, false)
if err != nil {
return nil, err
}
rd.Close(context.Background())
wr.Close(context.Background())
schVal, err := encoding.MarshalSchemaAsNomsValue(ctx, vrw, wr.GetSchema())
if err != nil {
return nil, ErrMarshallingSchema
}
empty, err := types.NewMap(ctx, vrw)
if err != nil {
return nil, err
}
newDocsTbl, err := doltdb.NewTable(ctx, vrw, schVal, wr.GetMap(), empty, nil)
if err != nil {
return nil, err
}
return newDocsTbl, nil
if len(rows) == 0 {
return nil, nil
}
empty, err := types.NewMap(ctx, vrw)
if err != nil {
return nil, err
}
me := empty.Edit()
for _, r := range rows {
k, v := r.NomsMapKey(DocsSchema), r.NomsMapValue(DocsSchema)
me.Set(k, v)
}
rowMap, err := me.Map(ctx)
if err != nil {
return nil, err
}
schVal, err := encoding.MarshalSchemaAsNomsValue(ctx, vrw, DocsSchema)
if err != nil {
return nil, err
}
newDocsTbl, err := doltdb.NewTable(ctx, vrw, schVal, rowMap, empty, nil)
if err != nil {
return nil, err
}
return newDocsTbl, nil
return nil, nil
}

27
go/libraries/doltcore/dtestutils/environment.go
View File

@@ -23,9 +23,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding"
"github.com/dolthub/dolt/go/libraries/doltcore/table"
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
"github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/types"
)
@@ -65,25 +63,28 @@ func CreateEnvWithSeedData(t *testing.T) *env.DoltEnv {
ctx := context.Background()
vrw := dEnv.DoltDB.ValueReadWriter()
rd := table.NewInMemTableReader(imt)
wr := noms.NewNomsMapCreator(ctx, vrw, sch)
_, _, err := table.PipeRows(ctx, rd, wr, false)
rowMap, err := types.NewMap(ctx, vrw)
require.NoError(t, err)
err = rd.Close(ctx)
me := rowMap.Edit()
for i := 0; i < imt.NumRows(); i++ {
r, err := imt.GetRow(i)
require.NoError(t, err)
k, v := r.NomsMapKey(sch), r.NomsMapValue(sch)
me.Set(k, v)
}
rowMap, err = me.Map(ctx)
require.NoError(t, err)
err = wr.Close(ctx)
require.NoError(t, err)
ai := sch.Indexes().AllIndexes()
sch = wr.GetSchema()
sch.Indexes().Merge(ai...)
schVal, err := encoding.MarshalSchemaAsNomsValue(ctx, vrw, sch)
require.NoError(t, err)
empty, err := types.NewMap(ctx, vrw)
require.NoError(t, err)
tbl, err := doltdb.NewTable(ctx, vrw, schVal, wr.GetMap(), empty, nil)
ai := sch.Indexes().AllIndexes()
sch.Indexes().Merge(ai...)
tbl, err := doltdb.NewTable(ctx, vrw, schVal, rowMap, empty, nil)
require.NoError(t, err)
tbl, err = editor.RebuildAllIndexes(ctx, tbl, editor.TestEditorOptions(vrw))
require.NoError(t, err)

18
go/libraries/doltcore/dtestutils/schema.go
View File

@@ -29,7 +29,6 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding"
"github.com/dolthub/dolt/go/libraries/doltcore/table"
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
"github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms"
"github.com/dolthub/dolt/go/store/types"
)
@@ -100,21 +99,24 @@ func CreateTestTable(t *testing.T, dEnv *env.DoltEnv, tableName string, sch sche
ctx := context.Background()
vrw := dEnv.DoltDB.ValueReadWriter()
rd := table.NewInMemTableReader(imt)
wr := noms.NewNomsMapCreator(ctx, vrw, sch)
_, _, err := table.PipeRows(ctx, rd, wr, false)
rowMap, err := types.NewMap(ctx, vrw)
require.NoError(t, err)
err = rd.Close(ctx)
require.NoError(t, err)
err = wr.Close(ctx)
me := rowMap.Edit()
for i := 0; i < imt.NumRows(); i++ {
r, err := imt.GetRow(i)
require.NoError(t, err)
k, v := r.NomsMapKey(sch), r.NomsMapValue(sch)
me.Set(k, v)
}
rowMap, err = me.Map(ctx)
require.NoError(t, err)
schVal, err := encoding.MarshalSchemaAsNomsValue(ctx, vrw, sch)
require.NoError(t, err)
empty, err := types.NewMap(ctx, vrw)
require.NoError(t, err)
tbl, err := doltdb.NewTable(ctx, vrw, schVal, wr.GetMap(), empty, nil)
tbl, err := doltdb.NewTable(ctx, vrw, schVal, rowMap, empty, nil)
require.NoError(t, err)
tbl, err = editor.RebuildAllIndexes(ctx, tbl, editor.TestEditorOptions(vrw))
require.NoError(t, err)

27
go/libraries/doltcore/dtestutils/testcommands/multienv.go
View File

@@ -28,9 +28,7 @@ import (
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding"
"github.com/dolthub/dolt/go/libraries/doltcore/table"
"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
"github.com/dolthub/dolt/go/libraries/doltcore/table/typed/noms"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/types"
)
@@ -316,29 +314,24 @@ func (mr *MultiRepoTestSetup) PushToRemote(dbName, remoteName, branchName string
// createTestTable creates a new test table with the name, schema, and rows given.
func createTestTable(dEnv *env.DoltEnv, tableName string, sch schema.Schema, errhand func(args ...interface{}), rs ...row.Row) {
imt := table.NewInMemTable(sch)
for _, r := range rs {
_ = imt.AppendRow(r)
}
ctx := context.Background()
vrw := dEnv.DoltDB.ValueReadWriter()
rd := table.NewInMemTableReader(imt)
wr := noms.NewNomsMapCreator(ctx, vrw, sch)
_, _, err := table.PipeRows(ctx, rd, wr, false)
rowMap, err := types.NewMap(ctx, vrw)
if err != nil {
errhand(err)
}
err = rd.Close(ctx)
if err != nil {
errhand(err)
}
err = wr.Close(ctx)
me := rowMap.Edit()
for _, r := range rs {
k, v := r.NomsMapKey(sch), r.NomsMapValue(sch)
me.Set(k, v)
}
rowMap, err = me.Map(ctx)
if err != nil {
errhand(err)
}
schVal, err := encoding.MarshalSchemaAsNomsValue(ctx, vrw, sch)
if err != nil {
errhand(err)
@@ -347,7 +340,7 @@ func createTestTable(dEnv *env.DoltEnv, tableName string, sch schema.Schema, err
if err != nil {
errhand(err)
}
tbl, err := doltdb.NewTable(ctx, vrw, schVal, wr.GetMap(), empty, nil)
tbl, err := doltdb.NewTable(ctx, vrw, schVal, rowMap, empty, nil)
if err != nil {
errhand(err)
}

2
go/libraries/doltcore/sqle/sqlddl_test.go
View File

@@ -1164,7 +1164,7 @@ func TestAlterSystemTables(t *testing.T) {
CreateTestDatabase(dEnv, t)
dtestutils.CreateTestTable(t, dEnv, "dolt_docs",
doltdocs.Schema,
doltdocs.DocsSchema,
NewRow(types.String("LICENSE.md"), types.String("A license")))
dtestutils.CreateTestTable(t, dEnv, doltdb.DoltQueryCatalogTableName,
dtables.DoltQueryCatalogSchema,

2
go/libraries/doltcore/sqle/sqldelete_test.go
View File

@@ -193,7 +193,7 @@ var systemTableDeleteTests = []DeleteTest{
{
Name: "delete dolt_docs",
AdditionalSetup: CreateTableFn("dolt_docs",
doltdocs.Schema,
doltdocs.DocsSchema,
NewRow(types.String("LICENSE.md"), types.String("A license"))),
DeleteQuery: "delete from dolt_docs",
ExpectedErr: "cannot delete from table",

2
go/libraries/doltcore/sqle/sqlinsert_test.go
View File

@@ -398,7 +398,7 @@ var systemTableInsertTests = []InsertTest{
{
Name: "insert into dolt_docs",
AdditionalSetup: CreateTableFn("dolt_docs",
doltdocs.Schema,
doltdocs.DocsSchema,
NewRow(types.String("LICENSE.md"), types.String("A license"))),
InsertQuery: "insert into dolt_docs (doc_name, doc_text) values ('README.md', 'Some text')",
ExpectedErr: "cannot insert into table",

2
go/libraries/doltcore/sqle/sqlreplace_test.go
View File

@@ -255,7 +255,7 @@ var systemTableReplaceTests = []ReplaceTest{
{
Name: "replace into dolt_docs",
AdditionalSetup: CreateTableFn("dolt_docs",
doltdocs.Schema,
doltdocs.DocsSchema,
NewRow(types.String("LICENSE.md"), types.String("A license"))),
ReplaceQuery: "replace into dolt_docs (doc_name, doc_text) values ('README.md', 'Some text')",
ExpectedErr: "cannot insert into table",

8
go/libraries/doltcore/sqle/sqlselect_test.go
View File

@@ -1484,15 +1484,15 @@ var systemTableSelectTests = []SelectTest{
{
Name: "select from dolt_docs",
AdditionalSetup: CreateTableFn("dolt_docs",
doltdocs.Schema,
NewRowWithSchema(doltdocs.Schema,
doltdocs.DocsSchema,
NewRowWithSchema(doltdocs.DocsSchema,
types.String("LICENSE.md"),
types.String("A license")),
),
Query: "select * from dolt_docs",
ExpectedRows: ToSqlRows(CompressSchema(doltdocs.Schema),
ExpectedRows: ToSqlRows(CompressSchema(doltdocs.DocsSchema),
NewRow(types.String("LICENSE.md"), types.String("A license"))),
ExpectedSchema: CompressSchema(doltdocs.Schema),
ExpectedSchema: CompressSchema(doltdocs.DocsSchema),
},
{
Name: "select from dolt_query_catalog",

2
go/libraries/doltcore/sqle/sqlupdate_test.go
View File

@@ -377,7 +377,7 @@ var systemTableUpdateTests = []UpdateTest{
{
Name: "update dolt_docs",
AdditionalSetup: CreateTableFn("dolt_docs",
doltdocs.Schema,
doltdocs.DocsSchema,
NewRow(types.String("LICENSE.md"), types.String("A license"))),
UpdateQuery: "update dolt_docs set doc_text = 'Some text')",
ExpectedErr: "cannot insert into table",

142
go/libraries/doltcore/table/typed/noms/noms_map_creator.go
View File

@@ -1,142 +0,0 @@
// Copyright 2019 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package noms
import (
"context"
"errors"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/types"
)
// NomsMapUpdater is a TableWriter that creates a new noms types.Map. It is backed by a StreamingMap which requires
// the rows to be written in order. If the keys being written to WriteRow are not sorted an error will be returned from
// WriteRow. Once all rows are written Close() should be called and GetMap will then return the new map.
type NomsMapCreator struct {
sch schema.Schema
vrw types.ValueReadWriter
lastPK types.LesserValuable
kvsChan chan<- types.Value
streamingMap *types.StreamingMap
err error
result *types.Map
}
// NewNomsMapCreator creates a new NomsMapCreator.
func NewNomsMapCreator(ctx context.Context, vrw types.ValueReadWriter, sch schema.Schema) *NomsMapCreator {
kvsChan := make(chan types.Value)
streamingMap := types.NewStreamingMap(ctx, vrw, kvsChan)
return &NomsMapCreator{sch, vrw, nil, kvsChan, streamingMap, nil, nil}
}
// GetSchema gets the schema of the rows that this writer writes
func (nmc *NomsMapCreator) GetSchema() schema.Schema {
return nmc.sch
}
// WriteRow will write a row to a table. The primary key for each row must be greater than the primary key of the row
// written before it.
func (nmc *NomsMapCreator) WriteRow(ctx context.Context, r row.Row) error {
if nmc.err != nil {
return nmc.err
}
if nmc.kvsChan == nil {
return errors.New("writing to NomsMapCreator after closing")
}
send := func(v types.Value) error {
select {
case nmc.kvsChan <- v:
return nil
case <-nmc.streamingMap.Done():
_, err := nmc.streamingMap.Wait()
return err
}
}
err := func() error {
pk := r.NomsMapKey(nmc.sch)
fieldVals := r.NomsMapValue(nmc.sch)
isOK := nmc.lastPK == nil
if !isOK {
var err error
isOK, err = nmc.lastPK.Less(nmc.vrw.Format(), pk)
if err != nil {
return err
}
}
if isOK {
pkVal, err := pk.Value(ctx)
if err != nil {
return err
}
fv, err := fieldVals.Value(ctx)
if err != nil {
return err
}
if err := send(pkVal); err != nil {
return err
}
if err := send(fv); err != nil {
return err
}
nmc.lastPK = pk
return nil
} else {
return errors.New("Input was not sorted by the primary key")
}
}()
if err != nil {
close(nmc.kvsChan)
nmc.kvsChan = nil
nmc.err = err
}
return err
}
// Close should flush all writes, release resources being held. After this call is made no more rows may be written,
// and the value of GetMap becomes valid.
func (nmc *NomsMapCreator) Close(ctx context.Context) error {
if nmc.err != nil {
return nmc.err
}
if nmc.result == nil {
close(nmc.kvsChan)
nmc.kvsChan = nil
m, err := nmc.streamingMap.Wait()
nmc.result = &m
return err
} else {
return errors.New("Already closed.")
}
}
// GetMap retrieves the resulting types.Map once close is called
func (nmc *NomsMapCreator) GetMap() types.Map {
// Might want to panic if this was never closed
return *nmc.result
}

189
go/libraries/doltcore/table/typed/noms/read_write_test.go
View File

@@ -1,189 +0,0 @@
// Copyright 2019 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package noms
import (
"context"
"io"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
"github.com/dolthub/dolt/go/libraries/doltcore/row"
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
"github.com/dolthub/dolt/go/store/types"
)
const (
idCol = "id"
nameCol = "name"
ageCol = "age"
titleCol = "title"
idColTag = 4
nameColTag = 3
ageColTag = 2
titleColTag = 1
)
var colColl = schema.NewColCollection(
schema.NewColumn(idCol, idColTag, types.UUIDKind, true, schema.NotNullConstraint{}),
schema.NewColumn(nameCol, nameColTag, types.StringKind, false),
schema.NewColumn(ageCol, ageColTag, types.UintKind, false),
schema.NewColumn(titleCol, titleColTag, types.StringKind, false),
)
var sch = schema.MustSchemaFromCols(colColl)
var uuids = []uuid.UUID{
uuid.Must(uuid.Parse("00000000-0000-0000-0000-000000000000")),
uuid.Must(uuid.Parse("00000000-0000-0000-0000-000000000001")),
uuid.Must(uuid.Parse("00000000-0000-0000-0000-000000000002"))}
var names = []string{"Bill Billerson", "John Johnson", "Rob Robertson"}
var ages = []uint{32, 25, 21}
var titles = []string{"Senior Dufus", "Dufus", ""}
var updatedIndices = []bool{false, true, true}
var updatedAges = []uint{0, 26, 20}
func createRows(t *testing.T, onlyUpdated, updatedAge bool) []row.Row {
rows := make([]row.Row, 0, len(names))
for i := 0; i < len(names); i++ {
if !onlyUpdated || updatedIndices[i] {
age := ages[i]
if updatedAge && updatedIndices[i] {
age = updatedAges[i]
}
rowVals := row.TaggedValues{
idColTag: types.UUID(uuids[i]),
nameColTag: types.String(names[i]),
ageColTag: types.Uint(age),
titleColTag: types.String(titles[i]),
}
r, err := row.New(types.Format_Default, sch, rowVals)
assert.NoError(t, err)
rows = append(rows, r)
}
}
return rows
}
func TestReadWrite(t *testing.T) {
db, _ := dbfactory.MemFactory{}.CreateDB(context.Background(), types.Format_Default, nil, nil)
rows := createRows(t, false, false)
initialMapVal := testNomsMapCreator(t, db, rows)
testReadAndCompare(t, initialMapVal, rows)
updatedRows := createRows(t, true, true)
updatedMap := testNomsMapUpdate(t, db, initialMapVal, updatedRows)
expectedRows := createRows(t, false, true)
testReadAndCompare(t, updatedMap, expectedRows)
}
func testNomsMapCreator(t *testing.T, vrw types.ValueReadWriter, rows []row.Row) types.Map {
mc := NewNomsMapCreator(context.Background(), vrw, sch)
for _, r := range rows {
err := mc.WriteRow(context.Background(), r)
if err != nil {
t.Error("Failed to write row.", err)
}
}
err := mc.Close(context.Background())
if err != nil {
t.Fatal("Failed to close writer")
}
err = mc.Close(context.Background())
if err == nil {
t.Error("Should give error for having already been closed")
}
err = mc.WriteRow(context.Background(), rows[0])
if err == nil {
t.Error("Should give error for writing after closing.")
}
return mc.GetMap()
}
func testNomsMapUpdate(t *testing.T, vrw types.ValueReadWriter, initialMapVal types.Map, rows []row.Row) types.Map {
mu := NewNomsMapUpdater(context.Background(), vrw, initialMapVal, sch, nil)
for _, r := range rows {
err := mu.WriteRow(context.Background(), r)
if err != nil {
t.Error("Failed to write row.", err)
}
}
err := mu.Close(context.Background())
if err != nil {
t.Fatal("Failed to close writer")
}
err = mu.Close(context.Background())
if err == nil {
t.Error("Should give error for having already been closed")
}
err = mu.WriteRow(context.Background(), rows[0])
if err == nil {
t.Error("Should give error for writing after closing.")
}
return mu.GetMap()
}
func testReadAndCompare(t *testing.T, initialMapVal types.Map, expectedRows []row.Row) {
ctx := context.Background()
mr, err := NewNomsMapReader(context.Background(), initialMapVal, sch)
assert.NoError(t, err)
var r row.Row
var actualRows []row.Row
for {
r, err = mr.ReadRow(ctx)
if err == io.EOF {
break
}
assert.NoError(t, err)
actualRows = append(actualRows, r)
}
assert.Equal(t, len(actualRows), len(expectedRows))
for i := 0; i < len(expectedRows); i++ {
if !row.AreEqual(actualRows[i], expectedRows[i], sch) {
t.Error(row.Fmt(ctx, actualRows[i], sch), "!=", row.Fmt(context.Background(), expectedRows[i], sch))
}
}
}