From bde6a0c764e465328d52c0347f593f3a231b948a Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 3 Mar 2022 11:03:00 -0800 Subject: [PATCH 01/16] column union works like a set union --- go/libraries/doltcore/schema/col_coll.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/libraries/doltcore/schema/col_coll.go b/go/libraries/doltcore/schema/col_coll.go index b8160a916a..f0d4b73f1d 100644 --- a/go/libraries/doltcore/schema/col_coll.go +++ b/go/libraries/doltcore/schema/col_coll.go @@ -292,10 +292,16 @@ func FilterColCollection(cc *ColCollection, cb func(col Column) bool) *ColCollec } func ColCollUnion(colColls ...*ColCollection) (*ColCollection, error) { + var allTags = make(map[uint64]bool) var allCols []Column for _, sch := range colColls { err := sch.Iter(func(tag uint64, col Column) (stop bool, err error) { + // skip if already seen + if _, ok := allTags[tag]; ok { + return false, nil + } allCols = append(allCols, col) + allTags[tag] = true return false, nil }) From a48b32c2c789cffb4059f255db2d31af65ac56d0 Mon Sep 17 00:00:00 2001 From: James Cor Date: Thu, 3 Mar 2022 11:51:26 -0800 Subject: [PATCH 02/16] adding bats test --- integration-tests/bats/sql-merge.bats | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/integration-tests/bats/sql-merge.bats b/integration-tests/bats/sql-merge.bats index 2ddb859b5b..5858c3e382 100644 --- a/integration-tests/bats/sql-merge.bats +++ b/integration-tests/bats/sql-merge.bats @@ -741,6 +741,26 @@ SQL [[ "$output" =~ "current fast forward from a to b. a is ahead of b already" ]] || false } +@test "sql-merge: identical schema changes with data changes merges correctly" { + dolt sql -q "create table t (i int primary key)" + dolt commit -am "initial commit" + dolt branch b1 + dolt branch b2 + dolt checkout b1 + dolt sql -q "alter table t add column j int" + dolt sql -q "insert into t values (1, 1)" + dolt commit -am "changes to b1" + dolt checkout b2 + dolt sql -q "alter table t add column j int" + dolt sql -q "insert into t values (2, 2)" + dolt commit -am "changes to b2" + dolt checkout main + run dolt merge b1 + [ $status -eq 0 ] + run dolt merge b2 + [ $status -eq 0 ] +} + get_head_commit() { dolt log -n 1 | grep -m 1 commit | cut -c 13-44 } From e631e9bdf459f3e1ce37d582d02b0d965e5f586a Mon Sep 17 00:00:00 2001 From: Dhruv Sringari Date: Fri, 4 Mar 2022 13:30:12 -0800 Subject: [PATCH 03/16] io.Copy from file io.ReaderClose to response writer to get smaller http chunking --- go/utils/remotesrv/http.go | 187 +++++++++++++++---------------------- go/utils/remotesrv/main.go | 2 + 2 files changed, 77 insertions(+), 112 deletions(-) diff --git a/go/utils/remotesrv/http.go b/go/utils/remotesrv/http.go index fca1c772da..6640c20402 100644 --- a/go/utils/remotesrv/http.go +++ b/go/utils/remotesrv/http.go @@ -28,10 +28,14 @@ import ( remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1" - "github.com/dolthub/dolt/go/libraries/utils/iohelp" "github.com/dolthub/dolt/go/store/hash" ) +var ( + ErrReadOutOfBounds = errors.New("cannot read file for given length and " + + "offset since the read would exceed the size of the file") +) + var expectedFiles = make(map[string]*remotesapi.TableFileDetails) func ServeHTTP(respWr http.ResponseWriter, req *http.Request) { @@ -53,13 +57,7 @@ func ServeHTTP(respWr http.ResponseWriter, req *http.Request) { statusCode := http.StatusMethodNotAllowed switch req.Method { case http.MethodGet: - rangeStr := req.Header.Get("Range") - - if rangeStr == "" { - statusCode = readFile(logger, org, repo, hashStr, respWr) - } else { - statusCode = readChunk(logger, org, repo, hashStr, rangeStr, respWr) - } + statusCode = readTableFile(logger, org, repo, hashStr, respWr, req) case http.MethodPost, http.MethodPut: statusCode = writeTableFile(logger, org, repo, hashStr, req) @@ -70,6 +68,58 @@ func ServeHTTP(respWr http.ResponseWriter, req *http.Request) { } } +func readTableFile(logger func(string), org, repo, fileId string, respWr http.ResponseWriter, req *http.Request) int { + rangeStr := req.Header.Get("Range") + path := filepath.Join(org, repo, fileId) + + var r io.ReadCloser + var readSize int64 + var fileErr error + { + if rangeStr == "" { + r, readSize, fileErr = getFileReader(path) + } else { + offset, length, err := offsetAndLenFromRange(rangeStr) + if err != nil { + logger(err.Error()) + return http.StatusBadRequest + } + readSize = length + r, fileErr = getFileReaderAt(path, offset, length) + } + } + defer func() { + err := r.Close() + if err != nil { + err = fmt.Errorf("failed to close file at path %s: %w", path, err) + logger(err.Error()) + } + }() + + if fileErr != nil { + logger(fileErr.Error()) + if errors.Is(fileErr, os.ErrNotExist) { + return http.StatusNotFound + } else if errors.Is(fileErr, ErrReadOutOfBounds) { + return http.StatusBadRequest + } + return http.StatusInternalServerError + } + + n, err := io.Copy(respWr, r) + if err != nil { + err = fmt.Errorf("failed to write data to response writer: %w", err) + logger(err.Error()) + return http.StatusInternalServerError + } + if n != readSize { + logger(fmt.Sprintf("wanted to write %d bytes from file (%s) but only wrote %d", readSize, path, n)) + return http.StatusInternalServerError + } + + return -1 +} + func writeTableFile(logger func(string), org, repo, fileId string, request *http.Request) int { _, ok := hash.MaybeParse(fileId) @@ -157,127 +207,40 @@ func offsetAndLenFromRange(rngStr string) (int64, int64, error) { return int64(start), int64(end-start) + 1, nil } -func readFile(logger func(string), org, repo, fileId string, writer io.Writer) int { - path := filepath.Join(org, repo, fileId) +// getFileReader opens a file at the given path and returns an io.ReadCloser, +// the corresponding file's filesize, and a http status. +func getFileReader(path string) (io.ReadCloser, int64, error) { + return openFile(path) +} +func openFile(path string) (*os.File, int64, error) { info, err := os.Stat(path) - if err != nil { - logger("file not found. path: " + path) - return http.StatusNotFound + return nil, 0, fmt.Errorf("failed to get stats for file at path %s: %w", path, err) } f, err := os.Open(path) - if err != nil { - logger("failed to open file. file: " + path + " err: " + err.Error()) - return http.StatusInternalServerError + return nil, 0, fmt.Errorf("failed to open file at path %s: %w", path, err) } - defer func() { - err := f.Close() - - if err != nil { - logger(fmt.Sprintf("Close failed. file: %s, err: %v", path, err)) - } else { - logger("Close Successful") - } - }() - - n, err := io.Copy(writer, f) - - if err != nil { - logger("failed to write data to response. err : " + err.Error()) - return -1 - } - - if n != info.Size() { - logger(fmt.Sprintf("failed to write entire file to response. Copied %d of %d err: %v", n, info.Size(), err)) - return -1 - } - - return -1 + return f, info.Size(), nil } -func readChunk(logger func(string), org, repo, fileId, rngStr string, writer io.Writer) int { - offset, length, err := offsetAndLenFromRange(rngStr) - +func getFileReaderAt(path string, offset int64, length int64) (io.ReadCloser, error) { + f, fSize, err := openFile(path) if err != nil { - logger(fmt.Sprintln(rngStr, "is not a valid range")) - return http.StatusBadRequest + return nil, err } - data, retVal := readLocalRange(logger, org, repo, fileId, int64(offset), int64(length)) - - if retVal != -1 { - return retVal + if fSize < int64(offset+length) { + return nil, fmt.Errorf("failed to read file %s at offset %d, length %d: %w", path, offset, length, ErrReadOutOfBounds) } - logger(fmt.Sprintf("writing %d bytes", len(data))) - err = iohelp.WriteAll(writer, data) - + _, err = f.Seek(int64(offset), 0) if err != nil { - logger("failed to write data to response " + err.Error()) - return -1 + return nil, fmt.Errorf("failed to seek file at path %s to offset %d: %w", path, offset, err) } - logger("Successfully wrote data") - return -1 -} - -func readLocalRange(logger func(string), org, repo, fileId string, offset, length int64) ([]byte, int) { - path := filepath.Join(org, repo, fileId) - - logger(fmt.Sprintf("Attempting to read bytes %d to %d from %s", offset, offset+length, path)) - info, err := os.Stat(path) - - if err != nil { - logger(fmt.Sprintf("file %s not found", path)) - return nil, http.StatusNotFound - } - - logger(fmt.Sprintf("Verified file %s exists", path)) - - if info.Size() < int64(offset+length) { - logger(fmt.Sprintf("Attempted to read bytes %d to %d, but the file is only %d bytes in size", offset, offset+length, info.Size())) - return nil, http.StatusBadRequest - } - - logger(fmt.Sprintf("Verified the file is large enough to contain the range")) - f, err := os.Open(path) - - if err != nil { - logger(fmt.Sprintf("Failed to open %s: %v", path, err)) - return nil, http.StatusInternalServerError - } - - defer func() { - err := f.Close() - - if err != nil { - logger(fmt.Sprintf("Close failed. file: %s, err: %v", path, err)) - } else { - logger("Close Successful") - } - }() - - logger(fmt.Sprintf("Successfully opened file")) - pos, err := f.Seek(int64(offset), 0) - - if err != nil { - logger(fmt.Sprintf("Failed to seek to %d: %v", offset, err)) - return nil, http.StatusInternalServerError - } - - logger(fmt.Sprintf("Seek succeeded. Current position is %d", pos)) - diff := offset - pos - data, err := iohelp.ReadNBytes(f, int(diff+int64(length))) - - if err != nil { - logger(fmt.Sprintf("Failed to read %d bytes: %v", diff+length, err)) - return nil, http.StatusInternalServerError - } - - logger(fmt.Sprintf("Successfully read %d bytes", len(data))) - return data[diff:], -1 + return io.LimitReader(f, length).(io.ReadCloser), nil } diff --git a/go/utils/remotesrv/main.go b/go/utils/remotesrv/main.go index 40f3c3a415..823203faf5 100644 --- a/go/utils/remotesrv/main.go +++ b/go/utils/remotesrv/main.go @@ -38,6 +38,8 @@ func main() { httpHostParam := flag.String("http-host", "localhost", "host url that this command will assume.") flag.Parse() + log.SetOutput(os.Stdout) + if dirParam != nil && len(*dirParam) > 0 { err := os.Chdir(*dirParam) From b6cf9de9b4ea66c3f861189eb8dc4dc788e75806 Mon Sep 17 00:00:00 2001 From: Dhruv Sringari Date: Fri, 4 Mar 2022 13:36:23 -0800 Subject: [PATCH 04/16] fix cast --- go/utils/remotesrv/http.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/go/utils/remotesrv/http.go b/go/utils/remotesrv/http.go index 6640c20402..19d7fdd52c 100644 --- a/go/utils/remotesrv/http.go +++ b/go/utils/remotesrv/http.go @@ -227,6 +227,11 @@ func openFile(path string) (*os.File, int64, error) { return f, info.Size(), nil } +type closerReaderWrapper struct { + io.Reader + io.Closer +} + func getFileReaderAt(path string, offset int64, length int64) (io.ReadCloser, error) { f, fSize, err := openFile(path) if err != nil { @@ -242,5 +247,6 @@ func getFileReaderAt(path string, offset int64, length int64) (io.ReadCloser, er return nil, fmt.Errorf("failed to seek file at path %s to offset %d: %w", path, offset, err) } - return io.LimitReader(f, length).(io.ReadCloser), nil + r := closerReaderWrapper{io.LimitReader(f, length), f} + return r, nil } From 738cf4bcadaf4a971eeff2344fc6317e34fb0f85 Mon Sep 17 00:00:00 2001 From: andrew-wm-arthur Date: Fri, 4 Mar 2022 21:38:56 +0000 Subject: [PATCH 05/16] [ga-bump-dep] Bump dependency in Dolt by andrew-wm-arthur --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index 19a8e5f507..ab209d8f05 100644 --- a/go/go.mod +++ b/go/go.mod @@ -68,7 +68,7 @@ require ( ) require ( - github.com/dolthub/go-mysql-server v0.11.1-0.20220304002938-823e425edf58 + github.com/dolthub/go-mysql-server v0.11.1-0.20220304213711-4d7d9a2c6f81 github.com/google/flatbuffers v2.0.5+incompatible github.com/gosuri/uilive v0.0.4 github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 diff --git a/go/go.sum b/go/go.sum index 199001d2c8..f3a9ea2d5e 100755 --- a/go/go.sum +++ b/go/go.sum @@ -170,8 +170,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.11.1-0.20220304002938-823e425edf58 h1:Fgi+KjilXXJe1fYsdlr+xf7tubprV3wtKP+Y3WyCILM= -github.com/dolthub/go-mysql-server v0.11.1-0.20220304002938-823e425edf58/go.mod h1:5WoXPdkIrkNBjKH+Y1XMfwREEtPXOW/yN8QfulFpZ1s= +github.com/dolthub/go-mysql-server v0.11.1-0.20220304213711-4d7d9a2c6f81 h1:uk9aHMW7ji1rbSBhAq0h/Ncy4/mIN+7cFqk/zQES3Zo= +github.com/dolthub/go-mysql-server v0.11.1-0.20220304213711-4d7d9a2c6f81/go.mod h1:5WoXPdkIrkNBjKH+Y1XMfwREEtPXOW/yN8QfulFpZ1s= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g= github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms= github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8= From b4f003d7be1e87db1606b66b04e91c4887fb1961 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 4 Mar 2022 13:42:51 -0800 Subject: [PATCH 06/16] updated DoltIndex to satisfy sql.FilteredIndex interface --- go/libraries/doltcore/sqle/index/dolt_index.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/sqle/index/dolt_index.go b/go/libraries/doltcore/sqle/index/dolt_index.go index 45d74bf91c..66122d5167 100644 --- a/go/libraries/doltcore/sqle/index/dolt_index.go +++ b/go/libraries/doltcore/sqle/index/dolt_index.go @@ -31,7 +31,7 @@ import ( ) type DoltIndex interface { - sql.Index + sql.FilteredIndex Schema() schema.Schema IndexSchema() schema.Schema TableData() durable.Index @@ -289,6 +289,10 @@ RangeLoop: }, nil } +func (di doltIndex) HandledFilters(filters []sql.Expression) []sql.Expression { + return filters +} + // Database implement sql.Index func (di doltIndex) Database() string { return di.dbName From 96240e664b74860beb0e1555e8fd5ae9a25cfb98 Mon Sep 17 00:00:00 2001 From: Dhruv Sringari Date: Fri, 4 Mar 2022 13:44:39 -0800 Subject: [PATCH 07/16] add happy path logging --- go/utils/remotesrv/http.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/utils/remotesrv/http.go b/go/utils/remotesrv/http.go index 19d7fdd52c..376e0dc77d 100644 --- a/go/utils/remotesrv/http.go +++ b/go/utils/remotesrv/http.go @@ -77,6 +77,7 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re var fileErr error { if rangeStr == "" { + logger("going to write entire file") r, readSize, fileErr = getFileReader(path) } else { offset, length, err := offsetAndLenFromRange(rangeStr) @@ -84,6 +85,7 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re logger(err.Error()) return http.StatusBadRequest } + logger(fmt.Sprintf("going to write bytes at offset %d, length %d", offset, length)) readSize = length r, fileErr = getFileReaderAt(path, offset, length) } @@ -106,6 +108,8 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re return http.StatusInternalServerError } + logger(fmt.Sprintf("opened file at path %s, going to write %d bytes", path, readSize)) + n, err := io.Copy(respWr, r) if err != nil { err = fmt.Errorf("failed to write data to response writer: %w", err) @@ -117,6 +121,8 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re return http.StatusInternalServerError } + logger(fmt.Sprintf("wrote %d bytes", n)) + return -1 } From 3ab70d8b84fb05bf241e04af60a4194aa79e4717 Mon Sep 17 00:00:00 2001 From: Dhruv Sringari Date: Fri, 4 Mar 2022 13:46:25 -0800 Subject: [PATCH 08/16] better log entries --- go/utils/remotesrv/http.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/utils/remotesrv/http.go b/go/utils/remotesrv/http.go index 376e0dc77d..965ddec0b6 100644 --- a/go/utils/remotesrv/http.go +++ b/go/utils/remotesrv/http.go @@ -77,7 +77,7 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re var fileErr error { if rangeStr == "" { - logger("going to write entire file") + logger("going to read entire file") r, readSize, fileErr = getFileReader(path) } else { offset, length, err := offsetAndLenFromRange(rangeStr) @@ -85,7 +85,7 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re logger(err.Error()) return http.StatusBadRequest } - logger(fmt.Sprintf("going to write bytes at offset %d, length %d", offset, length)) + logger(fmt.Sprintf("going to read file at offset %d, length %d", offset, length)) readSize = length r, fileErr = getFileReaderAt(path, offset, length) } @@ -108,7 +108,7 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re return http.StatusInternalServerError } - logger(fmt.Sprintf("opened file at path %s, going to write %d bytes", path, readSize)) + logger(fmt.Sprintf("opened file at path %s, going to read %d bytes", path, readSize)) n, err := io.Copy(respWr, r) if err != nil { From cd2d5290b1a87c5395c66319a60b902326a4a122 Mon Sep 17 00:00:00 2001 From: Dhruv Sringari Date: Fri, 4 Mar 2022 13:51:37 -0800 Subject: [PATCH 09/16] fix close --- go/utils/remotesrv/http.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/go/utils/remotesrv/http.go b/go/utils/remotesrv/http.go index 965ddec0b6..7bedd9e71d 100644 --- a/go/utils/remotesrv/http.go +++ b/go/utils/remotesrv/http.go @@ -90,14 +90,6 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re r, fileErr = getFileReaderAt(path, offset, length) } } - defer func() { - err := r.Close() - if err != nil { - err = fmt.Errorf("failed to close file at path %s: %w", path, err) - logger(err.Error()) - } - }() - if fileErr != nil { logger(fileErr.Error()) if errors.Is(fileErr, os.ErrNotExist) { @@ -107,6 +99,13 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re } return http.StatusInternalServerError } + defer func() { + err := r.Close() + if err != nil { + err = fmt.Errorf("failed to close file at path %s: %w", path, err) + logger(err.Error()) + } + }() logger(fmt.Sprintf("opened file at path %s, going to read %d bytes", path, readSize)) From 7884b1f72114706320cfabc6c77025d562c6a1d4 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 4 Mar 2022 13:55:48 -0800 Subject: [PATCH 10/16] added stub for transaction tests --- .../transactions/concurrent_tx_test.go | 41 +++++++++++++++++++ integration-tests/transactions/go.mod | 12 ++++++ integration-tests/transactions/go.sum | 12 ++++++ 3 files changed, 65 insertions(+) create mode 100644 integration-tests/transactions/concurrent_tx_test.go create mode 100644 integration-tests/transactions/go.mod create mode 100644 integration-tests/transactions/go.sum diff --git a/integration-tests/transactions/concurrent_tx_test.go b/integration-tests/transactions/concurrent_tx_test.go new file mode 100644 index 0000000000..a0ea42f259 --- /dev/null +++ b/integration-tests/transactions/concurrent_tx_test.go @@ -0,0 +1,41 @@ +// Copyright 2022 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transactions + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConcurrentTransactions(t *testing.T) { + for _, test := range txTests { + t.Run(test.name, func(t *testing.T) { + testConcurrentTx(t, test) + }) + } +} + +type ConcurrentTxTest struct { + name string +} + +var txTests = []ConcurrentTxTest{ + {name: "todo"}, +} + +func testConcurrentTx(t *testing.T, test ConcurrentTxTest) { + assert.True(t, true) +} diff --git a/integration-tests/transactions/go.mod b/integration-tests/transactions/go.mod new file mode 100644 index 0000000000..1df3fd375e --- /dev/null +++ b/integration-tests/transactions/go.mod @@ -0,0 +1,12 @@ +module github.com/dolthub/dolt/integration-tests/transactions + +go 1.17 + +require github.com/go-sql-driver/mysql v1.6.0 + +require ( + github.com/davecgh/go-spew v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.7.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect +) diff --git a/integration-tests/transactions/go.sum b/integration-tests/transactions/go.sum new file mode 100644 index 0000000000..4b6ee84190 --- /dev/null +++ b/integration-tests/transactions/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From fefe68f6df71e3c0afe159c55d7afe1ee99c0727 Mon Sep 17 00:00:00 2001 From: Dhruv Sringari Date: Fri, 4 Mar 2022 14:17:16 -0800 Subject: [PATCH 11/16] remove log redirection --- go/utils/remotesrv/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/utils/remotesrv/main.go b/go/utils/remotesrv/main.go index 823203faf5..40f3c3a415 100644 --- a/go/utils/remotesrv/main.go +++ b/go/utils/remotesrv/main.go @@ -38,8 +38,6 @@ func main() { httpHostParam := flag.String("http-host", "localhost", "host url that this command will assume.") flag.Parse() - log.SetOutput(os.Stdout) - if dirParam != nil && len(*dirParam) > 0 { err := os.Chdir(*dirParam) From 55c1cfdecd63ca3a9663b5d7ca13212ea61f393d Mon Sep 17 00:00:00 2001 From: Dhruv Sringari Date: Fri, 4 Mar 2022 14:49:23 -0800 Subject: [PATCH 12/16] Update go/utils/remotesrv/http.go Co-authored-by: Aaron Son --- go/utils/remotesrv/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/utils/remotesrv/http.go b/go/utils/remotesrv/http.go index 7bedd9e71d..3514940725 100644 --- a/go/utils/remotesrv/http.go +++ b/go/utils/remotesrv/http.go @@ -122,7 +122,7 @@ func readTableFile(logger func(string), org, repo, fileId string, respWr http.Re logger(fmt.Sprintf("wrote %d bytes", n)) - return -1 + return http.StatusOK } func writeTableFile(logger func(string), org, repo, fileId string, request *http.Request) int { From 0950ec814369fd04f6f1936943f27a0fe6619d48 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 4 Mar 2022 15:29:44 -0800 Subject: [PATCH 13/16] smoke test working --- .../transactions/concurrent_tx_test.go | 135 +++++++++++++++++- integration-tests/transactions/go.mod | 1 + integration-tests/transactions/go.sum | 2 + 3 files changed, 135 insertions(+), 3 deletions(-) diff --git a/integration-tests/transactions/concurrent_tx_test.go b/integration-tests/transactions/concurrent_tx_test.go index a0ea42f259..aadc661836 100644 --- a/integration-tests/transactions/concurrent_tx_test.go +++ b/integration-tests/transactions/concurrent_tx_test.go @@ -15,11 +15,25 @@ package transactions import ( + "fmt" "testing" "github.com/stretchr/testify/assert" + + "github.com/stretchr/testify/require" + + _ "github.com/go-sql-driver/mysql" + "github.com/gocraft/dbr/v2" ) +var defaultConfig = ServerConfig{ + database: "mysql", + host: "127.0.0.1", + port: 3316, + user: "root", + password: "toor", +} + func TestConcurrentTransactions(t *testing.T) { for _, test := range txTests { t.Run(test.name, func(t *testing.T) { @@ -29,13 +43,128 @@ func TestConcurrentTransactions(t *testing.T) { } type ConcurrentTxTest struct { - name string + name string + queries []concurrentQuery } +type concurrentQuery struct { + conn string + write string + query selector + expected []testRow +} + +type selector func(s *dbr.Session) *dbr.SelectStmt + +type testRow struct { + Pk, C0 int +} + +const ( + one = "one" + two = "two" +) + var txTests = []ConcurrentTxTest{ - {name: "todo"}, + { + name: "smoke test", + queries: []concurrentQuery{ + { + conn: one, + query: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("data") + }, + expected: []testRow{ + {1, 1}, + {2, 2}, + {3, 3}, + }, + }, + }, + }, +} + +func setupCommon(sess *dbr.Session) (err error) { + queries := []string{ + "CREATE DATABASE IF NOT EXISTS tx;", + "USE tx;", + "CREATE TABLE data (pk int primary key, c0 int);", + "INSERT INTO data VALUES (1,1),(2,2),(3,3);", + } + + for _, q := range queries { + if _, err = sess.Exec(q); err != nil { + return + } + } + return } func testConcurrentTx(t *testing.T, test ConcurrentTxTest) { - assert.True(t, true) + conns, err := createNamedConnections(defaultConfig, one, two) + require.NoError(t, err) + defer func() { require.NoError(t, closeNamedConnections(conns)) }() + + err = setupCommon(conns[one]) + defer func() { require.NoError(t, teardownCommon(conns[one])) }() + + for _, q := range test.queries { + conn := conns[q.conn] + if q.write != "" { + _, err = conn.Query(q.write) + require.NoError(t, err) + } + + var actual []testRow + _, err = q.query(conn).Load(&actual) + require.NoError(t, err) + assert.Equal(t, q.expected, actual) + } +} + +func teardownCommon(sess *dbr.Session) (err error) { + _, err = sess.Exec("DROP DATABASE tx;") + return +} + +type ServerConfig struct { + database string + host string + port int + user string + password string +} + +type namedConnections map[string]*dbr.Session + +// ConnectionString returns a Data Source Name (DSN) to be used by go clients for connecting to a running server. +func ConnectionString(config ServerConfig) string { + return fmt.Sprintf("%v:%v@tcp(%v:%v)/%s", + config.user, + config.password, + config.host, + config.port, + config.database, + ) +} + +func createNamedConnections(config ServerConfig, names ...string) (nc namedConnections, err error) { + nc = make(namedConnections, len(names)) + for _, name := range names { + var c *dbr.Connection + if c, err = dbr.Open("mysql", ConnectionString(config), nil); err != nil { + return nil, err + } + nc[name] = c.NewSession(nil) + } + return +} + +func closeNamedConnections(nc namedConnections) (err error) { + for _, conn := range nc { + if err = conn.Close(); err != nil { + return + } + } + return } diff --git a/integration-tests/transactions/go.mod b/integration-tests/transactions/go.mod index 1df3fd375e..7bc4d414d2 100644 --- a/integration-tests/transactions/go.mod +++ b/integration-tests/transactions/go.mod @@ -6,6 +6,7 @@ require github.com/go-sql-driver/mysql v1.6.0 require ( github.com/davecgh/go-spew v1.1.0 // indirect + github.com/gocraft/dbr/v2 v2.7.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/testify v1.7.0 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect diff --git a/integration-tests/transactions/go.sum b/integration-tests/transactions/go.sum index 4b6ee84190..5c53f6164d 100644 --- a/integration-tests/transactions/go.sum +++ b/integration-tests/transactions/go.sum @@ -2,6 +2,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/gocraft/dbr/v2 v2.7.3 h1:5/PTRiBkdD2FoHpnrCMoEUw5Wf/Cl3l3PjJ02Wm+pwM= +github.com/gocraft/dbr/v2 v2.7.3/go.mod h1:8IH98S8M8J0JSEiYk0MPH26ZDUKemiQ/GvmXL5jo+Uw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 91a3476d52e7d07191c2194591fe34575e3f5f06 Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 4 Mar 2022 15:56:55 -0800 Subject: [PATCH 14/16] added concurrent insert test --- .../transactions/concurrent_tx_test.go | 88 +++++++++++++++++-- 1 file changed, 79 insertions(+), 9 deletions(-) diff --git a/integration-tests/transactions/concurrent_tx_test.go b/integration-tests/transactions/concurrent_tx_test.go index aadc661836..64ca78d981 100644 --- a/integration-tests/transactions/concurrent_tx_test.go +++ b/integration-tests/transactions/concurrent_tx_test.go @@ -16,6 +16,7 @@ package transactions import ( "fmt" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -35,8 +36,11 @@ var defaultConfig = ServerConfig{ } func TestConcurrentTransactions(t *testing.T) { + sequential := &sync.Mutex{} for _, test := range txTests { t.Run(test.name, func(t *testing.T) { + sequential.Lock() + defer sequential.Unlock() testConcurrentTx(t, test) }) } @@ -48,10 +52,10 @@ type ConcurrentTxTest struct { } type concurrentQuery struct { - conn string - write string - query selector - expected []testRow + conn string + query string + assertion selector + expected []testRow } type selector func(s *dbr.Session) *dbr.SelectStmt @@ -71,8 +75,8 @@ var txTests = []ConcurrentTxTest{ queries: []concurrentQuery{ { conn: one, - query: func(s *dbr.Session) *dbr.SelectStmt { - return s.Select("*").From("data") + assertion: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("tx.data") }, expected: []testRow{ {1, 1}, @@ -82,10 +86,72 @@ var txTests = []ConcurrentTxTest{ }, }, }, + { + name: "concurrent transactions", + queries: []concurrentQuery{ + { + conn: one, + query: "BEGIN;", + }, + { + conn: two, + query: "BEGIN;", + }, + { + conn: two, + assertion: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("tx.data") + }, + expected: []testRow{ + {1, 1}, {2, 2}, {3, 3}, + }, + }, + { + conn: one, + query: "INSERT INTO tx.data VALUES (4,4)", + }, + { + conn: two, + assertion: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("tx.data") + }, + expected: []testRow{ + {1, 1}, {2, 2}, {3, 3}, + }, + }, + { + conn: one, + query: "COMMIT", + }, + { + conn: two, + assertion: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("tx.data") + }, + expected: []testRow{ + {1, 1}, {2, 2}, {3, 3}, + }, + }, + { + conn: two, + query: "COMMIT", + }, + { + conn: two, + assertion: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("tx.data") + }, + expected: []testRow{ + {1, 1}, {2, 2}, {3, 3}, {4, 4}, + }, + }, + }, + }, } func setupCommon(sess *dbr.Session) (err error) { queries := []string{ + "DROP DATABASE IF EXISTS tx;", "CREATE DATABASE IF NOT EXISTS tx;", "USE tx;", "CREATE TABLE data (pk int primary key, c0 int);", @@ -110,13 +176,17 @@ func testConcurrentTx(t *testing.T, test ConcurrentTxTest) { for _, q := range test.queries { conn := conns[q.conn] - if q.write != "" { - _, err = conn.Query(q.write) + if q.query != "" { + _, err = conn.Exec(q.query) require.NoError(t, err) } + if q.assertion == nil { + continue + } + var actual []testRow - _, err = q.query(conn).Load(&actual) + _, err = q.assertion(conn).Load(&actual) require.NoError(t, err) assert.Equal(t, q.expected, actual) } From 5bcec82c9d2bbba721f2a3b9420224298387ebad Mon Sep 17 00:00:00 2001 From: Andy Arthur Date: Fri, 4 Mar 2022 16:08:39 -0800 Subject: [PATCH 15/16] added concurrent update test --- .../transactions/concurrent_tx_test.go | 71 +++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/integration-tests/transactions/concurrent_tx_test.go b/integration-tests/transactions/concurrent_tx_test.go index 64ca78d981..2f5c6512e0 100644 --- a/integration-tests/transactions/concurrent_tx_test.go +++ b/integration-tests/transactions/concurrent_tx_test.go @@ -19,12 +19,10 @@ import ( "sync" "testing" - "github.com/stretchr/testify/assert" - - "github.com/stretchr/testify/require" - _ "github.com/go-sql-driver/mysql" "github.com/gocraft/dbr/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var defaultConfig = ServerConfig{ @@ -147,6 +145,71 @@ var txTests = []ConcurrentTxTest{ }, }, }, + { + name: "concurrent updates", + queries: []concurrentQuery{ + { + conn: one, + query: "BEGIN;", + }, + { + conn: two, + query: "BEGIN;", + }, + { + conn: two, + assertion: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("tx.data").Where("pk = 1") + }, + expected: []testRow{ + {1, 1}, + }, + }, + { + conn: one, + query: "UPDATE tx.data SET c0 = c0 + 10 WHERE pk = 1;", + }, + { + conn: two, + assertion: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("tx.data").Where("pk = 1") + }, + expected: []testRow{ + {1, 1}, + }, + }, + { + conn: one, + query: "COMMIT", + }, + { + conn: two, + assertion: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("tx.data").Where("pk = 1") + }, + expected: []testRow{ + {1, 1}, + }, + }, + { + conn: two, + query: "UPDATE tx.data SET c0 = c0 + 10 WHERE pk = 1;", + }, + { + conn: two, + assertion: func(s *dbr.Session) *dbr.SelectStmt { + return s.Select("*").From("tx.data").Where("pk = 1") + }, + expected: []testRow{ + {1, 21}, + }, + }, + { + conn: two, + query: "COMMIT", + }, + }, + }, } func setupCommon(sess *dbr.Session) (err error) { From 6b13ae0a6129935a6fc6fa8a2dd05aa178eb98e1 Mon Sep 17 00:00:00 2001 From: Vinai Rachakonda Date: Sat, 5 Mar 2022 18:55:41 -0800 Subject: [PATCH 16/16] Edit Stats display in Import and fix stats computation (#2901) --- go/cmd/dolt/commands/tblcmds/import.go | 4 +- go/go.mod | 2 +- .../doltcore/mvdata/engine_table_writer.go | 21 ++++----- .../bats/import-create-tables.bats | 2 +- .../bats/import-replace-tables.bats | 2 +- .../bats/import-update-tables.bats | 43 ++++++++++++++++--- 6 files changed, 53 insertions(+), 21 deletions(-) diff --git a/go/cmd/dolt/commands/tblcmds/import.go b/go/cmd/dolt/commands/tblcmds/import.go index 69d7dcaaf5..622cdbb352 100644 --- a/go/cmd/dolt/commands/tblcmds/import.go +++ b/go/cmd/dolt/commands/tblcmds/import.go @@ -26,6 +26,7 @@ import ( "github.com/dolthub/go-mysql-server/sql" "github.com/fatih/color" "golang.org/x/sync/errgroup" + "golang.org/x/text/message" "github.com/dolthub/dolt/go/cmd/dolt/cli" "github.com/dolthub/dolt/go/cmd/dolt/commands" @@ -422,7 +423,8 @@ var displayStrLen int func importStatsCB(stats types.AppliedEditStats) { noEffect := stats.NonExistentDeletes + stats.SameVal total := noEffect + stats.Modifications + stats.Additions - displayStr := fmt.Sprintf("Rows Processed: %d, Additions: %d, Modifications: %d, Had No Effect: %d", total, stats.Additions, stats.Modifications, noEffect) + p := message.NewPrinter(message.MatchLanguage("en")) // adds commas + displayStr := p.Sprintf("Rows Processed: %d, Additions: %d, Modifications: %d, Had No Effect: %d", total, stats.Additions, stats.Modifications, noEffect) displayStrLen = cli.DeleteAndPrint(displayStrLen, displayStr) } diff --git a/go/go.mod b/go/go.mod index ab209d8f05..9d06bd960e 100644 --- a/go/go.mod +++ b/go/go.mod @@ -76,6 +76,7 @@ require ( github.com/shirou/gopsutil/v3 v3.22.1 github.com/xitongsys/parquet-go v1.6.1 github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371 + golang.org/x/text v0.3.7 ) require ( @@ -120,7 +121,6 @@ require ( golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/mod v0.5.1 // indirect golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect - golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.9 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go/libraries/doltcore/mvdata/engine_table_writer.go b/go/libraries/doltcore/mvdata/engine_table_writer.go index 9e41fa758d..9db7cafe88 100644 --- a/go/libraries/doltcore/mvdata/engine_table_writer.go +++ b/go/libraries/doltcore/mvdata/engine_table_writer.go @@ -93,12 +93,9 @@ func NewSqlEngineTableWriter(ctx context.Context, dEnv *env.DoltEnv, createTable return nil, err } - var doltCreateTableSchema sql.PrimaryKeySchema - if options.Operation == CreateOp { - doltCreateTableSchema, err = sqlutil.FromDoltSchema(options.TableToWriteTo, createTableSchema) - if err != nil { - return nil, err - } + doltCreateTableSchema, err := sqlutil.FromDoltSchema(options.TableToWriteTo, createTableSchema) + if err != nil { + return nil, err } doltRowOperationSchema, err := sqlutil.FromDoltSchema(options.TableToWriteTo, rowOperationSchema) @@ -183,11 +180,11 @@ func (s *SqlEngineTableWriter) WriteRows(ctx context.Context, inputChannel chan } // If the length of the row does not match the schema then we have an update operation. - if len(row) != len(s.rowOperationSchema.Schema) { + if len(row) != len(s.tableSchema.Schema) { oldRow := row[:len(row)/2] newRow := row[len(row)/2:] - if ok, err := oldRow.Equals(newRow, s.rowOperationSchema.Schema); err == nil { + if ok, err := oldRow.Equals(newRow, s.tableSchema.Schema); err == nil { if ok { s.stats.SameVal++ } else { @@ -208,11 +205,11 @@ func (s *SqlEngineTableWriter) WriteRows(ctx context.Context, inputChannel chan if err != nil { return err } + defer func() { - if err != nil { - iter.Close(s.sqlCtx) // save the error that should be propagated. - } else { - err = iter.Close(s.sqlCtx) + rerr := iter.Close(s.sqlCtx) + if err == nil { + err = rerr } }() diff --git a/integration-tests/bats/import-create-tables.bats b/integration-tests/bats/import-create-tables.bats index 8c4fc344e5..eab34c8f88 100755 --- a/integration-tests/bats/import-create-tables.bats +++ b/integration-tests/bats/import-create-tables.bats @@ -698,7 +698,7 @@ DELIM run dolt table import -s schema.sql -c keyless data.csv [ "$status" -eq 0 ] - [[ "$output" =~ "Rows Processed: 1, Additions: 0, Modifications: 1, Had No Effect: 0" ]] || false + [[ "$output" =~ "Rows Processed: 1, Additions: 1, Modifications: 0, Had No Effect: 0" ]] || false [[ "$output" =~ "Import completed successfully." ]] || false run dolt sql -r csv -q "select * from keyless" diff --git a/integration-tests/bats/import-replace-tables.bats b/integration-tests/bats/import-replace-tables.bats index d6644d9856..bc13d70b13 100644 --- a/integration-tests/bats/import-replace-tables.bats +++ b/integration-tests/bats/import-replace-tables.bats @@ -335,7 +335,7 @@ DELIM run dolt table import -r test 1pk5col-ints-updt.csv [ "$status" -eq 0 ] - [[ "$output" =~ "Rows Processed: 1, Additions: 0, Modifications: 1, Had No Effect: 0" ]] || false + [[ "$output" =~ "Rows Processed: 1, Additions: 1, Modifications: 0, Had No Effect: 0" ]] || false [[ "$output" =~ "Import completed successfully." ]] || false run dolt sql -r csv -q "select * from test" diff --git a/integration-tests/bats/import-update-tables.bats b/integration-tests/bats/import-update-tables.bats index 92f2f1830e..cae450cf5a 100644 --- a/integration-tests/bats/import-update-tables.bats +++ b/integration-tests/bats/import-update-tables.bats @@ -49,7 +49,7 @@ SQL cat < check-constraint-sch.sql CREATE TABLE persons ( - ID int NOT NULL, + ID int PRIMARY KEY, LastName varchar(255) NOT NULL, FirstName varchar(255), Age int CHECK (Age>=18) @@ -209,7 +209,6 @@ CREATE TABLE employees ( ); SQL run dolt table import -u employees `batshelper employees-tbl-schema-unordered.json` - echo "$output" [ "$status" -eq 0 ] [[ "$output" =~ "Rows Processed: 3, Additions: 3, Modifications: 0, Had No Effect: 0" ]] || false [[ "$output" =~ "Import completed successfully." ]] || false @@ -532,7 +531,7 @@ DELIM run dolt table import -u test 1pk5col-ints-updt.csv [ "$status" -eq 0 ] - [[ "$output" =~ "Rows Processed: 1, Additions: 0, Modifications: 1, Had No Effect: 0" ]] || false + [[ "$output" =~ "Rows Processed: 1, Additions: 1, Modifications: 0, Had No Effect: 0" ]] || false [[ "$output" =~ "Import completed successfully." ]] || false run dolt sql -r csv -q "select * from test" @@ -552,7 +551,7 @@ DELIM run dolt table import -u test 1pk5col-ints-updt.csv [ "$status" -eq 0 ] - [[ "$output" =~ "Rows Processed: 1, Additions: 0, Modifications: 1, Had No Effect: 0" ]] || false + [[ "$output" =~ "Rows Processed: 1, Additions: 1, Modifications: 0, Had No Effect: 0" ]] || false [[ "$output" =~ "Import completed successfully." ]] || false run dolt sql -r csv -q "select * from test" @@ -653,7 +652,7 @@ DELIM run dolt table import -u keyless data.csv [ "$status" -eq 0 ] - [[ "$output" =~ "Rows Processed: 1, Additions: 0, Modifications: 1, Had No Effect: 0" ]] || false + [[ "$output" =~ "Rows Processed: 1, Additions: 1, Modifications: 0, Had No Effect: 0" ]] || false [[ "$output" =~ "Import completed successfully." ]] || false run dolt sql -r csv -q "select * from keyless order by c0, c1 DESC" @@ -682,4 +681,38 @@ DELIM ! [[ "$output" =~ "[4,little,doe,1]" ]] || false [[ "$output" =~ "Rows Processed: 1, Additions: 1, Modifications: 0, Had No Effect: 0" ]] || false [[ "$output" =~ "Import completed successfully." ]] || false + + run dolt sql -r csv -q "select * from persons" + [[ "$output" =~ "1,jon,doe,20" ]] || false +} + +@test "import-update-tables: large amounts of no effect rows" { + dolt sql -q "create table t(pk int primary key)" + dolt sql -q "alter table t add constraint cx CHECK (pk < 10)" + dolt sql -q "Insert into t values (1),(2),(3),(4),(5),(6),(7),(8),(9) " + + cat < file.csv +pk +1 +2 +3 +4 +5 +6 +10000 +DELIM + + run dolt table import -u --continue t file.csv + [ "$status" -eq 0 ] + [[ "$output" =~ "Rows Processed: 6, Additions: 0, Modifications: 0, Had No Effect: 6" ]] || false + [[ "$output" =~ "The following rows were skipped:" ]] || false + [[ "$output" =~ "[10000]" ]] || false + + run dolt sql -r csv -q "select * from t" + [[ "$output" =~ "1" ]] || false + [[ "$output" =~ "2" ]] || false + [[ "$output" =~ "3" ]] || false + [[ "$output" =~ "4" ]] || false + [[ "$output" =~ "5" ]] || false + [[ "$output" =~ "6" ]] || false }