From 4f6cae5b3b8b04c225f212b228824f6d6bd46ba9 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 4 Nov 2021 15:53:42 -0700 Subject: [PATCH 1/9] fix fetch newline issue --- go/cmd/dolt/commands/push.go | 1 - go/libraries/doltcore/doltdb/doltdb.go | 2 +- go/libraries/doltcore/env/actions/remotes.go | 6 ++++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go/cmd/dolt/commands/push.go b/go/cmd/dolt/commands/push.go index ec73595a57..1099f8b7e1 100644 --- a/go/cmd/dolt/commands/push.go +++ b/go/cmd/dolt/commands/push.go @@ -303,7 +303,6 @@ func stopProgFuncs(cancel context.CancelFunc, wg *sync.WaitGroup, progChan chan close(pullerEventCh) wg.Wait() - cli.Println() } func bytesPerSec(bytes uint64, start time.Time) string { diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index 3433e79344..5690ba6129 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -1280,7 +1280,7 @@ func (ddb *DoltDB) PullChunks(ctx context.Context, tempDir string, srcDB *DoltDB puller, err := datas.NewPuller(ctx, tempDir, defaultChunksPerTF, srcDB.db, ddb.db, stRef.TargetHash(), pullerEventCh) if err == datas.ErrDBUpToDate { - return nil + return datas.ErrDBUpToDate } else if err != nil { return err } diff --git a/go/libraries/doltcore/env/actions/remotes.go b/go/libraries/doltcore/env/actions/remotes.go index 9b4c1f537d..a8b8d1fe5a 100644 --- a/go/libraries/doltcore/env/actions/remotes.go +++ b/go/libraries/doltcore/env/actions/remotes.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "github.com/dolthub/dolt/go/cmd/dolt/cli" "sync" eventsapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi/v1alpha1" @@ -352,6 +353,11 @@ func FetchRemoteBranch(ctx context.Context, tempTablesDir string, rem env.Remote wg, progChan, pullerEventCh := progStarter(newCtx) err = FetchCommit(ctx, tempTablesDir, srcDB, destDB, srcDBCommit, progChan, pullerEventCh) progStopper(cancelFunc, wg, progChan, pullerEventCh) + if err == nil { + cli.Println() + } else if err == datas.ErrDBUpToDate { + err = nil + } if err != nil { return nil, err From d06afc69be16362b72b73da1cbe8ba914b0a257f Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 4 Nov 2021 22:59:28 +0000 Subject: [PATCH 2/9] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/env/actions/remotes.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/env/actions/remotes.go b/go/libraries/doltcore/env/actions/remotes.go index a8b8d1fe5a..32764f0802 100644 --- a/go/libraries/doltcore/env/actions/remotes.go +++ b/go/libraries/doltcore/env/actions/remotes.go @@ -18,9 +18,10 @@ import ( "context" "errors" "fmt" - "github.com/dolthub/dolt/go/cmd/dolt/cli" "sync" + "github.com/dolthub/dolt/go/cmd/dolt/cli" + eventsapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi/v1alpha1" "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" From eaa2c426a55184068570f4efdcdd8503486a00f3 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Fri, 5 Nov 2021 11:44:02 -0700 Subject: [PATCH 3/9] add test for fetch new line issue --- integration-tests/bats/remotes.bats | 78 ++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index 675164b4ac..310caa9d2f 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -429,6 +429,82 @@ SQL [[ "$output" =~ "remotes/test-remote/poop" ]] || false } +@test "remotes: dolt fetch check for printing many newlines" { + # create_main_remote_branch + dolt remote add origin http://localhost:50051/test-org/test-repo + dolt sql -q 'create table test (id int primary key);' + dolt add . + dolt commit -m 'create test table.' + dolt push origin main:main + + # create_two_more_remote_branches + dolt checkout -b branch1 + dolt sql -q 'insert into test (id) values (1), (2), (3);' + dolt add . + dolt commit -m 'add some values to branch 1.' + dolt push --set-upstream origin branch1 + + dolt checkout -b branch2 + dolt sql -q 'insert into test (id) values (4), (5), (6);' + dolt add . + dolt commit -m 'add some values to branch 2.' + dolt push --set-upstream origin branch2 + + # create first clone + cd dolt-repo-clones + dolt clone http://localhost:50051/test-org/test-repo + cd test-repo + dolt status + run dolt status + [ "$status" -eq 0 ] + [[ "$output" =~ "On branch main" ]] || false + [[ "$output" =~ "nothing to commit, working tree clean" ]] || false + + cd ../.. + cd "dolt-repo-clones" + dolt clone http://localhost:50051/test-org/test-repo test-repo2 + cd test-repo2 + dolt status + run dolt status + [ "$status" -eq 0 ] + [[ "$output" =~ "On branch main" ]] || false + [[ "$output" =~ "nothing to commit, working tree clean" ]] || false + + # CHANGE 1: add more data to branch1 + dolt checkout -b branch1 remotes/origin/branch1 + dolt sql -q 'insert into test (id) values (100), (101), (102);' + dolt add . + dolt commit -m 'add more values to branch 1.' + dolt push --set-upstream origin branch1 + + # CHANGE 2: add more data to branch1 + dolt checkout -b branch2 remotes/origin/branch2 + dolt sql -q 'insert into test (id) values (103), (104), (105);' + dolt add . + dolt commit -m 'add more values to branch 2.' + dolt push --set-upstream origin branch2 + + # CHANGE 3: create_remote_branch "branch-three" + dolt checkout -b branch3 + dolt sql -q 'insert into test (id) values (7), (8), (9);' + dolt add . + dolt commit -m 'add some values to branch 3.' + dolt push --set-upstream origin branch3 + + # CHANGE 4: create_remote_branch "branch-four" + dolt checkout -b branch4 + dolt sql -q 'insert into test (id) values (10), (11), (12);' + dolt add . + dolt commit -m 'add some values to branch 4.' + dolt push --set-upstream origin branch4 + + cd .. + cd test-repo + run dolt fetch + [ "$status" -eq 0 ] + [ "${#lines[@]}" -eq 4 ] +} + @test "remotes: dolt fetch with docs" { # Initial commit of docs on remote echo "initial-license" > LICENSE.md @@ -457,7 +533,7 @@ SQL run cat README.md [ "$status" -eq 0 ] [[ "$output" =~ "initial-readme" ]] || false - # Change the docs + # Change the docs echo "dolt-repo-clones-license" > LICENSE.md echo "dolt-repo-clones-readme" > README.md dolt add . From acb5b5d2bbd3f98619fe42eeaa790d53c4e7cbcf Mon Sep 17 00:00:00 2001 From: jennifersp Date: Fri, 5 Nov 2021 11:46:29 -0700 Subject: [PATCH 4/9] update doltdb.go --- go/libraries/doltcore/doltdb/doltdb.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index 5690ba6129..5149b7ee2b 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -1278,10 +1278,7 @@ func (ddb *DoltDB) PushChunksForRefHash(ctx context.Context, tempDir string, src func (ddb *DoltDB) PullChunks(ctx context.Context, tempDir string, srcDB *DoltDB, stRef types.Ref, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error { if datas.CanUsePuller(srcDB.db) && datas.CanUsePuller(ddb.db) { puller, err := datas.NewPuller(ctx, tempDir, defaultChunksPerTF, srcDB.db, ddb.db, stRef.TargetHash(), pullerEventCh) - - if err == datas.ErrDBUpToDate { - return datas.ErrDBUpToDate - } else if err != nil { + if err != nil { return err } From 4a0d170091b2c4cc519ac39a8b577d7ef8c0eceb Mon Sep 17 00:00:00 2001 From: jennifersp <44716627+jennifersp@users.noreply.github.com> Date: Fri, 5 Nov 2021 13:46:26 -0700 Subject: [PATCH 5/9] Update remotes.bats --- integration-tests/bats/remotes.bats | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index 310caa9d2f..f48c13f3c4 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -477,21 +477,21 @@ SQL dolt commit -m 'add more values to branch 1.' dolt push --set-upstream origin branch1 - # CHANGE 2: add more data to branch1 + # CHANGE 2: add more data to branch2 dolt checkout -b branch2 remotes/origin/branch2 dolt sql -q 'insert into test (id) values (103), (104), (105);' dolt add . dolt commit -m 'add more values to branch 2.' dolt push --set-upstream origin branch2 - # CHANGE 3: create_remote_branch "branch-three" + # CHANGE 3: create_remote_branch "branch3" dolt checkout -b branch3 dolt sql -q 'insert into test (id) values (7), (8), (9);' dolt add . dolt commit -m 'add some values to branch 3.' dolt push --set-upstream origin branch3 - # CHANGE 4: create_remote_branch "branch-four" + # CHANGE 4: create_remote_branch "branch4" dolt checkout -b branch4 dolt sql -q 'insert into test (id) values (10), (11), (12);' dolt add . From 04b77f3d80675688625b68bce3093ba5951f40cf Mon Sep 17 00:00:00 2001 From: jennifersp Date: Fri, 5 Nov 2021 14:58:05 -0700 Subject: [PATCH 6/9] add fixes --- .../env/actions/commitwalk/commitwalk_test.go | 3 + go/libraries/doltcore/env/actions/remotes.go | 9 ++- integration-tests/bats/remotes.bats | 65 +++++++++++++++++-- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/go/libraries/doltcore/env/actions/commitwalk/commitwalk_test.go b/go/libraries/doltcore/env/actions/commitwalk/commitwalk_test.go index 05878ed8db..bf67b3ad70 100644 --- a/go/libraries/doltcore/env/actions/commitwalk/commitwalk_test.go +++ b/go/libraries/doltcore/env/actions/commitwalk/commitwalk_test.go @@ -231,6 +231,9 @@ func mustForkDB(t *testing.T, fromDB *doltdb.DoltDB, bn string, cm *doltdb.Commi } }() err = forkEnv.DoltDB.PullChunks(context.Background(), "", fromDB, stref, p1, p2) + if err == datas.ErrDBUpToDate { + err = nil + } require.NoError(t, err) err = forkEnv.DoltDB.SetHead(context.Background(), ref.NewBranchRef(bn), stref) require.NoError(t, err) diff --git a/go/libraries/doltcore/env/actions/remotes.go b/go/libraries/doltcore/env/actions/remotes.go index 32764f0802..142497c379 100644 --- a/go/libraries/doltcore/env/actions/remotes.go +++ b/go/libraries/doltcore/env/actions/remotes.go @@ -260,7 +260,7 @@ func FetchCommit(ctx context.Context, tempTablesDir string, srcDB, destDB *doltd return destDB.PullChunks(ctx, tempTablesDir, srcDB, stRef, progChan, pullerEventCh) } -// FetchCommit takes a fetches a commit tag and all underlying data from a remote source database to the local destination database. +// FetchTag takes a fetches a commit tag and all underlying data from a remote source database to the local destination database. func FetchTag(ctx context.Context, tempTableDir string, srcDB, destDB *doltdb.DoltDB, srcDBTag *doltdb.Tag, progChan chan datas.PullProgress, pullerEventCh chan datas.PullerEvent) error { stRef, err := srcDBTag.GetStRef() @@ -276,7 +276,7 @@ func Clone(ctx context.Context, srcDB, destDB *doltdb.DoltDB, eventCh chan<- dat return srcDB.Clone(ctx, destDB, eventCh) } -// fetchFollowTags fetches all tags from the source DB whose commits have already +// FetchFollowTags fetches all tags from the source DB whose commits have already // been fetched into the destination DB. // todo: potentially too expensive to iterate over all srcDB tags func FetchFollowTags(ctx context.Context, tempTableDir string, srcDB, destDB *doltdb.DoltDB, progStarter ProgStarter, progStopper ProgStopper) error { @@ -315,6 +315,11 @@ func FetchFollowTags(ctx context.Context, tempTableDir string, srcDB, destDB *do wg, progChan, pullerEventCh := progStarter(newCtx) err = FetchTag(ctx, tempTableDir, srcDB, destDB, tag, progChan, pullerEventCh) progStopper(cancelFunc, wg, progChan, pullerEventCh) + if err == nil { + cli.Println() + } else if err == datas.ErrDBUpToDate { + err = nil + } if err != nil { return true, err diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index f48c13f3c4..1dc161f1d9 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -429,21 +429,22 @@ SQL [[ "$output" =~ "remotes/test-remote/poop" ]] || false } -@test "remotes: dolt fetch check for printing many newlines" { - # create_main_remote_branch +@test "remotes: fetch output" { + # create main remote branch dolt remote add origin http://localhost:50051/test-org/test-repo dolt sql -q 'create table test (id int primary key);' dolt add . dolt commit -m 'create test table.' dolt push origin main:main - # create_two_more_remote_branches + # create remote branch "branch1" dolt checkout -b branch1 dolt sql -q 'insert into test (id) values (1), (2), (3);' dolt add . dolt commit -m 'add some values to branch 1.' dolt push --set-upstream origin branch1 + # create remote branch "branch2" dolt checkout -b branch2 dolt sql -q 'insert into test (id) values (4), (5), (6);' dolt add . @@ -461,6 +462,8 @@ SQL [[ "$output" =~ "nothing to commit, working tree clean" ]] || false cd ../.. + + # create second clone cd "dolt-repo-clones" dolt clone http://localhost:50051/test-org/test-repo test-repo2 cd test-repo2 @@ -484,14 +487,14 @@ SQL dolt commit -m 'add more values to branch 2.' dolt push --set-upstream origin branch2 - # CHANGE 3: create_remote_branch "branch3" + # CHANGE 3: create remote branch "branch3" dolt checkout -b branch3 dolt sql -q 'insert into test (id) values (7), (8), (9);' dolt add . dolt commit -m 'add some values to branch 3.' dolt push --set-upstream origin branch3 - # CHANGE 4: create_remote_branch "branch4" + # CHANGE 4: create remote branch "branch4" dolt checkout -b branch4 dolt sql -q 'insert into test (id) values (10), (11), (12);' dolt add . @@ -503,6 +506,58 @@ SQL run dolt fetch [ "$status" -eq 0 ] [ "${#lines[@]}" -eq 4 ] + [[ "${lines[0]}" != "\n" ]] || false + [[ "${lines[1]}" != "\n" ]] || false + [[ "${lines[2]}" != "\n" ]] || false + [[ "${lines[3]}" != "\n" ]] || false +} + +@test "remotes: fetch output with up-to-date branches" { + dolt remote add origin http://localhost:50051/test-org/test-repo + dolt sql -q 'create table test (id int primary key);' + dolt add . + dolt commit -m 'create test table.' + dolt push origin main:main + + dolt checkout -b branch1 + dolt sql -q 'insert into test (id) values (1), (2), (3);' + dolt add . + dolt commit -m 'add some values to branch 1.' + dolt push --set-upstream origin branch1 + + dolt checkout -b branch2 + dolt sql -q 'insert into test (id) values (4), (5), (6);' + dolt add . + dolt commit -m 'add some values to branch 2.' + dolt push --set-upstream origin branch2 + + dolt checkout -b branch3 + dolt sql -q 'insert into test (id) values (7), (8), (9);' + dolt add . + dolt commit -m 'add some values to branch 3.' + dolt push --set-upstream origin branch3 + + dolt checkout -b branch4 + dolt sql -q 'insert into test (id) values (10), (11), (12);' + dolt add . + dolt commit -m 'add some values to branch 4.' + dolt push --set-upstream origin branch4 + + # create first clone + cd dolt-repo-clones + dolt clone http://localhost:50051/test-org/test-repo + cd test-repo + dolt status + run dolt status + [ "$status" -eq 0 ] + [[ "$output" =~ "On branch main" ]] || false + [[ "$output" =~ "nothing to commit, working tree clean" ]] || false + + cd .. + cd test-repo + run dolt fetch + [ "$status" -eq 0 ] + [ "${#lines[@]}" -eq 0 ] } @test "remotes: dolt fetch with docs" { From 7532713f32a6c855552fe3e8033106d17bd8e685 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Fri, 5 Nov 2021 15:57:32 -0700 Subject: [PATCH 7/9] fix bats test --- integration-tests/bats/remotes.bats | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index 1dc161f1d9..0f2318e5c4 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -506,10 +506,11 @@ SQL run dolt fetch [ "$status" -eq 0 ] [ "${#lines[@]}" -eq 4 ] - [[ "${lines[0]}" != "\n" ]] || false - [[ "${lines[1]}" != "\n" ]] || false - [[ "${lines[2]}" != "\n" ]] || false - [[ "${lines[3]}" != "\n" ]] || false + [ "${lines[0]}" != "" ] + [ "${lines[1]}" != "" ] + [ "${lines[2]}" != "" ] + [ "${lines[3]}" != "" ] + } @test "remotes: fetch output with up-to-date branches" { @@ -557,7 +558,7 @@ SQL cd test-repo run dolt fetch [ "$status" -eq 0 ] - [ "${#lines[@]}" -eq 0 ] + [ "$output" = "" ] } @test "remotes: dolt fetch with docs" { From 105c8f8321cb49d1e072f783ea8e4f77adb4c2c5 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Mon, 8 Nov 2021 12:01:55 -0800 Subject: [PATCH 8/9] combine uptodate detch test --- integration-tests/bats/remotes.bats | 53 +++-------------------------- 1 file changed, 4 insertions(+), 49 deletions(-) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index 0f2318e5c4..ea34ea930c 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -461,6 +461,10 @@ SQL [[ "$output" =~ "On branch main" ]] || false [[ "$output" =~ "nothing to commit, working tree clean" ]] || false + run dolt fetch + [ "$status" -eq 0 ] + [ "$output" = "" ] + cd ../.. # create second clone @@ -510,55 +514,6 @@ SQL [ "${lines[1]}" != "" ] [ "${lines[2]}" != "" ] [ "${lines[3]}" != "" ] - -} - -@test "remotes: fetch output with up-to-date branches" { - dolt remote add origin http://localhost:50051/test-org/test-repo - dolt sql -q 'create table test (id int primary key);' - dolt add . - dolt commit -m 'create test table.' - dolt push origin main:main - - dolt checkout -b branch1 - dolt sql -q 'insert into test (id) values (1), (2), (3);' - dolt add . - dolt commit -m 'add some values to branch 1.' - dolt push --set-upstream origin branch1 - - dolt checkout -b branch2 - dolt sql -q 'insert into test (id) values (4), (5), (6);' - dolt add . - dolt commit -m 'add some values to branch 2.' - dolt push --set-upstream origin branch2 - - dolt checkout -b branch3 - dolt sql -q 'insert into test (id) values (7), (8), (9);' - dolt add . - dolt commit -m 'add some values to branch 3.' - dolt push --set-upstream origin branch3 - - dolt checkout -b branch4 - dolt sql -q 'insert into test (id) values (10), (11), (12);' - dolt add . - dolt commit -m 'add some values to branch 4.' - dolt push --set-upstream origin branch4 - - # create first clone - cd dolt-repo-clones - dolt clone http://localhost:50051/test-org/test-repo - cd test-repo - dolt status - run dolt status - [ "$status" -eq 0 ] - [[ "$output" =~ "On branch main" ]] || false - [[ "$output" =~ "nothing to commit, working tree clean" ]] || false - - cd .. - cd test-repo - run dolt fetch - [ "$status" -eq 0 ] - [ "$output" = "" ] } @test "remotes: dolt fetch with docs" { From 148484a2fab10cb0b56c47ae24e8d5b6c5d19b14 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Mon, 8 Nov 2021 12:14:18 -0800 Subject: [PATCH 9/9] update remotes.bats --- integration-tests/bats/remotes.bats | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/bats/remotes.bats b/integration-tests/bats/remotes.bats index ea34ea930c..7620d0c61f 100644 --- a/integration-tests/bats/remotes.bats +++ b/integration-tests/bats/remotes.bats @@ -461,10 +461,6 @@ SQL [[ "$output" =~ "On branch main" ]] || false [[ "$output" =~ "nothing to commit, working tree clean" ]] || false - run dolt fetch - [ "$status" -eq 0 ] - [ "$output" = "" ] - cd ../.. # create second clone @@ -514,6 +510,10 @@ SQL [ "${lines[1]}" != "" ] [ "${lines[2]}" != "" ] [ "${lines[3]}" != "" ] + + run dolt fetch + [ "$status" -eq 0 ] + [ "$output" = "" ] } @test "remotes: dolt fetch with docs" {