Merge pull request #5066 from dolthub/andy/journal-rebase

[no-release-notes] Implement rebase for chunk journal

.github/workflows/ci-bats-unix.yaml

@@ -23,15 +23,22 @@ jobs:
       matrix:
         os: [ ubuntu-22.04, macos-latest ]
         dolt_fmt: [ "__DOLT__", "__LD_1__" ]
+        journal: [ "" ]
+        include:
+          - os: "ubuntu-22.04"
+            dolt_fmt: "__DOLT__"
+            journal: "true"
         exclude:
           - os: "macos-latest"
             dolt_fmt: "__LD_1__"
+            journal: ""
     env:
       use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }}
     steps:
       - name: Conditionally Set ENV VARS for AWS tests
         env:
           DOLT_FMT: ${{ matrix.dolt_fmt }}
+          DOLT_ENABLE_CHUNK_JOURNAL: ${{ matrix.journal }}
         run: |
           if [[ $use_credentials == true ]]; then
             echo "AWS_SDK_LOAD_CONFIG=1" >> $GITHUB_ENV

go/store/nbs/chunk_journal.go

@@ -241,6 +241,23 @@ func (j *chunkJournal) Update(ctx context.Context, lastLock addr, next manifestC
 		}
 	}
 
+	// if |next| has a different table file set, flush to |j.backing|
+	if j.contents.lock != next.lock || j.contents.gcGen != next.gcGen {
+		// todo: why is this necessary?
+		_, mc, err := j.backing.ParseIfExists(ctx, stats, nil)
+		if err != nil {
+			return manifestContents{}, err
+		}
+		lastLock = mc.lock
+
+		mc, err = j.backing.Update(ctx, lastLock, next, stats, nil)
+		if err != nil {
+			return manifestContents{}, err
+		} else if mc.lock != next.lock {
+			return manifestContents{}, errOptimisticLockFailedTables
+		}
+	}
+
 	if err := j.journal.writeRootHash(next.root); err != nil {
 		return manifestContents{}, err
 	}

integration-tests/bats/remotesrv.bats

@@ -15,8 +15,12 @@ setup() {
 
 teardown() {
     teardown_common
+    stop_remotesrv
+}
+
+stop_remotesrv() {
     if [ -n "$remotesrv_pid" ]; then
-        kill $remotesrv_pid
+        kill $remotesrv_pid || :
     fi
 }
 
@@ -40,10 +44,14 @@ teardown() {
     run dolt sql -q 'select count(*) from vals'
     [[ "$output" =~ "5" ]] || false
 
+    stop_remotesrv
     cd ../remote
     dolt sql -q 'insert into vals (i) values (6), (7), (8), (9), (10);'
     dolt commit -am 'add some vals'
 
+    remotesrv --http-port 1234 --repo-mode &
+    remotesrv_pid=$!
+
     cd ../repo1
     dolt pull
     run dolt sql -q 'select count(*) from vals;'
@@ -68,6 +76,7 @@ teardown() {
     dolt commit -am 'insert some values'
     dolt push origin main:main
 
+    stop_remotesrv
     cd ../remote
     # Have to reset the working set, which was not updated by the push...
     dolt reset --hard