From 95b810651e6c5210d0631e84630f471d8dfaea1a Mon Sep 17 00:00:00 2001 From: Neil Macneale IV Date: Mon, 6 Oct 2025 15:29:55 -0700 Subject: [PATCH] Update Archive tests to account for new defaults --- integration-tests/bats/admin-conjoin.bats | 40 +++++++------- integration-tests/bats/archive.bats | 53 +++++++++---------- .../bats/helper/data-generation.bash | 12 ++++- 3 files changed, 53 insertions(+), 52 deletions(-) diff --git a/integration-tests/bats/admin-conjoin.bats b/integration-tests/bats/admin-conjoin.bats index ac2a2c2c78..82c9d11e66 100644 --- a/integration-tests/bats/admin-conjoin.bats +++ b/integration-tests/bats/admin-conjoin.bats @@ -70,9 +70,9 @@ get_chunk_count() { } @test "admin-conjoin: test conjoin with specific IDs" { - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" + dolt sql -q "$(mutations_and_gc_statement 0)" + dolt sql -q "$(mutations_and_gc_statement 0)" storage_ids=($(get_oldgen_table_ids)) [ "${#storage_ids[@]}" -eq 3 ] @@ -99,11 +99,11 @@ get_chunk_count() { @test "admin-conjoin: test conjoin with --all" { - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" + dolt sql -q "$(mutations_and_gc_statement 0)" + dolt sql -q "$(mutations_and_gc_statement 0)" + dolt sql -q "$(mutations_and_gc_statement 0)" + dolt sql -q "$(mutations_and_gc_statement 0)" storage_ids=($(get_oldgen_table_ids)) [ "${#storage_ids[@]}" -eq 5 ] @@ -126,14 +126,12 @@ get_chunk_count() { } @test "admin-conjoin: test conjoin with archive files" { - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 1)" + dolt sql -q "$(mutations_and_gc_statement 1)" + dolt sql -q "$(mutations_and_gc_statement 1)" - storage_ids=($(get_oldgen_table_ids)) + storage_ids=($(get_oldgen_archive_ids)) [ "${#storage_ids[@]}" -eq 3 ] - - dolt archive --purge # Get archive file IDs - these should be .darc files in oldgen archive_ids=($(get_oldgen_archive_ids)) @@ -162,16 +160,14 @@ get_chunk_count() { } @test "admin-conjoin: test conjoin with mixed file types" { - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" - - dolt archive --purge + dolt sql -q "$(mutations_and_gc_statement 1)" + dolt sql -q "$(mutations_and_gc_statement 1)" + dolt sql -q "$(mutations_and_gc_statement 1)" # Table files, not archives. - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" + dolt sql -q "$(mutations_and_gc_statement 0)" + dolt sql -q "$(mutations_and_gc_statement 0)" # sanity check that we have three table files and three archive files table_ids=($(get_oldgen_table_ids)) diff --git a/integration-tests/bats/archive.bats b/integration-tests/bats/archive.bats index 2c40fc5bcc..959173c005 100755 --- a/integration-tests/bats/archive.bats +++ b/integration-tests/bats/archive.bats @@ -26,7 +26,7 @@ teardown() { # bats test_tags=no_lambda @test "archive: too few chunks" { dolt sql -q "$(update_statement)" - dolt gc + dolt gc --archive-level 0 run dolt archive [ "$status" -eq 0 ] @@ -36,18 +36,19 @@ teardown() { } @test "archive: single archive oldgen" { - dolt sql -q "$(mutations_and_gc_statement)" - dolt archive + dolt sql -q "$(mutations_and_gc_statement 1)" - files=$(find . -name "*darc" | wc -l | sed 's/[ \t]//g') + find . -type f + + files=$(find .dolt/noms/oldgen -name "*darc" | wc -l | sed 's/[ \t]//g') [ "$files" -eq "1" ] # Ensure updates continue to work. dolt sql -q "$(update_statement)" } -@test "archive: single archive newgen" { - dolt sql -q "$(mutations_and_gc_statement)" +@test "archive: single archive cloned in newgen" { + dolt sql -q "$(mutations_and_gc_statement 0)" mkdir remote dolt remote add origin file://remote @@ -56,8 +57,6 @@ teardown() { dolt clone file://remote cloned cd cloned - dolt archive - files=$(find . -name "*darc" | wc -l | sed 's/[ \t]//g') [ "$files" -eq "1" ] @@ -67,7 +66,7 @@ teardown() { @test "archive: multi archive newgen then revert" { # Getting multiple table files in `newgen` is a little gross. - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" mkdir remote dolt remote add origin file://remote dolt push origin main @@ -79,7 +78,7 @@ teardown() { [ "$files" -eq "1" ] cd .. - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" dolt push origin main cd cloned @@ -96,13 +95,11 @@ teardown() { } @test "archive: multiple archives" { - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 1)" + dolt sql -q "$(mutations_and_gc_statement 1)" + dolt sql -q "$(mutations_and_gc_statement 1)" - dolt archive - - files=$(find . -name "*darc" | wc -l | sed 's/[ \t]//g') + files=$(find .dolt/noms/oldgen -name "*darc" | wc -l | sed 's/[ \t]//g') [ "$files" -eq "3" ] # dolt log --stat will load every single chunk. @@ -111,10 +108,10 @@ teardown() { } @test "archive: archive multiple times" { - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" dolt archive - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" dolt archive files=$(find . -name "*darc" | wc -l | sed 's/[ \t]//g') @@ -122,7 +119,7 @@ teardown() { } @test "archive: archive --revert (fast)" { - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" dolt archive dolt archive --revert @@ -132,7 +129,7 @@ teardown() { } @test "archive: archive --revert (rebuild)" { - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" dolt archive dolt archive --revert @@ -142,7 +139,7 @@ teardown() { } @test "archive: archive --purge" { - dolt sql -q "$(mutations_and_gc_statement)" + dolt sql -q "$(mutations_and_gc_statement 0)" # find impl differences by platform makes this a pain. tablefile=$(find .dolt/noms/oldgen -type f -print | awk -F/ 'length($NF) == 32 && $NF ~ /^[a-v0-9]{32}$/') @@ -318,7 +315,7 @@ teardown() { # Push, and enable the archive streamer. In the future this will be the default. dolt remote add origin http://localhost:$port/test-org/test-repo - DOLT_ARCHIVE_PULL_STREAMER=1 dolt push origin main + dolt push origin main cd .. @@ -330,7 +327,7 @@ teardown() { } @test "archive: large push remote without archive default produces no new archives" { - unset DOLT_ARCHIVE_PULL_STREAMER + export DOLT_ARCHIVE_PULL_STREAMER=0 mkdir -p remote/.dolt cp -R $BATS_TEST_DIRNAME/archive-test-repos/base/* remote/.dolt @@ -360,7 +357,7 @@ teardown() { } @test "archive: small push remote without archive default produces no new archives" { - unset DOLT_ARCHIVE_PULL_STREAMER + export DOLT_ARCHIVE_PULL_STREAMER=0 mkdir -p remote/.dolt cp -R $BATS_TEST_DIRNAME/archive-test-repos/base/* remote/.dolt @@ -390,7 +387,7 @@ teardown() { } @test "archive: large push remote with archive default produces new archive with converted snappy chunks" { - export DOLT_ARCHIVE_PULL_STREAMER=1 + unset DOLT_ARCHIVE_PULL_STREAMER mkdir -p remote/.dolt cp -R $BATS_TEST_DIRNAME/archive-test-repos/base/* remote/.dolt @@ -422,7 +419,7 @@ teardown() { } @test "archive: small push remote with archive default produces archive with snappy chunks" { - export DOLT_ARCHIVE_PULL_STREAMER=1 + unset DOLT_ARCHIVE_PULL_STREAMER mkdir -p remote/.dolt cp -R $BATS_TEST_DIRNAME/archive-test-repos/base/* remote/.dolt @@ -464,7 +461,7 @@ teardown() { cd ../ dolt remote add r1 http://localhost:$port/test-org/test-repo - DOLT_ARCHIVE_PULL_STREAMER=1 dolt fetch r1 + dolt fetch r1 run dolt admin storage [ $status -eq 0 ] @@ -482,7 +479,7 @@ teardown() { } @test "archive: fetch into empty database with archive disabled" { - unset DOLT_ARCHIVE_PULL_STREAMER + export DOLT_ARCHIVE_PULL_STREAMER=0 mkdir -p remote/.dolt cp -R $BATS_TEST_DIRNAME/archive-test-repos/base/* remote/.dolt diff --git a/integration-tests/bats/helper/data-generation.bash b/integration-tests/bats/helper/data-generation.bash index de074e11c4..144aba1126 100644 --- a/integration-tests/bats/helper/data-generation.bash +++ b/integration-tests/bats/helper/data-generation.bash @@ -52,16 +52,24 @@ UPDATE tbl SET guid = UUID() WHERE i >= @random_id LIMIT 1;" # A series of 10 update-and-commit-then-insert-and-commit pairs, followed by a dolt_gc call # +# The optional argument is the archive level for the GC call (0 or 1). Default is 1. +# # This is useful because we need at least 25 retained chunks to create an archive. mutations_and_gc_statement() { - query=`update_statement` + local level="${1:-1}" # default to 1 if no arg + if [[ "$level" != "0" && "$level" != "1" ]]; then + echo "Error: archive-level must be 0 or 1" + return + fi + + query=$(update_statement) for ((j=1; j<=9; j++)) do query="$query $(insert_statement)" query="$query $(update_statement)" done query="$query $(insert_statement)" - query="$query call dolt_gc();" + query="$query call dolt_gc('--archive-level=$level');" echo "$query" }