diff --git a/.github/workflows/ci-compatibility-tests.yaml b/.github/workflows/ci-compatibility-tests.yaml index 3ed141d074..bd99788945 100644 --- a/.github/workflows/ci-compatibility-tests.yaml +++ b/.github/workflows/ci-compatibility-tests.yaml @@ -41,4 +41,4 @@ jobs: dolt config --global --add user.email 'actions@liquidata.co' - name: Test all run: ./runner.sh - working-directory: ./integration-tests/bats/compatibility + working-directory: ./integration-tests/compatibility diff --git a/go/libraries/doltcore/sqle/sqlselect_test.go b/go/libraries/doltcore/sqle/sqlselect_test.go index b2d9fa9082..7c7b61d9cd 100644 --- a/go/libraries/doltcore/sqle/sqlselect_test.go +++ b/go/libraries/doltcore/sqle/sqlselect_test.go @@ -711,7 +711,7 @@ var BasicSelectTests = []SelectTest{ Query: "select * from dolt_log", ExpectedRows: []sql.Row{ { - "1aqqa9u6kdaafn03p9dkskt444kvf69f", + "m8lrhp8bmfesmknc6d5iatmjbcjf17al", "billy bob", "bigbillieb@fake.horse", time.Date(1970, 1, 1, 0, 0, 0, 0, &time.Location{}), @@ -741,7 +741,7 @@ var BasicSelectTests = []SelectTest{ ExpectedRows: []sql.Row{ { "master", - "1aqqa9u6kdaafn03p9dkskt444kvf69f", + "m8lrhp8bmfesmknc6d5iatmjbcjf17al", "billy bob", "bigbillieb@fake.horse", time.Date(1970, 1, 1, 0, 0, 0, 0, &time.Location{}), "Initialize data repository", diff --git a/integration-tests/bats/compatibility/README.md b/integration-tests/bats/compatibility/README.md deleted file mode 100644 index 4de34a299b..0000000000 --- a/integration-tests/bats/compatibility/README.md +++ /dev/null @@ -1,23 +0,0 @@ -## Compatibility Tests - -These tests attempt to ensure forward and backward compatibility for Dolt versions. - -The testing script `runner.sh` checks out and builds older Dolt release versions to ensure that they can read data from -newer repositories and vice-versa. -A test directory `/env_test` is created outside of source control to preserve the environment across -`git checkout` commands. - -For each Dolt release version listed in `versions.txt`, `runner.sh` creates a legacy Dolt repository using the -`/test_files/setup_repo.sh` script in a directory named with the corresponding version. -An additional Dolt repository is created using Dolt built from the initial git branch. -BATS tests, located in `test_files/bats/`, are used to verify the forward and backward compatibility of all Dolt versions -and the repositories created with those versions. - -### Updating - -The BATS tests used to verify compatibility are inherently fragile. -Our primary integration tests in `/dolt/bats/` setup and tear down their environment for each test. -Because the tests rely on creating a repo with one version of Dolt and running BATS tests with a different version, -we cannot isolate their environment without building Dolt twice per test or setting up a different Dolt repo per test. -The initial version of these tests does all write operations in the `setup_repo.sh` script, and limits state modifications -within the BATS test to `dolt checkout` branch changes. Take care when editing the BATS tests to follow this pattern. \ No newline at end of file diff --git a/integration-tests/bats/compatibility/corona-virus-test.sh b/integration-tests/bats/compatibility/corona-virus-test.sh deleted file mode 100755 index 904393f299..0000000000 --- a/integration-tests/bats/compatibility/corona-virus-test.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -function download_release() { - ver=$1 - dirname=binaries/"$ver" - mkdir "$dirname" - basename=dolt-"$PLATFORM_TUPLE" - filename="$basename".tar.gz - filepath=binaries/"$ver"/"$filename" - url="https://github.com/dolthub/dolt/releases/download/$ver/$filename" - curl -L -o "$filepath" "$url" - cd "$dirname" && tar zxf "$filename" - echo "$dirname"/"$basename"/bin -} - -get_platform_tuple() { - OS=$(uname) - ARCH=$(uname -m) - if [ "$OS" != Linux -a "$OS" != Darwin ]; then - echo "tests only support linux or macOS." 1>&2 - exit 1 - fi - if [ "$ARCH" != x86_64 -a "$ARCH" != i386 -a "$ARCH" != i686 ]; then - echo "tests only support x86_64 or x86." 1>&2 - exit 1 - fi - if [ "$OS" == Linux ]; then - PLATFORM_TUPLE=linux - else - PLATFORM_TUPLE=darwin - fi - if [ "$ARCH" == x86_64 ]; then - PLATFORM_TUPLE="$PLATFORM_TUPLE"-amd64 - else - PLATFORM_TUPLE="$PLATFORM_TUPLE"-386 - fi - echo "$PLATFORM_TUPLE" -} - -PLATFORM_TUPLE=`get_platform_tuple` - -function export_tables() { - dv=`dolt version` - echo "exporting tables with dolt version $dv" - for table in \ - case_details \ - cases \ - characteristics_age \ - characteristics_case_severity \ - characteristics_comorbid_condition \ - characteristics_occupation \ - characteristics_onset_date_range \ - characteristics_province \ - characteristics_sex \ - characteristics_wuhan_exposed \ - dolt_query_catalog \ - dolt_schemas \ - places - do - dolt table export "$table" "$table$1.csv" - dolt sql -r csv -q "select * from $table" | sed 's/NULL//g' > "$table$1.sql.csv" - done -} - -function diff_tables() { - for table in \ - case_details \ - cases \ - characteristics_age \ - characteristics_case_severity \ - characteristics_comorbid_condition \ - characteristics_occupation \ - characteristics_onset_date_range \ - characteristics_province \ - characteristics_sex \ - characteristics_wuhan_exposed \ - dolt_query_catalog \ - dolt_schemas \ - places - do - diff "$table-pre.csv" "$table-post.csv" - diff "$table-pre.sql.csv" "$table-post.sql.csv" - done -} - -function cleanup() { - popd - rm -rf binaries - rm -rf "corona-virus" -} -mkdir binaries -trap cleanup "EXIT" - -bin=`download_release "v0.15.0"` -local_bin="`pwd`"/"$bin" - -PATH="$local_bin":"$PATH" dolt clone Liquidata/corona-virus -pushd "corona-virus" -PATH="$local_bin":"$PATH" export_tables "-pre" -time dolt migrate -export_tables "-post" -diff_tables -echo "success!" diff --git a/integration-tests/bats/compatibility/migration_test_files/migrate-push-pull.sh b/integration-tests/bats/compatibility/migration_test_files/migrate-push-pull.sh deleted file mode 100755 index 1e5424f47c..0000000000 --- a/integration-tests/bats/compatibility/migration_test_files/migrate-push-pull.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -function download_release() { - ver=$1 - dirname=binaries/"$ver" - mkdir "$dirname" - basename=dolt-"$PLATFORM_TUPLE" - filename="$basename".tar.gz - filepath=binaries/"$ver"/"$filename" - url="https://github.com/dolthub/dolt/releases/download/$ver/$filename" - curl -L -o "$filepath" "$url" - cd "$dirname" && tar zxf "$filename" - echo "$dirname"/"$basename"/bin -} - -get_platform_tuple() { - OS=$(uname) - ARCH=$(uname -m) - if [ "$OS" != Linux -a "$OS" != Darwin ]; then - echo "tests only support linux or macOS." 1>&2 - exit 1 - fi - if [ "$ARCH" != x86_64 -a "$ARCH" != i386 -a "$ARCH" != i686 ]; then - echo "tests only support x86_64 or x86." 1>&2 - exit 1 - fi - if [ "$OS" == Linux ]; then - PLATFORM_TUPLE=linux - else - PLATFORM_TUPLE=darwin - fi - if [ "$ARCH" == x86_64 ]; then - PLATFORM_TUPLE="$PLATFORM_TUPLE"-amd64 - else - PLATFORM_TUPLE="$PLATFORM_TUPLE"-386 - fi - echo "$PLATFORM_TUPLE" -} - -PLATFORM_TUPLE=`get_platform_tuple` - -setup_test_repos() { - ./setup_repo.sh "$1" - mkdir "$1-remote" - pushd "$1" - dolt remote add origin "file://../$1-remote" - # branches created in setup_repo.sh - dolt push origin init - dolt push origin master - dolt push origin other - popd - dolt clone "file://$1-remote" "$1-clone" -} - -TOP_DIR=`pwd` -function cleanup() { - pushd $TOP_DIR - rm -rf binaries - rm -rf repo* - popd -} -mkdir binaries -trap cleanup "EXIT" - -bin=`download_release "v0.15.2"` -PATH="`pwd`"/"$bin":"$PATH" setup_test_repos "repo" -TEST_REPO="repo" bats migrate.bats diff --git a/integration-tests/bats/compatibility/migration_test_files/migrate.bats b/integration-tests/bats/compatibility/migration_test_files/migrate.bats deleted file mode 100644 index 872aca0be7..0000000000 --- a/integration-tests/bats/compatibility/migration_test_files/migrate.bats +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bats -load $BATS_TEST_DIRNAME/helper/common.bash - -setup() { - setup_common -} - -teardown() { - teardown_common -} - -@test "dolt migrate --push & dolt migrate --pull" { - pushd "$TEST_REPO" - - run dolt migrate --pull - [ "$status" -ne "0" ] - [[ "$output" =~ "Local repo must be migrated before pulling, run 'dolt migrate'" ]] || false - - run dolt migrate --push - [ "$status" -ne "0" ] - [[ "$output" =~ "Local repo must be migrated before pushing, run 'dolt migrate'" ]] || false - - run dolt migrate - [ "$status" -eq "0" ] - [[ "$output" =~ "Migrating repository to the latest format" ]] || false - - run dolt migrate --pull - [ "$status" -ne "0" ] - [[ "$output" =~ "Remote origin has not been migrated" ]] || false - [[ "$output" =~ "Run 'dolt migrate --push origin' to push migration" ]] || false - - run dolt migrate --push - [ "$status" -eq "0" ] - - popd - pushd "$TEST_REPO-clone" - - run dolt migrate --pull - [ "$status" -ne "0" ] - [[ "$output" =~ "Local repo must be migrated before pulling, run 'dolt migrate'" ]] || false - - run dolt migrate --push - [ "$status" -ne "0" ] - [[ "$output" =~ "Local repo must be migrated before pushing, run 'dolt migrate'" ]] || false - - run dolt migrate - [ "$status" -eq "0" ] - [[ "$output" =~ "Migrating repository to the latest format" ]] || false - - run dolt migrate --push - [ "$status" -ne "0" ] - [[ "$output" =~ "Remote origin has been migrated" ]] || false - [[ "$output" =~ "Run 'dolt migrate --pull' to update refs" ]] || false - - run dolt migrate --pull - [ "$status" -eq "0" ] - - popd -} diff --git a/integration-tests/bats/compatibility/migration_test_files/setup_repo.sh b/integration-tests/bats/compatibility/migration_test_files/setup_repo.sh deleted file mode 100755 index 9ff3645981..0000000000 --- a/integration-tests/bats/compatibility/migration_test_files/setup_repo.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -mkdir "$1" -cd "$1" - -dolt init - -dolt sql < /dev/null -} - -teardown_common() { - echo "teardown" > /dev/null -} - -dolt config --global --add metrics.disabled true > /dev/null 2>&1 diff --git a/integration-tests/bats/compatibility/test_files/bats/helper/windows-compat.bash b/integration-tests/bats/compatibility/test_files/bats/helper/windows-compat.bash deleted file mode 100644 index 3a7049eccc..0000000000 --- a/integration-tests/bats/compatibility/test_files/bats/helper/windows-compat.bash +++ /dev/null @@ -1,24 +0,0 @@ -nativepath() { echo "$1"; } -nativevar() { eval export "$1"="$2"; } -skiponwindows() { :; } - -IS_WINDOWS=false - -if [ -d /mnt/c/Windows/System32 ]; then - IS_WINDOWS=true - if [ ! -d /mnt/c/batstmp ]; then - mkdir /mnt/c/batstmp - fi - BATS_TMPDIR=`TMPDIR=/mnt/c/batstmp mktemp -d -t dolt-bats-tests-XXXXXX` - export BATS_TMPDIR - nativepath() { - wslpath -w "$1" - } - nativevar() { - eval export "$1"="$2" - export WSLENV="$1$3" - } - skiponwindows() { - skip "$1" - } -fi diff --git a/integration-tests/bats/helper/common.bash b/integration-tests/bats/helper/common.bash index a9d5e15ef9..483ad96727 100644 --- a/integration-tests/bats/helper/common.bash +++ b/integration-tests/bats/helper/common.bash @@ -43,7 +43,7 @@ setup_no_dolt_init() { assert_feature_version() { run dolt version --feature - [[ "$output" =~ "feature version: 0" ]] || exit 1 + [[ "$output" =~ "feature version: 1" ]] || exit 1 } setup_common() { diff --git a/integration-tests/compatibility/README.md b/integration-tests/compatibility/README.md new file mode 100644 index 0000000000..5a1c0e62b7 --- /dev/null +++ b/integration-tests/compatibility/README.md @@ -0,0 +1,11 @@ +## Compatibility Tests + +These tests attempt to ensure forward and backward compatibility for Dolt versions. + +For each Dolt release version listed in `test_files/backward_compatible_versions.txt`, a legacy repository is created +using the corresponding release and populated with data from `test_files/setup_repo.sh`. +Then, using a Dolt client build at HEAD, a series of BATS tests are run against each legacy repository. + +To test forward compatibility, a repository is created and populated using Dolt built at HEAD. +For each Dolt release version listed in `test_files/forward_compatible_versions.txt`, the same BATS tests are run +against the repo created with Dolt at HEAD. diff --git a/integration-tests/bats/compatibility/runner.sh b/integration-tests/compatibility/runner.sh similarity index 54% rename from integration-tests/bats/compatibility/runner.sh rename to integration-tests/compatibility/runner.sh index 603e94f75c..6cc28c2661 100755 --- a/integration-tests/bats/compatibility/runner.sh +++ b/integration-tests/compatibility/runner.sh @@ -41,10 +41,6 @@ get_platform_tuple() { PLATFORM_TUPLE=`get_platform_tuple` -function list_dolt_versions() { - grep -v '^ *#' < test_files/dolt_versions.txt -} - function cleanup() { rm -rf repos binaries } @@ -56,21 +52,56 @@ function setup_repo() { ./test_files/setup_repo.sh "$dir" } -setup_repo HEAD -function test_dolt_version() { +# +# Backward Compatibility +# + +function list_backward_compatible_versions() { + grep -v '^ *#' < test_files/backward_compatible_versions.txt +} + +function test_backward_compatibility() { ver=$1 bin=`download_release "$ver"` - echo testing "$ver" at "$bin" + + # create a Dolt repository using version "$ver" PATH="`pwd`"/"$bin":"$PATH" setup_repo "$ver" - # Run the bats tests with old dolt version hitting repositories from new dolt version - PATH="`pwd`"/"$bin":"$PATH" REPO_DIR="`pwd`"/repos/HEAD bats ./test_files/bats - - # Run the bats tests with new dolt version hitting repositories from old dolt version + echo "Run the bats tests with current Dolt version hitting repositories from older Dolt version $ver" REPO_DIR="`pwd`"/repos/"$ver" bats ./test_files/bats } -list_dolt_versions | while IFS= read -r ver; do - test_dolt_version "$ver" +list_backward_compatible_versions | while IFS= read -r ver; do + test_backward_compatibility "$ver" done + + +# +# Forward Compatibility +# + +setup_repo HEAD + +function list_forward_compatible_versions() { + grep -v '^ *#' < test_files/forward_compatible_versions.txt +} + +function test_forward_compatibility() { + ver=$1 + bin=`download_release "$ver"` + + echo "Run the bats tests using older Dolt version $ver hitting repositories from the current Dolt version" + PATH="`pwd`"/"$bin":"$PATH" dolt version + PATH="`pwd`"/"$bin":"$PATH" REPO_DIR="`pwd`"/repos/HEAD bats ./test_files/bats +} + +if [ -s "test_files/forward_compatible_versions.txt" ]; then + list_forward_compatible_versions | while IFS= read -r ver; do + test_forward_compatibility "$ver" + done +fi + +# sanity check +echo "Run the bats tests using current Dolt version hitting repositories from the current Dolt version" +REPO_DIR="`pwd`"/repos/HEAD bats ./test_files/bats diff --git a/integration-tests/bats/compatibility/test_files/dolt_versions.txt b/integration-tests/compatibility/test_files/backward_compatible_versions.txt similarity index 58% rename from integration-tests/bats/compatibility/test_files/dolt_versions.txt rename to integration-tests/compatibility/test_files/backward_compatible_versions.txt index dd9561fcab..be98ca9076 100644 --- a/integration-tests/bats/compatibility/test_files/dolt_versions.txt +++ b/integration-tests/compatibility/test_files/backward_compatible_versions.txt @@ -1,6 +1,3 @@ -v0.13.0 -v0.14.0 -v0.15.0 v0.16.0 v0.17.0 v0.18.0 @@ -8,4 +5,8 @@ v0.19.0 v0.20.0 v0.21.0 v0.22.0 -v0.22.7 \ No newline at end of file +v0.22.7 +v0.23.0 +v0.24.0 +v0.25.0 +v0.25.1 \ No newline at end of file diff --git a/integration-tests/bats/compatibility/test_files/bats/compatibility.bats b/integration-tests/compatibility/test_files/bats/compatibility.bats similarity index 75% rename from integration-tests/bats/compatibility/test_files/bats/compatibility.bats rename to integration-tests/compatibility/test_files/bats/compatibility.bats index 264724835b..effb705d94 100755 --- a/integration-tests/bats/compatibility/test_files/bats/compatibility.bats +++ b/integration-tests/compatibility/test_files/bats/compatibility.bats @@ -14,9 +14,6 @@ teardown() { } @test "dolt version" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - run dolt version [ "$status" -eq 0 ] regex='dolt version [0-9]+.[0-9]+.[0-9]+' @@ -24,7 +21,6 @@ teardown() { } @test "dolt status" { - skip "These compatibility tests fail now due to a backwards incompatibility with the dolt_docs table. Before v0.16.0 dolt_docs used tags 0 and 1, and these values were hard coded in the logic that syncs the docs table with the file system." run dolt status [ "$status" -eq 0 ] [[ "$output" =~ "On branch master" ]] || false @@ -32,32 +28,22 @@ teardown() { } @test "dolt ls" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - run dolt ls [ "$status" -eq 0 ] [[ "${lines[0]}" =~ "Tables in working set:" ]] || false } @test "dolt branch" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - run dolt branch [ "$status" -eq 0 ] } @test "dolt diff" { - skip "These compatibility tests fail now due to a backwards incompatibility with the dolt_docs table. Before v0.16.0 dolt_docs used tags 0 and 1, and these values were hard coded in the logic that syncs the docs table with the file system." run dolt diff [ "$status" -eq 0 ] } @test "dolt schema show on branch init" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - dolt checkout init run dolt schema show abc [ "$status" -eq 0 ] @@ -73,9 +59,6 @@ teardown() { } @test "dolt sql 'select * from abc' on branch init" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - dolt checkout init run dolt sql -q 'select * from abc;' [ "$status" -eq 0 ] @@ -89,9 +72,6 @@ teardown() { } @test "dolt schema show on branch master" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - run dolt schema show abc [ "$status" -eq 0 ] output=`echo $output | tr '[:upper:]' '[:lower:]'` # lowercase the output @@ -107,9 +87,6 @@ teardown() { @test "dolt sql 'select * from abc' on branch master" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - run dolt sql -q 'select * from abc;' [ "$status" -eq 0 ] [[ "${lines[1]}" =~ "| pk | a | b | x | y |" ]] || false @@ -120,9 +97,6 @@ teardown() { } @test "dolt schema show on branch other" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - dolt checkout other run dolt schema show abc [ "$status" -eq 0 ] @@ -139,9 +113,6 @@ teardown() { } @test "dolt sql 'select * from abc' on branch other" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - dolt checkout other run dolt sql -q 'select * from abc;' [ "$status" -eq 0 ] @@ -155,9 +126,6 @@ teardown() { } @test "dolt table import" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - run dolt table import -c -pk=pk abc2 abc.csv [ "$status" -eq 0 ] [[ "$output" =~ "Import completed successfully." ]] || false @@ -167,9 +135,6 @@ teardown() { @test "dolt migrate no-data" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - dolt checkout no-data run dolt sql -q 'show tables;' [ "$status" -eq 0 ] @@ -180,9 +145,6 @@ teardown() { } @test "dolt_schemas" { - # this will fail for older dolt versions but BATS will swallow the error - run dolt migrate - run dolt sql -q "select * from dolt_schemas" [ "$status" -eq 0 ] [[ "${lines[1]}" =~ "| type | name | fragment |" ]] || false diff --git a/integration-tests/bats/compatibility/migration_test_files/helper/common.bash b/integration-tests/compatibility/test_files/bats/helper/common.bash similarity index 100% rename from integration-tests/bats/compatibility/migration_test_files/helper/common.bash rename to integration-tests/compatibility/test_files/bats/helper/common.bash diff --git a/integration-tests/bats/compatibility/migration_test_files/helper/windows-compat.bash b/integration-tests/compatibility/test_files/bats/helper/windows-compat.bash similarity index 100% rename from integration-tests/bats/compatibility/migration_test_files/helper/windows-compat.bash rename to integration-tests/compatibility/test_files/bats/helper/windows-compat.bash diff --git a/integration-tests/compatibility/test_files/forward_compatible_versions.txt b/integration-tests/compatibility/test_files/forward_compatible_versions.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integration-tests/bats/compatibility/test_files/setup_repo.sh b/integration-tests/compatibility/test_files/setup_repo.sh similarity index 100% rename from integration-tests/bats/compatibility/test_files/setup_repo.sh rename to integration-tests/compatibility/test_files/setup_repo.sh