Merge pull request #1541 from dolthub/andy/update-compat-tests

/integration-tests/compatibility: Split Compatibility tests into forward and backward

.github/workflows/ci-compatibility-tests.yaml

@@ -41,4 +41,4 @@ jobs:
           dolt config --global --add user.email 'actions@liquidata.co'
       - name: Test all
         run: ./runner.sh
-        working-directory: ./integration-tests/bats/compatibility
+        working-directory: ./integration-tests/compatibility

go/libraries/doltcore/sqle/sqlselect_test.go

@@ -711,7 +711,7 @@ var BasicSelectTests = []SelectTest{
 		Query: "select * from dolt_log",
 		ExpectedRows: []sql.Row{
 			{
-				"1aqqa9u6kdaafn03p9dkskt444kvf69f",
+				"m8lrhp8bmfesmknc6d5iatmjbcjf17al",
 				"billy bob",
 				"bigbillieb@fake.horse",
 				time.Date(1970, 1, 1, 0, 0, 0, 0, &time.Location{}),
@@ -741,7 +741,7 @@ var BasicSelectTests = []SelectTest{
 		ExpectedRows: []sql.Row{
 			{
 				"master",
-				"1aqqa9u6kdaafn03p9dkskt444kvf69f",
+				"m8lrhp8bmfesmknc6d5iatmjbcjf17al",
 				"billy bob", "bigbillieb@fake.horse",
 				time.Date(1970, 1, 1, 0, 0, 0, 0, &time.Location{}),
 				"Initialize data repository",

integration-tests/bats/compatibility/README.md

@@ -1,23 +0,0 @@
-## Compatibility Tests
-
-These tests attempt to ensure forward and backward compatibility for Dolt versions.
-
-The testing script `runner.sh` checks out and builds older Dolt release versions to ensure that they can read data from
-newer repositories and vice-versa.
-A test directory `/env_test` is created outside of source control to preserve the environment across
-`git checkout` commands.
-
-For each Dolt release version listed in `versions.txt`, `runner.sh` creates a legacy Dolt repository using the
-`/test_files/setup_repo.sh` script in a directory named with the corresponding version.
-An additional Dolt repository is created using Dolt built from the initial git branch.
-BATS tests, located in `test_files/bats/`, are used to verify the forward and backward compatibility of all Dolt versions
-and the repositories created with those versions.
-
-### Updating
-
-The BATS tests used to verify compatibility are inherently fragile.
-Our primary integration tests in `/dolt/bats/` setup and tear down their environment for each test.
-Because the tests rely on creating a repo with one version of Dolt and running BATS tests with a different version, 
-we cannot isolate their environment without building Dolt twice per test or setting up a different Dolt repo per test.
-The initial version of these tests does all write operations in the `setup_repo.sh` script, and limits state modifications
-within the BATS test to `dolt checkout` branch changes. Take care when editing the BATS tests to follow this pattern.

integration-tests/bats/compatibility/corona-virus-test.sh

@@ -1,105 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-function download_release() {
-  ver=$1
-  dirname=binaries/"$ver"
-  mkdir "$dirname"
-  basename=dolt-"$PLATFORM_TUPLE"
-  filename="$basename".tar.gz
-  filepath=binaries/"$ver"/"$filename"
-  url="https://github.com/dolthub/dolt/releases/download/$ver/$filename"
-  curl -L -o "$filepath" "$url"
-  cd "$dirname" && tar zxf "$filename"
-  echo "$dirname"/"$basename"/bin
-}
-
-get_platform_tuple() {
-  OS=$(uname)
-  ARCH=$(uname -m)
-  if [ "$OS" != Linux -a "$OS" != Darwin ]; then
-    echo "tests only support linux or macOS." 1>&2
-    exit 1
-  fi
-  if [ "$ARCH" != x86_64 -a "$ARCH" != i386 -a "$ARCH" != i686 ]; then
-    echo "tests only support x86_64 or x86." 1>&2
-    exit 1
-  fi
-  if [ "$OS" == Linux ]; then
-    PLATFORM_TUPLE=linux
-  else
-    PLATFORM_TUPLE=darwin
-  fi
-  if [ "$ARCH" == x86_64 ]; then
-    PLATFORM_TUPLE="$PLATFORM_TUPLE"-amd64
-  else
-    PLATFORM_TUPLE="$PLATFORM_TUPLE"-386
-  fi
-  echo "$PLATFORM_TUPLE"
-}
-
-PLATFORM_TUPLE=`get_platform_tuple`
-
-function export_tables() {
-  dv=`dolt version`
-  echo "exporting tables with dolt version $dv"
-  for table in \
-    case_details \
-    cases \
-    characteristics_age \
-    characteristics_case_severity \
-    characteristics_comorbid_condition \
-    characteristics_occupation \
-    characteristics_onset_date_range \
-    characteristics_province \
-    characteristics_sex \
-    characteristics_wuhan_exposed \
-    dolt_query_catalog \
-    dolt_schemas \
-    places
-  do
-    dolt table export "$table" "$table$1.csv"
-    dolt sql -r csv -q "select * from $table" | sed 's/NULL//g' > "$table$1.sql.csv"
-  done
-}
-
-function diff_tables() {
-  for table in \
-    case_details \
-    cases \
-    characteristics_age \
-    characteristics_case_severity \
-    characteristics_comorbid_condition \
-    characteristics_occupation \
-    characteristics_onset_date_range \
-    characteristics_province \
-    characteristics_sex \
-    characteristics_wuhan_exposed \
-    dolt_query_catalog \
-    dolt_schemas \
-    places
-  do
-    diff "$table-pre.csv" "$table-post.csv"
-    diff "$table-pre.sql.csv" "$table-post.sql.csv"
-  done
-}
-
-function cleanup() {
-  popd
-  rm -rf binaries
-  rm -rf "corona-virus"
-}
-mkdir binaries
-trap cleanup "EXIT"
-
-bin=`download_release "v0.15.0"`
-local_bin="`pwd`"/"$bin"
-
-PATH="$local_bin":"$PATH" dolt clone Liquidata/corona-virus
-pushd "corona-virus"
-PATH="$local_bin":"$PATH" export_tables "-pre"
-time dolt migrate
-export_tables "-post"
-diff_tables
-echo "success!"

integration-tests/bats/compatibility/migration_test_files/migrate-push-pull.sh

@@ -1,69 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-function download_release() {
-  ver=$1
-  dirname=binaries/"$ver"
-  mkdir "$dirname"
-  basename=dolt-"$PLATFORM_TUPLE"
-  filename="$basename".tar.gz
-  filepath=binaries/"$ver"/"$filename"
-  url="https://github.com/dolthub/dolt/releases/download/$ver/$filename"
-  curl -L -o "$filepath" "$url"
-  cd "$dirname" && tar zxf "$filename"
-  echo "$dirname"/"$basename"/bin
-}
-
-get_platform_tuple() {
-  OS=$(uname)
-  ARCH=$(uname -m)
-  if [ "$OS" != Linux -a "$OS" != Darwin ]; then
-    echo "tests only support linux or macOS." 1>&2
-    exit 1
-  fi
-  if [ "$ARCH" != x86_64 -a "$ARCH" != i386 -a "$ARCH" != i686 ]; then
-    echo "tests only support x86_64 or x86." 1>&2
-    exit 1
-  fi
-  if [ "$OS" == Linux ]; then
-    PLATFORM_TUPLE=linux
-  else
-    PLATFORM_TUPLE=darwin
-  fi
-  if [ "$ARCH" == x86_64 ]; then
-    PLATFORM_TUPLE="$PLATFORM_TUPLE"-amd64
-  else
-    PLATFORM_TUPLE="$PLATFORM_TUPLE"-386
-  fi
-  echo "$PLATFORM_TUPLE"
-}
-
-PLATFORM_TUPLE=`get_platform_tuple`
-
-setup_test_repos() {
-  ./setup_repo.sh "$1"
-  mkdir "$1-remote"
-  pushd "$1"
-  dolt remote add origin "file://../$1-remote"
-  # branches created in setup_repo.sh
-  dolt push origin init
-  dolt push origin master
-  dolt push origin other
-  popd
-  dolt clone "file://$1-remote" "$1-clone"
-}
-
-TOP_DIR=`pwd`
-function cleanup() {
-  pushd $TOP_DIR
-  rm -rf binaries
-  rm -rf repo*
-  popd
-}
-mkdir binaries
-trap cleanup "EXIT"
-
-bin=`download_release "v0.15.2"`
-PATH="`pwd`"/"$bin":"$PATH" setup_test_repos "repo"
-TEST_REPO="repo" bats migrate.bats

integration-tests/bats/compatibility/migration_test_files/migrate.bats

@@ -1,59 +0,0 @@
-#!/usr/bin/env bats
-load $BATS_TEST_DIRNAME/helper/common.bash
-
-setup() {
-    setup_common
-}
-
-teardown() {
-    teardown_common
-}
-
-@test "dolt migrate --push & dolt migrate --pull" {
-    pushd "$TEST_REPO"
-
-    run dolt migrate --pull
-    [ "$status" -ne "0" ]
-    [[ "$output" =~ "Local repo must be migrated before pulling, run 'dolt migrate'" ]] || false
-
-    run dolt migrate --push
-    [ "$status" -ne "0" ]
-    [[ "$output" =~ "Local repo must be migrated before pushing, run 'dolt migrate'" ]] || false
-
-    run dolt migrate
-    [ "$status" -eq "0" ]
-    [[ "$output" =~ "Migrating repository to the latest format" ]] || false
-
-    run dolt migrate --pull
-    [ "$status" -ne "0" ]
-    [[ "$output" =~ "Remote origin has not been migrated" ]] || false
-    [[ "$output" =~ "Run 'dolt migrate --push origin' to push migration" ]] || false
-
-    run dolt migrate --push
-    [ "$status" -eq "0" ]
-
-    popd
-    pushd "$TEST_REPO-clone"
-
-    run dolt migrate --pull
-    [ "$status" -ne "0" ]
-    [[ "$output" =~ "Local repo must be migrated before pulling, run 'dolt migrate'" ]] || false
-
-    run dolt migrate --push
-    [ "$status" -ne "0" ]
-    [[ "$output" =~ "Local repo must be migrated before pushing, run 'dolt migrate'" ]] || false
-
-    run dolt migrate
-    [ "$status" -eq "0" ]
-    [[ "$output" =~ "Migrating repository to the latest format" ]] || false
-
-    run dolt migrate --push
-    [ "$status" -ne "0" ]
-    [[ "$output" =~ "Remote origin has been migrated" ]] || false
-    [[ "$output" =~ "Run 'dolt migrate --pull' to update refs" ]] || false
-
-    run dolt migrate --pull
-    [ "$status" -eq "0" ]
-
-    popd
-}

integration-tests/bats/compatibility/migration_test_files/setup_repo.sh

@@ -1,67 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-mkdir "$1"
-cd "$1"
-
-dolt init
-
-dolt sql <<SQL
-CREATE TABLE abc (
-  pk BIGINT NOT NULL COMMENT 'tag:0',
-  a LONGTEXT COMMENT 'tag:100',
-  b DOUBLE COMMENT 'tag:101',
-  w BIGINT COMMENT 'tag:102',
-  x BIGINT COMMENT 'tag:103',
-  PRIMARY KEY (pk)
-);
-INSERT INTO abc VALUES (0, 'asdf', 1.1, 0, 0);
-INSERT INTO abc VALUES (1, 'asdf', 1.1, 0, 0);
-INSERT INTO abc VALUES (2, 'asdf', 1.1, 0, 0);
-SQL
-dolt add .
-dolt commit -m "initialized data"
-dolt branch init
-
-
-dolt branch other
-dolt sql <<SQL
-DELETE FROM abc WHERE pk=1;
-INSERT INTO abc VALUES (3, 'data', 1.1, 0, 0);
-ALTER TABLE abc DROP COLUMN w;
-ALTER TABLE abc ADD COLUMN y BIGINT COMMENT 'tag:104';
-SQL
-dolt add .
-dolt commit -m "made changes to master"
-
-dolt checkout other
-dolt sql <<SQL
-DELETE FROM abc WHERE pk=2;
-INSERT INTO abc VALUES (4, 'data', 1.1, 0, 0);
-ALTER TABLE abc DROP COLUMN x;
-ALTER TABLE abc ADD COLUMN z BIGINT COMMENT 'tag:105';
-SQL
-dolt add .
-dolt commit -m "made changes to other"
-
-dolt checkout master
-dolt table export abc abc.csv
-dolt schema export abc abc_schema.json
-
-# add info to the log
-echo
-echo "dolt status"
-dolt status
-
-echo
-echo "dolt branch"
-dolt branch
-
-echo
-echo "dolt schema show"
-dolt schema show
-
-echo
-echo "dolt sql -q 'select * from abc;'"
-dolt sql -q 'select * from abc;'

integration-tests/bats/compatibility/test_files/bats/helper/common.bash

@@ -1,16 +0,0 @@
-load helper/windows-compat
-
-if [ -z "$BATS_TMPDIR" ]; then
-    export BATS_TMPDIR=$HOME/batstmp/
-    mkdir $BATS_TMPDIR
-fi
-
-setup_common() {
-    echo "setup" > /dev/null
-}
-
-teardown_common() {
-    echo "teardown" > /dev/null
-}
-
-dolt config --global --add metrics.disabled true > /dev/null 2>&1

integration-tests/bats/compatibility/test_files/bats/helper/windows-compat.bash

@@ -1,24 +0,0 @@
-nativepath() { echo "$1"; }
-nativevar() { eval export "$1"="$2"; }
-skiponwindows() { :; }
-
-IS_WINDOWS=false
-
-if [ -d /mnt/c/Windows/System32 ]; then
-    IS_WINDOWS=true
-    if [ ! -d /mnt/c/batstmp ]; then
-        mkdir /mnt/c/batstmp
-    fi
-    BATS_TMPDIR=`TMPDIR=/mnt/c/batstmp mktemp -d -t dolt-bats-tests-XXXXXX`
-    export BATS_TMPDIR
-    nativepath() {
-        wslpath -w "$1"
-    }
-    nativevar() {
-        eval export "$1"="$2"
-        export WSLENV="$1$3"
-    }
-    skiponwindows() {
-        skip "$1"
-    }
-fi

integration-tests/bats/helper/common.bash

@@ -43,7 +43,7 @@ setup_no_dolt_init() {
 
 assert_feature_version() {
     run dolt version --feature
-    [[ "$output" =~ "feature version: 0" ]] || exit 1
+    [[ "$output" =~ "feature version: 1" ]] || exit 1
 }
 
 setup_common() {

integration-tests/compatibility/README.md

@@ -0,0 +1,11 @@
+## Compatibility Tests
+
+These tests attempt to ensure forward and backward compatibility for Dolt versions.
+
+For each Dolt release version listed in `test_files/backward_compatible_versions.txt`, a legacy repository is created 
+using the corresponding release and populated with data from `test_files/setup_repo.sh`.
+Then, using a Dolt client build at HEAD, a series of BATS tests are run against each legacy repository. 
+
+To test forward compatibility, a repository is created and populated using Dolt built at HEAD.
+For each Dolt release version listed in `test_files/forward_compatible_versions.txt`, the same BATS tests are run 
+against the repo created with Dolt at HEAD.

integration-tests/compatibility/runner.sh

@@ -41,10 +41,6 @@ get_platform_tuple() {
 
 PLATFORM_TUPLE=`get_platform_tuple`
 
-function list_dolt_versions() {
-  grep -v '^ *#' < test_files/dolt_versions.txt
-}
-
 function cleanup() {
   rm -rf repos binaries
 }
@@ -56,21 +52,56 @@ function setup_repo() {
   ./test_files/setup_repo.sh "$dir"
 }
 
-setup_repo HEAD
 
-function test_dolt_version() {
+#
+#   Backward Compatibility
+#
+
+function list_backward_compatible_versions() {
+  grep -v '^ *#' < test_files/backward_compatible_versions.txt
+}
+
+function test_backward_compatibility() {
   ver=$1
   bin=`download_release "$ver"`
-  echo testing "$ver" at "$bin"
+
+  # create a Dolt repository using version "$ver"
   PATH="`pwd`"/"$bin":"$PATH" setup_repo "$ver"
 
-  # Run the bats tests with old dolt version hitting repositories from new dolt version
-  PATH="`pwd`"/"$bin":"$PATH" REPO_DIR="`pwd`"/repos/HEAD bats ./test_files/bats
-
-  # Run the bats tests with new dolt version hitting repositories from old dolt version
+  echo "Run the bats tests with current Dolt version hitting repositories from older Dolt version $ver"
   REPO_DIR="`pwd`"/repos/"$ver" bats ./test_files/bats
 }
 
-list_dolt_versions | while IFS= read -r ver; do
-  test_dolt_version "$ver"
+list_backward_compatible_versions | while IFS= read -r ver; do
+  test_backward_compatibility "$ver"
 done
+
+
+#
+#   Forward Compatibility
+#
+
+setup_repo HEAD
+
+function list_forward_compatible_versions() {
+  grep -v '^ *#' < test_files/forward_compatible_versions.txt
+}
+
+function test_forward_compatibility() {
+  ver=$1
+  bin=`download_release "$ver"`
+
+  echo "Run the bats tests using older Dolt version $ver hitting repositories from the current Dolt version"
+  PATH="`pwd`"/"$bin":"$PATH" dolt version
+  PATH="`pwd`"/"$bin":"$PATH" REPO_DIR="`pwd`"/repos/HEAD bats ./test_files/bats
+}
+
+if [ -s "test_files/forward_compatible_versions.txt" ]; then
+    list_forward_compatible_versions | while IFS= read -r ver; do
+      test_forward_compatibility "$ver"
+    done
+fi
+
+# sanity check
+echo "Run the bats tests using current Dolt version hitting repositories from the current Dolt version"
+REPO_DIR="`pwd`"/repos/HEAD bats ./test_files/bats

integration-tests/compatibility/test_files/backward_compatible_versions.txt

@@ -1,6 +1,3 @@
-v0.13.0
-v0.14.0
-v0.15.0
 v0.16.0
 v0.17.0
 v0.18.0
@@ -8,4 +5,8 @@ v0.19.0
 v0.20.0
 v0.21.0
 v0.22.0
-v0.22.7
+v0.22.7
+v0.23.0
+v0.24.0
+v0.25.0
+v0.25.1

integration-tests/compatibility/test_files/bats/compatibility.bats

@@ -14,9 +14,6 @@ teardown() {
 }
 
 @test "dolt version" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     run dolt version
     [ "$status" -eq 0 ]
     regex='dolt version [0-9]+.[0-9]+.[0-9]+'
@@ -24,7 +21,6 @@ teardown() {
 }
 
 @test "dolt status" {
-    skip "These compatibility tests fail now due to a backwards incompatibility with the dolt_docs table. Before v0.16.0 dolt_docs used tags 0 and 1, and these values were hard coded in the logic that syncs the docs table with the file system."
     run dolt status
     [ "$status" -eq 0 ]
     [[ "$output" =~ "On branch master" ]] || false
@@ -32,32 +28,22 @@ teardown() {
 }
 
 @test "dolt ls" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     run dolt ls
     [ "$status" -eq 0 ]
     [[ "${lines[0]}" =~ "Tables in working set:" ]] || false
 }
 
 @test "dolt branch" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     run dolt branch
     [ "$status" -eq 0 ]
 }
 
 @test "dolt diff" {
-    skip "These compatibility tests fail now due to a backwards incompatibility with the dolt_docs table. Before v0.16.0 dolt_docs used tags 0 and 1, and these values were hard coded in the logic that syncs the docs table with the file system."
     run dolt diff
     [ "$status" -eq 0 ]
 }
 
 @test "dolt schema show on branch init" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     dolt checkout init
     run dolt schema show abc
     [ "$status" -eq 0 ]
@@ -73,9 +59,6 @@ teardown() {
 }
 
 @test "dolt sql 'select * from abc' on branch init" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     dolt checkout init
     run dolt sql -q 'select * from abc;'
     [ "$status" -eq 0 ]
@@ -89,9 +72,6 @@ teardown() {
 }
 
 @test "dolt schema show on branch master" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     run dolt schema show abc
     [ "$status" -eq 0 ]
     output=`echo $output | tr '[:upper:]' '[:lower:]'` # lowercase the output
@@ -107,9 +87,6 @@ teardown() {
 
 
 @test "dolt sql 'select * from abc' on branch master" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     run dolt sql -q 'select * from abc;'
     [ "$status" -eq 0 ]
     [[ "${lines[1]}" =~ "| pk | a    | b   | x | y   |" ]] || false
@@ -120,9 +97,6 @@ teardown() {
 }
 
 @test "dolt schema show on branch other" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     dolt checkout other
     run dolt schema show abc
     [ "$status" -eq 0 ]
@@ -139,9 +113,6 @@ teardown() {
 }
 
 @test "dolt sql 'select * from abc' on branch other" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     dolt checkout other
     run dolt sql -q 'select * from abc;'
     [ "$status" -eq 0 ]
@@ -155,9 +126,6 @@ teardown() {
 }
 
 @test "dolt table import" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     run dolt table import -c -pk=pk abc2 abc.csv
     [ "$status" -eq 0 ]
     [[ "$output" =~ "Import completed successfully." ]] || false
@@ -167,9 +135,6 @@ teardown() {
 
 
 @test "dolt migrate no-data" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     dolt checkout no-data
     run dolt sql -q 'show tables;'
     [ "$status" -eq 0 ]
@@ -180,9 +145,6 @@ teardown() {
 }
 
 @test "dolt_schemas" {
-    # this will fail for older dolt versions but BATS will swallow the error
-    run dolt migrate
-
     run dolt sql -q "select * from dolt_schemas"
     [ "$status" -eq 0 ]
     [[ "${lines[1]}" =~ "| type | name  | fragment             |" ]] || false