Add storage format information to dolt version and add new sql function dolt_storage_format(). Add dolt init --new-format (#4068)

* wip

* add bats

* remove unused

* add SELECT dolt_storage_format()

* [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh

* prepared test

* rename skip_nbf_dolt_1 to skip_nbf_dolt

* multidb mode should load databases based on DOLT_DEFAULT_BIN_FORMAT

If there is a database in ./.dolt then we use its format to search for other compatible formats.

If there is no database in ./.dolt then we use `DOLT_DEFAULT_BIN_FORMAT` to search for other compatible formats.

* fix bats

* fix check_prepared_tests.sh

* fix bats

Co-authored-by: druvv <druvv@users.noreply.github.com>
This commit is contained in:
Dhruv Sringari
2022-08-11 21:04:58 -07:00
committed by GitHub
parent 043e26a869
commit 241cf79018
24 changed files with 317 additions and 24 deletions

View File

@@ -98,6 +98,7 @@ const (
RemoteParam = "remote"
BranchParam = "branch"
TrackFlag = "track"
NewFormatFlag = "new-format"
)
const (

View File

@@ -75,6 +75,7 @@ func (cmd InitCmd) ArgParser() *argparser.ArgParser {
ap.SupportsString(emailParamName, "", "email", fmt.Sprintf("The email address used. If not provided will be taken from {{.EmphasisLeft}}%s{{.EmphasisRight}} in the global config.", env.UserEmailKey))
ap.SupportsString(cli.DateParam, "", "date", "Specify the date used in the initial commit. If not specified the current system time is used.")
ap.SupportsString(initBranchParamName, "b", "branch", fmt.Sprintf("The branch name used to initialize this database. If not provided will be taken from {{.EmphasisLeft}}%s{{.EmphasisRight}} in the global config. If unset, the default initialized branch will be named '%s'.", env.InitBranchName, env.DefaultInitBranch))
ap.SupportsFlag(cli.NewFormatFlag, "", fmt.Sprintf("Specify this flag to use the new storage format (%s).", types.Format_DOLT.VersionString()))
return ap
}
@@ -89,6 +90,10 @@ func (cmd InitCmd) Exec(ctx context.Context, commandStr string, args []string, d
return 1
}
if apr.Contains(cli.NewFormatFlag) {
types.Format_Default = types.Format_DOLT
}
name, _ := apr.GetValue(usernameParamName)
email, _ := apr.GetValue(emailParamName)
initBranch, _ := apr.GetValue(initBranchParamName)

View File

@@ -17,10 +17,10 @@ package commands
import (
"context"
"github.com/dolthub/dolt/go/cmd/dolt/errhand"
"github.com/dolthub/dolt/go/cmd/dolt/cli"
"github.com/dolthub/dolt/go/cmd/dolt/errhand"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dfunctions"
"github.com/dolthub/dolt/go/libraries/utils/argparser"
)
@@ -63,6 +63,15 @@ func (cmd VersionCmd) ArgParser() *argparser.ArgParser {
func (cmd VersionCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv) int {
cli.Println("dolt version", cmd.VersionStr)
if dEnv.HasDoltDir() && !cli.CheckEnvIsValid(dEnv) {
return 2
} else if dEnv.HasDoltDir() {
nbf := dEnv.DoltDB.Format()
cli.Printf("database storage format: %s\n", dfunctions.GetStorageFormatDisplayString(nbf))
} else {
cli.Println("no valid database in this directory")
}
usage := func() {}
ap := cmd.ArgParser()
apr := cli.ParseArgsOrDie(ap, args, usage)

View File

@@ -274,6 +274,8 @@ func MultiEnvForDirectory(
if dEnv.Valid() {
binFormat = dEnv.DoltDB.Format()
mrEnv.AddEnv(dbName, dEnv)
} else {
binFormat = types.Format_Default
}
// If there are other directories in the directory, try to load them as additional databases

View File

@@ -21,6 +21,7 @@ var DoltFunctions = []sql.Function{
sql.FunctionN{Name: CommitFuncName, Fn: NewCommitFunc},
sql.FunctionN{Name: MergeFuncName, Fn: NewMergeFunc},
sql.Function0{Name: VersionFuncName, Fn: NewVersion},
sql.Function0{Name: StorageFormatFuncName, Fn: NewStorageFormat},
sql.FunctionN{Name: DoltCommitFuncName, Fn: NewDoltCommitFunc},
sql.FunctionN{Name: DoltAddFuncName, Fn: NewDoltAddFunc},
sql.FunctionN{Name: DoltCleanFuncName, Fn: NewDoltCleanFunc},
@@ -42,6 +43,7 @@ var DoltFunctions = []sql.Function{
var DolthubApiFunctions = []sql.Function{
sql.Function1{Name: HashOfFuncName, Fn: NewHashOf},
sql.Function0{Name: VersionFuncName, Fn: NewVersion},
sql.Function0{Name: StorageFormatFuncName, Fn: NewStorageFormat},
sql.Function0{Name: ActiveBranchFuncName, Fn: NewActiveBranchFunc},
sql.Function2{Name: DoltMergeBaseFuncName, Fn: NewMergeBase},
}

View File

@@ -0,0 +1,92 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dfunctions
import (
"errors"
"fmt"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/store/types"
)
const StorageFormatFuncName = "dolt_storage_format"
type StorageFormat struct{}
// NewStorageFormat creates a new StorageFormat expression.
func NewStorageFormat() sql.Expression {
return &StorageFormat{}
}
// Children implements the Expression interface.
func (*StorageFormat) Children() []sql.Expression {
return nil
}
// Eval implements the Expression interface.
func (*StorageFormat) Eval(ctx *sql.Context, row sql.Row) (interface{}, error) {
sesh := dsess.DSessFromSess(ctx.Session)
if sesh.GetCurrentDatabase() == "" {
return nil, sql.ErrNoDatabaseSelected.New()
}
dbName := sesh.GetCurrentDatabase()
ddb, ok := sesh.GetDoltDB(ctx, dbName)
if !ok {
return nil, errors.New("failed to load underlying dolt db")
}
f := GetStorageFormatDisplayString(ddb.Format())
return f, nil
}
func GetStorageFormatDisplayString(format *types.NomsBinFormat) string {
if types.IsFormat_DOLT(format) {
return fmt.Sprintf("NEW ( %s )", format.VersionString())
} else {
return fmt.Sprintf("OLD ( %s )", format.VersionString())
}
}
// IsNullable implements the Expression interface.
func (*StorageFormat) IsNullable() bool {
return false
}
// Resolved implements the Expression interface.
func (*StorageFormat) Resolved() bool {
return true
}
// String implements the Stringer interface.
func (*StorageFormat) String() string {
return "DOLT_STORAGE_FORMAT"
}
// Type implements the Expression interface.
func (*StorageFormat) Type() sql.Type {
return sql.Text
}
// WithChildren implements the Expression interface.
func (v *StorageFormat) WithChildren(children ...sql.Expression) (sql.Expression, error) {
if len(children) != 0 {
return nil, sql.ErrInvalidChildrenNumber.New(v, len(children), 0)
}
return NewVersion(), nil
}

View File

@@ -46,7 +46,7 @@ var skipPrepared bool
// SkipPreparedsCount is used by the "ci-check-repo CI workflow
// as a reminder to consider prepareds when adding a new
// enginetest suite.
const SkipPreparedsCount = 106
const SkipPreparedsCount = 81
const skipPreparedFlag = "DOLT_SKIP_PREPARED_ENGINETESTS"
@@ -1311,6 +1311,35 @@ func TestDoltVerifyConstraints(t *testing.T) {
}
}
func TestDoltStorageFormat(t *testing.T) {
var expectedFormatString string
if types.IsFormat_DOLT(types.Format_Default) {
expectedFormatString = "NEW ( __DOLT__ )"
} else {
expectedFormatString = "OLD ( __LD_1__ )"
}
script := queries.ScriptTest{
Name: "dolt storage format function works",
Assertions: []queries.ScriptTestAssertion{
{
Query: "select dolt_storage_format()",
Expected: []sql.Row{{expectedFormatString}},
},
},
}
enginetest.TestScript(t, newDoltHarness(t), script)
}
func TestDoltStorageFormatPrepared(t *testing.T) {
var expectedFormatString string
if types.IsFormat_DOLT(types.Format_Default) {
expectedFormatString = "NEW ( __DOLT__ )"
} else {
expectedFormatString = "OLD ( __LD_1__ )"
}
enginetest.TestPreparedQuery(t, newDoltHarness(t), "SELECT dolt_storage_format()", []sql.Row{{expectedFormatString}}, nil)
}
var newFormatSkippedScripts = []string{
// Different query plans
"Partial indexes are used and return the expected result",

View File

@@ -8,11 +8,12 @@ cd $script_dir/../..
enginetest_path=libraries/doltcore/sqle/enginetest/dolt_engine_test.go
all_cnt=$(grep "func Test.*" "$enginetest_path" | wc -l | tr -dc '0-9')
prep_cnt=$(grep "func Test.*Prepared" "$enginetest_path" | wc -l | tr -dc '0-9')
test_cnt=$((all_cnt-prep_cnt))
skip_cnt=$(grep "SkipPreparedsCount" "$enginetest_path" | awk '{print $4}' | tr -dc '0-9')
expected="$((all_cnt-skip_cnt))"
expected="$((test_cnt-skip_cnt))"
if [[ "$expected" != "$prep_cnt" ]]; then
echo "Expected '$expected' TestPrepared enginetests in dolt_engine_test.go, found: '$prep_cnt'"
echo "Expected '$expected' enginetests with name 'Test.*Prepared' in dolt_engine_test.go, found: '$prep_cnt'"
echo "Either increment SkipPreparedsCount or add a prepared test for the new test suite"
exit 1
fi

View File

@@ -53,7 +53,7 @@ SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "Updating" ]] || false
[[ "$output" =~ "1 tables changed" ]] || false
skip_nbf_dolt_1 "needs merge stats"
skip_nbf_dolt "needs merge stats"
[[ "$output" =~ "1 rows modified" ]] || false
}
@@ -89,7 +89,7 @@ SQL
[[ "$output" =~ "Updating" ]] || false
[[ "$output" =~ "1 tables changed" ]] || false
[[ ! "$output" =~ "CONFLICT" ]] || false
skip_nbf_dolt_1 "needs merge stats"
skip_nbf_dolt "needs merge stats"
[[ "$output" =~ "1 rows modified" ]] || false
run dolt status
[[ "$output" =~ "All conflicts and constraint violations fixed" ]] || false
@@ -125,7 +125,7 @@ SQL
[[ "$output" =~ "Updating" ]] || false
[[ "$output" =~ "1 tables changed" ]] || false
[[ ! "$output" =~ "CONFLICT" ]] || false
skip_nbf_dolt_1 "needs merge stats"
skip_nbf_dolt "needs merge stats"
[[ "$output" =~ "1 rows modified" ]] || false
run dolt status
[[ "$output" =~ "All conflicts and constraint violations fixed" ]] || false
@@ -191,7 +191,7 @@ SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "Updating" ]] || false
[[ "$output" =~ "1 tables changed" ]] || false
skip_nbf_dolt_1 "needs merge stats"
skip_nbf_dolt "needs merge stats"
[[ "$output" =~ "1 rows added" ]] || false
[[ ! "$output" =~ "CONFLICT" ]] || false
}

View File

@@ -116,7 +116,7 @@ EOF
[[ "$output" =~ "$EXPECTED" ]] || false
# No data diff
skip_nbf_dolt_1 "Adding a column necessarily rewrites row values in new format"
skip_nbf_dolt "Adding a column necessarily rewrites row values in new format"
[ "${#lines[@]}" -eq 9 ]
run dolt sql -r csv -q "select * from test as of 'HEAD'"
@@ -283,7 +283,7 @@ EOF
[[ "$output" =~ "$EXPECTED" ]] || false
# no data diff
skip_nbf_dolt_1 "Adding a column necessarily rewrites row values in new format"
skip_nbf_dolt "Adding a column necessarily rewrites row values in new format"
[ "${#lines[@]}" -eq 11 ]
}

View File

@@ -164,7 +164,7 @@ setup_merge() {
}
@test "garbage_collection: leave merge commit" {
skip_nbf_dolt_1
skip_nbf_dolt
setup_merge
dolt merge other

View File

@@ -63,7 +63,7 @@ assert_feature_version() {
fi
}
skip_nbf_dolt_1() {
skip_nbf_dolt() {
if [ "$DOLT_DEFAULT_BIN_FORMAT" = "__DOLT__" ]; then
skip "skipping test for nomsBinFormat __DOLT__"
fi

View File

@@ -567,7 +567,7 @@ DELIM
# assert that we already collected garbage
BEFORE=$(du -c .dolt/noms/ | grep total | sed 's/[^0-9]*//g')
skip_nbf_dolt_1 "dolt gc not implemented"
skip_nbf_dolt "dolt gc not implemented"
dolt gc
AFTER=$(du -c .dolt/noms/ | grep total | sed 's/[^0-9]*//g')

View File

@@ -2361,7 +2361,7 @@ SQL
[ "$status" -eq "0" ]
[[ "$output" =~ "table,num_violations" ]] || false
skip_nbf_dolt_1 "__DOLT__ documents all matching rows for any unique key violations"
skip_nbf_dolt "__DOLT__ documents all matching rows for any unique key violations"
[[ "$output" =~ "onepk,2" ]] || false
}

View File

@@ -152,6 +152,82 @@ teardown() {
[ "$status" -eq 1 ]
}
@test "init: running init with the new format, creates a new format database" {
set_dolt_user "baz", "baz@bash.com"
run dolt init --new-format
[ $status -eq 0 ]
run dolt init
[ "$status" -eq 1 ]
run cut -d ":" -f 2 .dolt/noms/manifest
[ "$output" = "__DOLT__" ]
}
@test "init: initing a new database displays the correct version" {
set_dolt_user "baz", "baz@bash.com"
run dolt init --new-format
[ $status -eq 0 ]
run dolt version
[ $status -eq 0 ]
[[ $output =~ "database storage format: NEW ( __DOLT__ )" ]] || false
run dolt sql -q "SELECT dolt_storage_format();"
[[ $output =~ "NEW ( __DOLT__ )" ]] || false
}
@test "init: initing an old database displays the correct version" {
set_dolt_user "baz", "bazbash.com"
DOLT_DEFAULT_BIN_FORMAT="__LD_1__" dolt init
run dolt version
[ "$status" -eq 0 ]
[[ $output =~ "database storage format: OLD ( __LD_1__ )" ]] || false
run dolt sql -q "SELECT dolt_storage_format();"
[[ $output =~ "OLD ( __LD_1__ )" ]] || false
}
@test "init: get format in multiple database mode" {
orig_bin_format=$DOLT_DEFAULT_BIN_FORMAT
mkdir old_fmt
cd old_fmt
DOLT_DEFAULT_BIN_FORMAT="__LD_1__" dolt init
cd ..
mkdir new_fmt
cd new_fmt
DOLT_DEFAULT_BIN_FORMAT="__DOLT__" dolt init
cd ..
# New format db gets chosen automatically, as it is the only db loaded
export DOLT_DEFAULT_BIN_FORMAT="__DOLT__"
run dolt sql -q "SELECT dolt_storage_format()"
[ $status -eq 0 ]
[[ $output =~ "NEW ( __DOLT__ )" ]] || false
# Old format db gets chosen automatically, as it is the only db loaded
export DOLT_DEFAULT_BIN_FORMAT="__LD_1__"
run dolt sql -q "SELECT dolt_storage_format()"
[ $status -eq 0 ]
[[ $output =~ "OLD ( __LD_1__ )" ]] || false
export DOLT_DEFAULT_BIN_FORMAT=$orig_bin_format
}
@test "init: empty database folder displays no version" {
set_dolt_user "baz", "bazbash.com"
run dolt version
[ $status -eq 0 ]
[[ $output =~ "no valid database in this directory" ]]
}
assert_valid_repository () {
run dolt log
[ "$status" -eq 0 ]

View File

@@ -3,7 +3,7 @@ load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_common
skip_nbf_dolt_1 "works fine but formatting of json strings is different"
skip_nbf_dolt "works fine but formatting of json strings is different"
}
teardown() {

View File

@@ -785,7 +785,7 @@ SQL
# commit it so we can merge again
dolt commit -afm "committing merge conflicts"
skip_nbf_dolt_1 "behavior in new format diverges"
skip_nbf_dolt "behavior in new format diverges"
# Merge should fail due to conflict and previous conflict and violation state should be retained
run dolt merge other2

View File

@@ -2,7 +2,7 @@
load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
skip_nbf_dolt_1
skip_nbf_dolt
skip_nbf_dolt_dev
TARGET_NBF="__DOLT__"

View File

@@ -2,7 +2,7 @@
load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
skip_nbf_dolt_1
skip_nbf_dolt
skip_nbf_dolt_dev
setup_common

View File

@@ -44,3 +44,79 @@ teardown() {
dolt sql -q "show tables"
}
make__LD_1__db() {
mkdir $1
cd $1
DOLT_DEFAULT_BIN_FORMAT=__LD_1__ dolt init
cd ..
}
make__DOLT__db() {
mkdir $1
cd $1
DOLT_DEFAULT_BIN_FORMAT=__DOLT__ dolt init
cd ..
}
@test "multidb: databases are hidden based on DOLT_DEFAULT_BIN_FORMAT where there is no database in ./.dolt" {
cd dbs1
make__DOLT__db new1
make__LD_1__db old1
make__LD_1__db old2
orig_bin_format=$DOLT_DEFAULT_BIN_FORMAT
export DOLT_DEFAULT_BIN_FORMAT=__LD_1__
run dolt sql -q "SELECT 1;"
[ $status -eq 0 ]
[[ "$output" =~ "incompatible format for database 'new1'; expected '__LD_1__', found '__DOLT__'" ]] || false
run dolt sql -q "SHOW DATABASES;"
[ $status -eq 0 ]
echo $output
[[ "$output" =~ "| old1" ]] || false
[[ "$output" =~ "| old2" ]] || false
[[ ! "$output" =~ "| new1" ]] || false
export DOLT_DEFAULT_BIN_FORMAT=__DOLT__
run dolt sql -q "SELECT 1;"
[ $status -eq 0 ]
[[ "$output" =~ "incompatible format for database 'old1'; expected '__DOLT__', found '__LD_1__'" ]] || false
[[ "$output" =~ "incompatible format for database 'old2'; expected '__DOLT__', found '__LD_1__'" ]] || false
run dolt sql -q "SHOW DATABASES;"
[ $status -eq 0 ]
[[ ! "$output" =~ "| old1" ]] || false
[[ ! "$output" =~ "| old2" ]] || false
[[ "$output" =~ "| new1" ]] || false
export DOLT_DEFAULT_BIN_FORMAT=$orig_bin_format
}
@test "multidb: additional databases are hidden based on the format of the database in ./.dolt" {
rm -r dbs1
mkdir test_db
cd test_db
make__DOLT__db new1
make__LD_1__db old1
DOLT_DEFAULT_BIN_FORMAT="__DOLT__" dolt init
run dolt sql -q "SHOW DATABASES;"
[ $status -eq 0 ]
[[ "$output" =~ "incompatible format for database 'old1'; expected '__DOLT__', found '__LD_1__'" ]] || false
[[ "$output" =~ "| test_db" ]] || false
[[ "$output" =~ "| new1" ]] || false
[[ ! "$output" =~ "| old1" ]] || false
rm -r .dolt
DOLT_DEFAULT_BIN_FORMAT="__LD_1__" dolt init
run dolt sql -q "SHOW DATABASES;"
[ $status -eq 0 ]
[[ "$output" =~ "incompatible format for database 'new1'; expected '__LD_1__', found '__DOLT__'" ]] || false
[[ "$output" =~ "| test_db" ]] || false
[[ ! "$output" =~ "| new1" ]] || false
[[ "$output" =~ "| old1" ]] || false
}

View File

@@ -4,7 +4,7 @@ load $BATS_TEST_DIRNAME/helper/common.bash
setup() {
setup_common
skip_nbf_dolt_1 "need to add a __DOLT__ repo for AWS tests"
skip_nbf_dolt "need to add a __DOLT__ repo for AWS tests"
}
teardown() {

View File

@@ -246,7 +246,7 @@ SQL
dolt add .
dolt commit -m "Created table with one row"
skip_nbf_dolt_1 "In __DOLT__ the following throws an error since the primary key types changed"
skip_nbf_dolt "In __DOLT__ the following throws an error since the primary key types changed"
dolt merge main
run dolt sql -q 'show create table test2'

View File

@@ -722,7 +722,7 @@ SQL
}
@test "sql-server: JSON queries" {
skip_nbf_dolt_1
skip_nbf_dolt
cd repo1
start_sql_server repo1
@@ -1203,7 +1203,7 @@ END""")
@test "sql-server: drop database with active connections" {
skiponwindows "Missing dependencies"
skip_nbf_dolt_1 "json ordering of keys differs"
skip_nbf_dolt "json ordering of keys differs"
mkdir no_dolt && cd no_dolt
start_sql_server

View File

@@ -2187,7 +2187,7 @@ SQL
@test "sql: dolt_version() func" {
SQL=$(dolt sql -q 'select dolt_version() from dual;' -r csv | tail -n 1)
CLI=$(dolt version | cut -f 3 -d ' ')
CLI=$(dolt version | sed '1p;d' | cut -d " " -f 3)
[ "$SQL" == "$CLI" ]
}