mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2025-12-30 16:12:39 -06:00
Code Issues Packages Projects Releases 114 Wiki Activity

Multi DB (#517)

Browse Source
This commit is contained in:
Brian Hendriks
2020-04-09 15:07:52 -07:00
committed by GitHub
parent 81371eade2
commit e3d42d5ee5
31 changed files with 1222 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -5,3 +5,4 @@
go.sum
go.mod
.DS_Store
.sqlhistory

2
bats/config.bats
View File

@@ -82,7 +82,7 @@ teardown() {
dolt config --global --add user.email "bats-tester@liquidata.co"
run dolt init
[ "$status" -eq 0 ]
[ "$output" = "Successfully initialized dolt data repository." ]
[[ "$output" =~ "Successfully initialized dolt data repository." ]] || false
}
@test "set a local config variable" {

6
bats/helper/query-server-common.bash
View File

@@ -20,12 +20,12 @@ import os
import sys
args = sys.argv[sys.argv.index('--') + 1:]
working_dir, port_str, timeout_ms = args
working_dir, port_str, timeout_ms, db = args
os.chdir(working_dir)
from pytest import wait_for_connection
wait_for_connection(port=int(port_str), timeout_ms=int(timeout_ms))
" -- $PYTEST_DIR $1 $2
wait_for_connection(port=int(port_str), timeout_ms=int(timeout_ms), database=db)
" -- $PYTEST_DIR $1 $2 dolt
}
start_sql_server() {

2
bats/no-repo.bats
View File

@@ -213,7 +213,7 @@ NOT_VALID_REPO_ERROR="The current directory is not a valid dolt repository."
cd dolt-repo-$$-new
run dolt init
[ "$status" -eq 0 ]
[ "$output" = "Successfully initialized dolt data repository." ]
[[ "$output" =~ "Successfully initialized dolt data repository." ]] || false
[ -d .dolt ]
[ -d .dolt/noms ]
[ -f .dolt/config.json ]

85
bats/sql-multi-db.bats Normal file
View File

@@ -0,0 +1,85 @@
load $BATS_TEST_DIRNAME/helper/common.bash
make_repo() {
mkdir $1
cd $1
echo $PWD
dolt init
cd ..
}
setup() {
setup_no_dolt_init
make_repo repo1
make_repo repo2
}
teardown() {
teardown_common
}
seed_repos_with_tables_with_use_statements() {
dolt sql -r csv --multi-db-dir ./ -b -q "
USE repo1;
CREATE TABLE r1_t1 (pk BIGINT, PRIMARY KEY(pk));
INSERT INTO r1_t1 (pk) values (0),(1),(2);
USE repo2;
CREATE TABLE r2_t1 (pk BIGINT, c1 BIGINT, PRIMARY KEY(pk));
INSERT INTO r2_t1 (pk, c1) values (2,200),(3,300),(4,400);"
}
@test "sql multi-db test show databases" {
EXPECTED=$(echo -e "Database\nrepo1\nrepo2")
run dolt sql -r csv --multi-db-dir ./ -q "SHOW DATABASES"
[ "$status" -eq 0 ]
[[ "$output" =~ "$EXPECTED" ]] || false
}
@test "sql use statement and table accessibility" {
seed_repos_with_tables_with_use_statements
EXPECTED_R1T1=$(echo -e "pk\n0\n1\n2")
run dolt sql -r csv --multi-db-dir ./ -b -q "USE repo1; SELECT * FROM r1_t1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "$EXPECTED_R1T1" ]] || false
EXPECTED_R2T1=$(echo -e "pk,c1\n2,200\n3,300\n4,400")
run dolt sql -r csv --multi-db-dir ./ -b -q "USE repo2; SELECT * FROM r2_t1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "$EXPECTED_R2T1" ]] || false
# test tables of other database inaccessible without database qualifier
run dolt sql -r csv --multi-db-dir ./ -b -q "USE repo1; SELECT * FROM r2_t1;"
[ ! "$status" -eq 0 ]
run dolt sql -r csv --multi-db-dir ./ -b -q "USE repo2; SELECT * FROM r1_t1;"
[ ! "$status" -eq 0 ]
# test tables in other databases accessible when qualified
run dolt sql -r csv --multi-db-dir ./ -b -q "USE repo1; SELECT * FROM repo2.r2_t1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "$EXPECTED_R2T1" ]] || false
run dolt sql -r csv --multi-db-dir ./ -b -q "USE repo2; SELECT * FROM repo1.r1_t1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "$EXPECTED_R1T1" ]] || false
}
@test "sql test use invalid db name" {
seed_repos_with_tables_with_use_statements
run dolt sql -r csv --multi-db-dir ./ -q "USE invalid_db_name;"
[ ! "$status" -eq 0 ]
echo $output
[[ "$output" =~ "database not found: invalid_db_name" ]] || false
}
@test "sql join tables in different databases" {
seed_repos_with_tables_with_use_statements
EXPECTED=$(echo -e "pk,c1\n2,200")
run dolt sql -r csv --multi-db-dir ./ -b -q "
USE repo1;
SELECT r1_t1.pk, repo2.r2_t1.c1 FROM r1_t1 JOIN repo2.r2_t1 ON r1_t1.pk=repo2.r2_t1.pk;"
echo \"\"\"$output\"\"\"
[ "$status" -eq 0 ]
[[ "$output" =~ "$EXPECTED" ]] || false
}

2
bats/sql-server.bats
View File

@@ -3,6 +3,7 @@ load $BATS_TEST_DIRNAME/helper/common.bash
load $BATS_TEST_DIRNAME/helper/query-server-common.bash
setup() {
skip "Need to update server tests now that each connection has state"
skiponwindows "Has dependencies that are missing on the Jenkins Windows installation."
setup_common
@@ -10,6 +11,7 @@ setup() {
}
teardown() {
skip "Need to update server tests now that each connection has state"
skiponwindows "Has dependencies that are missing on the Jenkins Windows installation."
stop_sql_server

33
go/cmd/dolt/cli/command.go
View File

@@ -135,17 +135,8 @@ func (hc SubCommandHandler) Exec(ctx context.Context, commandStr string, args []
}
if cmdRequiresRepo && !hasHelpFlag(args) {
if !dEnv.HasDoltDir() {
PrintErrln(color.RedString("The current directory is not a valid dolt repository."))
PrintErrln("run: dolt init before trying to run this command")
return 2
} else if dEnv.RSLoadErr != nil {
PrintErrln(color.RedString("The current directories repository state is invalid"))
PrintErrln(dEnv.RSLoadErr.Error())
return 2
} else if dEnv.DBLoadError != nil {
PrintErrln(color.RedString("Failed to load database."))
PrintErrln(dEnv.DBLoadError.Error())
isValid := CheckEnvIsValid(dEnv)
if !isValid {
return 2
}
}
@@ -174,6 +165,26 @@ func (hc SubCommandHandler) Exec(ctx context.Context, commandStr string, args []
return 1
}
// CheckEnvIsValid validates that a DoltEnv has been initialized properly and no errors occur during load, and prints
// error messages to the user if there are issues with the environment or if errors were encountered while loading it.
func CheckEnvIsValid(dEnv *env.DoltEnv) bool {
if !dEnv.HasDoltDir() {
PrintErrln(color.RedString("The current directory is not a valid dolt repository."))
PrintErrln("run: dolt init before trying to run this command")
return false
} else if dEnv.RSLoadErr != nil {
PrintErrln(color.RedString("The current directories repository state is invalid"))
PrintErrln(dEnv.RSLoadErr.Error())
return false
} else if dEnv.DBLoadError != nil {
PrintErrln(color.RedString("Failed to load database."))
PrintErrln(dEnv.DBLoadError.Error())
return false
}
return true
}
func (hc SubCommandHandler) printUsage(commandStr string) {
Println("Valid commands for", commandStr, "are")

351
go/cmd/dolt/commands/sql.go
View File

@@ -30,6 +30,7 @@ import (
"github.com/liquidata-inc/ishell"
sqle "github.com/src-d/go-mysql-server"
"github.com/src-d/go-mysql-server/sql"
"gopkg.in/src-d/go-errors.v1"
"vitess.io/vitess/go/vt/sqlparser"
"vitess.io/vitess/go/vt/vterrors"
@@ -71,6 +72,10 @@ var sqlDocs = cli.CommandDocumentationContent{
"Pipe SQL statements to dolt sql (no {{.EmphasisLeft}}-q{{.EmphasisRight}}) to execute a SQL import or update " +
"script.\n" +
"\n" +
"By default this command uses the dolt data repository in the current working directory as the one and only " +
"database. Running with {{.EmphasisLeft}}--multi-db-dir {{.LessThan}}directory{{.GreaterThan}}{{.EmphasisRight}} " +
"uses each of the subdirectories of the supplied directory (each subdirectory must be a valid dolt data repository) " +
"as databases. Subdirectories starting with '.' are ignored." +
"Known limitations:\n" +
"* No support for creating indexes\n" +
"* No support for foreign keys\n" +
@@ -80,30 +85,31 @@ var sqlDocs = cli.CommandDocumentationContent{
"join, which is very slow.",
Synopsis: []string{
"",
"-q {{.LessThan}}query{{.GreaterThan}}",
"-q {{.LessThan}}query;query{{.GreaterThan}} -b",
"-q {{.LessThan}}query{{.GreaterThan}} -r {{.LessThan}}result format{{.GreaterThan}}",
"-q {{.LessThan}}query{{.GreaterThan}} -s {{.LessThan}}name{{.GreaterThan}} -m {{.LessThan}}message{{.GreaterThan}}",
"[--multi-db-dir {{.LessThan}}directory{{.GreaterThan}}] [-r {{.LessThan}}result format{{.GreaterThan}}]",
"-q {{.LessThan}}query;query{{.GreaterThan}} [-r {{.LessThan}}result format{{.GreaterThan}}] -s {{.LessThan}}name{{.GreaterThan}} -m {{.LessThan}}message{{.GreaterThan}} [-b]",
"-q {{.LessThan}}query;query{{.GreaterThan}} --multi-db-dir {{.LessThan}}directory{{.GreaterThan}} [-r {{.LessThan}}result format{{.GreaterThan}}] [-b]",
"-x {{.LessThan}}name{{.GreaterThan}}",
"--list-saved",
},
}
const (
queryFlag = "query"
formatFlag = "result-format"
saveFlag = "save"
executeFlag = "execute"
listSavedFlag = "list-saved"
messageFlag = "message"
batchFlag = "batch"
welcomeMsg = `# Welcome to the DoltSQL shell.
queryFlag = "query"
formatFlag = "result-format"
saveFlag = "save"
executeFlag = "execute"
listSavedFlag = "list-saved"
messageFlag = "message"
batchFlag = "batch"
multiDBDirFlag = "multi-db-dir"
welcomeMsg = `# Welcome to the DoltSQL shell.
# Statements must be terminated with ';'.
# "exit" or "quit" (or Ctrl-D) to exit.`
)
type SqlCmd struct{}
type SqlCmd struct {
VersionStr string
}
// Name is returns the name of the Dolt cli command. This is what is used on the command line to invoke the command
func (cmd SqlCmd) Name() string {
@@ -130,6 +136,7 @@ func (cmd SqlCmd) createArgParser() *argparser.ArgParser {
ap.SupportsFlag(listSavedFlag, "l", "Lists all saved queries")
ap.SupportsString(messageFlag, "m", "saved query description", "Used with --query and --save, saves the query with the descriptive message given. See also --name")
ap.SupportsFlag(batchFlag, "b", "batch mode, to run more than one query with --query, separated by ';'. Piping input to sql with no arguments also uses batch mode")
ap.SupportsString(multiDBDirFlag, "", "directory", "Defines a directory whose subdirectories should all be dolt data repositories accessible as independent databases within ")
return ap
}
@@ -138,6 +145,14 @@ func (cmd SqlCmd) EventType() eventsapi.ClientEventType {
return eventsapi.ClientEventType_SQL
}
// RequiresRepo indicates that this command does not have to be run from within a dolt data repository directory.
// In this case it is because this command supports the multiDBDirFlag which can pass in a directory. In the event that
// that parameter is not provided there is additional error handling within this command to make sure that this was in
// fact run from within a dolt data repository directory.
func (cmd SqlCmd) RequiresRepo() bool {
return false
}
// Exec executes the command
func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv) int {
ap := cmd.createArgParser()
@@ -151,11 +166,7 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
args = apr.Args()
root, verr := GetWorkingWithVErr(dEnv)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
}
var verr errhand.VerboseError
format := formatTabular
if formatSr, ok := apr.GetValue(formatFlag); ok {
format, verr = getFormat(formatSr)
@@ -164,20 +175,65 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
}
}
origRoot := root
var mrEnv env.MultiRepoEnv
var initialRoots map[string]*doltdb.RootValue
if multiDir, ok := apr.GetValue(multiDBDirFlag); !ok {
if !cli.CheckEnvIsValid(dEnv) {
return 2
}
mrEnv = env.DoltEnvAsMultiEnv(dEnv)
initialRoots, err = mrEnv.GetWorkingRoots(ctx)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
} else {
mrEnv, err = env.LoadMultiEnvFromDir(ctx, env.GetCurrentUserHomeDir, dEnv.FS, multiDir, cmd.VersionStr)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
initialRoots, err = mrEnv.GetWorkingRoots(ctx)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
}
sqlCtx := sql.NewContext(ctx,
sql.WithSession(dsqle.DefaultDoltSession()),
sql.WithIndexRegistry(sql.NewIndexRegistry()),
sql.WithViewRegistry(sql.NewViewRegistry()))
roots := make(map[string]*doltdb.RootValue)
var name string
var root *doltdb.RootValue
for name, root = range initialRoots {
roots[name] = root
}
var currentDB string
if len(initialRoots) == 1 {
sqlCtx.SetCurrentDatabase(name)
currentDB = name
}
if err != nil {
return HandleVErrAndExitCode(err.(errhand.VerboseError), usage)
}
if query, queryOK := apr.GetValue(queryFlag); queryOK {
batchMode := apr.Contains(batchFlag)
if batchMode {
batchInput := strings.NewReader(query)
root, verr = execBatch(sqlCtx, dEnv, root, batchInput, format)
roots, verr = execBatch(sqlCtx, mrEnv, roots, batchInput, format)
} else {
root, verr = execQuery(sqlCtx, dEnv, root, query, format)
roots, verr = execQuery(sqlCtx, mrEnv, roots, query, format)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
@@ -187,20 +243,20 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
if saveName != "" {
saveMessage := apr.GetValueOrDefault(messageFlag, "")
root, verr = saveQuery(ctx, root, dEnv, query, saveName, saveMessage)
roots[currentDB], verr = saveQuery(ctx, roots[currentDB], dEnv, query, saveName, saveMessage)
}
}
} else if savedQueryName, exOk := apr.GetValue(executeFlag); exOk {
sq, err := dsqle.RetrieveFromQueryCatalog(ctx, root, savedQueryName)
sq, err := dsqle.RetrieveFromQueryCatalog(ctx, roots[currentDB], savedQueryName)
if err != nil {
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
cli.PrintErrf("Executing saved query '%s':\n%s\n", savedQueryName, sq.Query)
root, verr = execQuery(sqlCtx, dEnv, root, sq.Query, format)
roots, verr = execQuery(sqlCtx, mrEnv, roots, sq.Query, format)
} else if apr.Contains(listSavedFlag) {
hasQC, err := root.HasTable(ctx, doltdb.DoltQueryCatalogTableName)
hasQC, err := roots[currentDB].HasTable(ctx, doltdb.DoltQueryCatalogTableName)
if err != nil {
verr := errhand.BuildDError("error: Failed to read from repository.").AddCause(err).Build()
@@ -212,7 +268,7 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
}
query := "SELECT * FROM " + doltdb.DoltQueryCatalogTableName
_, verr = execQuery(sqlCtx, dEnv, root, query, format)
_, verr = execQuery(sqlCtx, mrEnv, roots, query, format)
} else {
// Run in either batch mode for piped input, or shell mode for interactive
runInBatchMode := true
@@ -227,9 +283,9 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
}
if runInBatchMode {
root, verr = execBatch(sqlCtx, dEnv, root, os.Stdin, format)
roots, verr = execBatch(sqlCtx, mrEnv, roots, os.Stdin, format)
} else {
root, verr = execShell(sqlCtx, dEnv, root, format)
roots, verr = execShell(sqlCtx, mrEnv, roots, format)
}
}
@@ -238,35 +294,40 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
}
// If the SQL session wrote a new root value, update the working set with it
if origRoot != root {
verr = UpdateWorkingWithVErr(dEnv, root)
for name, origRoot := range initialRoots {
root := roots[name]
if origRoot != root {
currEnv := mrEnv[name]
verr = UpdateWorkingWithVErr(currEnv, root)
}
}
return HandleVErrAndExitCode(verr, usage)
}
func execShell(sqlCtx *sql.Context, dEnv *env.DoltEnv, root *doltdb.RootValue, format resultFormat) (*doltdb.RootValue, errhand.VerboseError) {
se, err := newSqlEngine(sqlCtx, dEnv, root, dsqle.NewDatabase("dolt", root, dEnv.DoltDB, dEnv.RepoState), format)
func execShell(sqlCtx *sql.Context, mrEnv env.MultiRepoEnv, roots map[string]*doltdb.RootValue, format resultFormat) (map[string]*doltdb.RootValue, errhand.VerboseError) {
dbs := CollectDBs(mrEnv, roots, dsqle.NewDatabase)
se, err := newSqlEngine(sqlCtx, mrEnv, roots, format, dbs...)
if err != nil {
return nil, errhand.VerboseErrorFromError(err)
}
err = runShell(sqlCtx, se, dEnv)
err = runShell(sqlCtx, se, mrEnv)
if err != nil {
return nil, errhand.BuildDError("unable to start shell").AddCause(err).Build()
}
newRoot, err := se.sdb.GetRoot(sqlCtx)
newRoots, err := se.getRoots(sqlCtx)
if err != nil {
return nil, errhand.BuildDError("Failed to get root").AddCause(err).Build()
return nil, errhand.BuildDError("failed to get roots").AddCause(err).Build()
}
return newRoot, nil
return newRoots, nil
}
func execBatch(sqlCtx *sql.Context, dEnv *env.DoltEnv, root *doltdb.RootValue, batchInput io.Reader, format resultFormat) (*doltdb.RootValue, errhand.VerboseError) {
se, err := newSqlEngine(sqlCtx, dEnv, root, dsqle.NewBatchedDatabase("dolt", root, dEnv.DoltDB, dEnv.RepoState), format)
func execBatch(sqlCtx *sql.Context, mrEnv env.MultiRepoEnv, roots map[string]*doltdb.RootValue, batchInput io.Reader, format resultFormat) (map[string]*doltdb.RootValue, errhand.VerboseError) {
dbs := CollectDBs(mrEnv, roots, dsqle.NewBatchedDatabase)
se, err := newSqlEngine(sqlCtx, mrEnv, roots, format, dbs...)
if err != nil {
return nil, errhand.VerboseErrorFromError(err)
}
@@ -276,17 +337,17 @@ func execBatch(sqlCtx *sql.Context, dEnv *env.DoltEnv, root *doltdb.RootValue, b
return nil, errhand.BuildDError("Error processing batch").Build()
}
newRoot, err := se.sdb.GetRoot(sqlCtx)
newRoots, err := se.getRoots(sqlCtx)
if err != nil {
return nil, errhand.BuildDError("Failed to get root").AddCause(err).Build()
return nil, errhand.BuildDError("failed to get roots").AddCause(err).Build()
}
return newRoot, nil
return newRoots, nil
}
func execQuery(sqlCtx *sql.Context, dEnv *env.DoltEnv, root *doltdb.RootValue, query string, format resultFormat) (*doltdb.RootValue, errhand.VerboseError) {
se, err := newSqlEngine(sqlCtx, dEnv, root, dsqle.NewDatabase("dolt", root, dEnv.DoltDB, dEnv.RepoState), format)
func execQuery(sqlCtx *sql.Context, mrEnv env.MultiRepoEnv, roots map[string]*doltdb.RootValue, query string, format resultFormat) (map[string]*doltdb.RootValue, errhand.VerboseError) {
dbs := CollectDBs(mrEnv, roots, dsqle.NewDatabase)
se, err := newSqlEngine(sqlCtx, mrEnv, roots, format, dbs...)
if err != nil {
return nil, errhand.VerboseErrorFromError(err)
}
@@ -299,19 +360,34 @@ func execQuery(sqlCtx *sql.Context, dEnv *env.DoltEnv, root *doltdb.RootValue, q
if rowIter != nil {
defer rowIter.Close()
err = se.prettyPrintResults(sqlCtx, se.ddb.ValueReadWriter().Format(), sqlSch, rowIter)
err = se.prettyPrintResults(sqlCtx, sqlSch, rowIter)
if err != nil {
return nil, errhand.VerboseErrorFromError(err)
}
}
newRoot, err := se.sdb.GetRoot(sqlCtx)
newRoots, err := se.getRoots(sqlCtx)
if err != nil {
return nil, errhand.BuildDError("Failed to get root").AddCause(err).Build()
return nil, errhand.BuildDError("failed to get roots").AddCause(err).Build()
}
return newRoot, nil
return newRoots, nil
}
type createDBFunc func(name string, defRoot *doltdb.RootValue, ddb *doltdb.DoltDB, rsr env.RepoStateReader) dsqle.Database
// CollectDBs takes a MultiRepoEnv and creates Database objects from each environment and returns a slice of these
// objects.
func CollectDBs(mrEnv env.MultiRepoEnv, roots map[string]*doltdb.RootValue, createDB createDBFunc) []dsqle.Database {
dbs := make([]dsqle.Database, 0, len(mrEnv))
_ = mrEnv.Iter(func(name string, dEnv *env.DoltEnv) (stop bool, err error) {
root := roots[name]
db := createDB(name, root, dEnv.DoltDB, dEnv.RepoState)
dbs = append(dbs, db)
return false, nil
})
return dbs
}
func formatQueryError(query string, err error) errhand.VerboseError {
@@ -388,6 +464,7 @@ func validateSqlArgs(apr *argparser.ArgParseResults) error {
_, batch := apr.GetValue(batchFlag)
_, list := apr.GetValue(listSavedFlag)
_, execute := apr.GetValue(executeFlag)
_, multiDB := apr.GetValue(multiDBDirFlag)
if len(apr.Args()) > 0 && !query {
return errhand.BuildDError("Invalid Argument: use --query or -q to pass inline SQL queries").Build()
@@ -402,6 +479,8 @@ func validateSqlArgs(apr *argparser.ArgParseResults) error {
return errhand.BuildDError("Invalid Argument: --execute|-x is not compatible with --message|-m").Build()
} else if save {
return errhand.BuildDError("Invalid Argument: --execute|-x is not compatible with --save|-s").Build()
} else if multiDB {
return errhand.BuildDError("Invalid Argument: --execute|-x is not compatible with --multi-db-dir").Build()
}
}
@@ -414,9 +493,15 @@ func validateSqlArgs(apr *argparser.ArgParseResults) error {
return errhand.BuildDError("Invalid Argument: --list-saved is not compatible with --message|-m").Build()
} else if save {
return errhand.BuildDError("Invalid Argument: --list-saved is not compatible with --save|-s").Build()
} else if multiDB {
return errhand.BuildDError("Invalid Argument: --execute|-x is not compatible with --multi-db-dir").Build()
}
}
if save && multiDB {
return errhand.BuildDError("Invalid Argument: --multi-db-dir queries cannot be saved").Build()
}
if batch {
if !query {
return errhand.BuildDError("Invalid Argument: --batch|-b must be used with --query|-q").Build()
@@ -503,11 +588,7 @@ func runBatchMode(ctx *sql.Context, se *sqlEngine, input io.Reader) error {
cli.Println(err.Error())
}
if err := se.sdb.Flush(ctx); err != nil {
return err
}
return nil
return flushBatchedEdits(ctx, se)
}
// batchInsertEarlySemicolon loops through a string to check if Scan stopped too early on a semicolon
@@ -542,13 +623,15 @@ func batchInsertEarlySemicolon(query string) bool {
// runShell starts a SQL shell. Returns when the user exits the shell. The Root of the sqlEngine may
// be updated by any queries which were processed.
func runShell(ctx *sql.Context, se *sqlEngine, dEnv *env.DoltEnv) error {
func runShell(ctx *sql.Context, se *sqlEngine, mrEnv env.MultiRepoEnv) error {
_ = iohelp.WriteLine(cli.CliOut, welcomeMsg)
currentDB := ctx.Session.GetCurrentDatabase()
currEnv := mrEnv[currentDB]
// start the doltsql shell
historyFile := filepath.Join(dEnv.GetDoltDir(), ".sqlhistory")
historyFile := filepath.Join(".sqlhistory") // history file written to working dir
rlConf := readline.Config{
Prompt: "doltsql> ",
Prompt: fmt.Sprintf("%s>", ctx.GetCurrentDatabase()),
Stdout: cli.CliOut,
Stderr: cli.CliOut,
HistoryFile: historyFile,
@@ -567,7 +650,7 @@ func runShell(ctx *sql.Context, se *sqlEngine, dEnv *env.DoltEnv) error {
shell := ishell.NewUninterpreted(&shellConf)
shell.SetMultiPrompt(" -> ")
// TODO: update completer on create / drop / alter statements
completer, err := newCompleter(ctx, dEnv)
completer, err := newCompleter(ctx, currEnv)
if err != nil {
return err
}
@@ -597,7 +680,7 @@ func runShell(ctx *sql.Context, se *sqlEngine, dEnv *env.DoltEnv) error {
shell.Println(verr.Verbose())
} else if rowIter != nil {
defer rowIter.Close()
err = se.prettyPrintResults(ctx, se.ddb.ValueReadWriter().Format(), sqlSch, rowIter)
err = se.prettyPrintResults(ctx, sqlSch, rowIter)
if err != nil {
shell.Println(color.RedString(err.Error()))
}
@@ -612,6 +695,8 @@ func runShell(ctx *sql.Context, se *sqlEngine, dEnv *env.DoltEnv) error {
// TODO: handle better, like by turning off history writing for the rest of the session
shell.Println(color.RedString(err.Error()))
}
shell.SetPrompt(fmt.Sprintf("%s>", ctx.GetCurrentDatabase()))
})
shell.Run()
@@ -744,6 +829,19 @@ func processQuery(ctx *sql.Context, query string, se *sqlEngine) (sql.Schema, sq
switch s := sqlStatement.(type) {
case *sqlparser.Select, *sqlparser.Insert, *sqlparser.Update, *sqlparser.OtherRead, *sqlparser.Show, *sqlparser.Explain, *sqlparser.Union:
return se.query(ctx, query)
case *sqlparser.Use:
sch, rowIter, err := se.query(ctx, query)
if rowIter != nil {
_ = rowIter.Close()
}
if err != nil {
return nil, nil, err
}
cli.Println("Database changed")
return sch, nil, err
case *sqlparser.Delete:
ok := se.checkThenDeleteAllRows(ctx, s)
if ok {
@@ -792,13 +890,19 @@ func (s *stats) shouldFlush() bool {
}
func flushBatchedEdits(ctx *sql.Context, se *sqlEngine) error {
err := se.sdb.Flush(ctx)
if err != nil {
return err
}
err := se.iterDBs(func(_ string, db dsqle.Database) (bool, error) {
err := db.Flush(ctx)
if err != nil {
return false, err
}
return false, nil
})
batchEditStats.unflushedEdits = 0
return nil
return err
}
// Processes a single query in batch mode. The Root of the sqlEngine may or may not be changed.
@@ -857,7 +961,7 @@ func processNonInsertBatchQuery(ctx *sql.Context, se *sqlEngine, query string, s
cli.Print("\n")
displayStrLen = 0
}
err = se.prettyPrintResults(ctx, se.ddb.ValueReadWriter().Format(), sqlSch, rowIter)
err = se.prettyPrintResults(ctx, sqlSch, rowIter)
if err != nil {
return err
}
@@ -950,36 +1054,88 @@ const (
)
type sqlEngine struct {
sdb dsqle.Database
ddb *doltdb.DoltDB
dbs map[string]dsqle.Database
mrEnv env.MultiRepoEnv
engine *sqle.Engine
resultFormat resultFormat
}
var ErrDBNotFoundKind = errors.NewKind("database '%s' not found")
// sqlEngine packages up the context necessary to run sql queries against sqle.
func newSqlEngine(sqlCtx *sql.Context, dEnv *env.DoltEnv, root *doltdb.RootValue, db dsqle.Database, format resultFormat) (*sqlEngine, error) {
func newSqlEngine(sqlCtx *sql.Context, mrEnv env.MultiRepoEnv, roots map[string]*doltdb.RootValue, format resultFormat, dbs ...dsqle.Database) (*sqlEngine, error) {
engine := sqle.NewDefault()
engine.AddDatabase(db)
engine.AddDatabase(sql.NewInformationSchemaDatabase(engine.Catalog))
err := db.SetRoot(sqlCtx, root)
nameToDB := make(map[string]dsqle.Database)
for _, db := range dbs {
nameToDB[db.Name()] = db
root := roots[db.Name()]
engine.AddDatabase(db)
err := db.SetRoot(sqlCtx, root)
if err != nil {
return nil, err
}
err = dsqle.RegisterSchemaFragments(sqlCtx, db, root)
if err != nil {
return nil, err
}
}
sqlCtx.RegisterIndexDriver(dsqle.NewDoltIndexDriver(dbs...))
err := sqlCtx.LoadIndexes(sqlCtx, engine.Catalog.AllDatabases())
if err != nil {
return nil, err
}
sqlCtx.RegisterIndexDriver(dsqle.NewDoltIndexDriver(db))
err = sqlCtx.LoadIndexes(sqlCtx, engine.Catalog.AllDatabases())
if err != nil {
return nil, err
return &sqlEngine{nameToDB, mrEnv, engine, format}, nil
}
func (se *sqlEngine) getDB(name string) (dsqle.Database, error) {
db, ok := se.dbs[name]
if !ok {
return dsqle.Database{}, ErrDBNotFoundKind.New(name)
}
err = dsqle.RegisterSchemaFragments(sqlCtx, db, root)
return db, nil
}
if err != nil {
return nil, err
func (se *sqlEngine) iterDBs(cb func(name string, db dsqle.Database) (stop bool, err error)) error {
for name, db := range se.dbs {
stop, err := cb(name, db)
if err != nil {
return err
}
if stop {
break
}
}
return &sqlEngine{db, dEnv.DoltDB, engine, format}, nil
return nil
}
func (se *sqlEngine) getRoots(sqlCtx *sql.Context) (map[string]*doltdb.RootValue, error) {
newRoots := make(map[string]*doltdb.RootValue)
for name := range se.mrEnv {
db, err := se.getDB(name)
if err != nil {
return nil, err
}
newRoots[name], err = db.GetRoot(sqlCtx)
if err != nil {
return nil, err
}
}
return newRoots, nil
}
// Execute a SQL statement and return values for printing.
@@ -988,7 +1144,7 @@ func (se *sqlEngine) query(ctx *sql.Context, query string) (sql.Schema, sql.RowI
}
// Pretty prints the output of the new SQL engine
func (se *sqlEngine) prettyPrintResults(ctx context.Context, nbf *types.NomsBinFormat, sqlSch sql.Schema, rowIter sql.RowIter) error {
func (se *sqlEngine) prettyPrintResults(ctx context.Context, sqlSch sql.Schema, rowIter sql.RowIter) error {
var chanErr error
doltSch, err := dsqle.SqlSchemaToDoltResultSchema(sqlSch)
if err != nil {
@@ -1003,6 +1159,7 @@ func (se *sqlEngine) prettyPrintResults(ctx context.Context, nbf *types.NomsBinF
rowChannel := make(chan row.Row)
p := pipeline.NewPartialPipeline(pipeline.InFuncForChannel(rowChannel))
nbf := types.Format_Default
go func() {
defer close(rowChannel)
var sqlRow sql.Row
@@ -1088,16 +1245,29 @@ func (se *sqlEngine) prettyPrintResults(ctx context.Context, nbf *types.NomsBinF
return nil
}
var ErrNotNaked = fmt.Errorf("not a naked query.")
// Checks if the query is a naked delete and then deletes all rows if so. Returns true if it did so, false otherwise.
func (se *sqlEngine) checkThenDeleteAllRows(ctx *sql.Context, s *sqlparser.Delete) bool {
if s.Where == nil && s.Limit == nil && s.Partitions == nil && len(s.TableExprs) == 1 {
if ate, ok := s.TableExprs[0].(*sqlparser.AliasedTableExpr); ok {
if ste, ok := ate.Expr.(sqlparser.TableName); ok {
root, err := se.sdb.GetRoot(ctx)
dbName := ctx.Session.GetCurrentDatabase()
if !ste.Qualifier.IsEmpty() {
dbName = ste.Qualifier.String()
}
roots, err := se.getRoots(ctx)
if err != nil {
return false
}
root, ok := roots[dbName]
if !ok {
return false
}
tName := ste.Name.String()
table, ok, err := root.GetTable(ctx, tName)
if err == nil && ok {
@@ -1111,21 +1281,33 @@ func (se *sqlEngine) checkThenDeleteAllRows(ctx *sql.Context, s *sqlparser.Delet
if err != nil {
return false
}
printRowIter := sql.RowsToRowIter(sql.NewRow(rowData.Len()))
emptyMap, err := types.NewMap(ctx, root.VRW())
if err != nil {
return false
}
newTable, err := table.UpdateRows(ctx, emptyMap)
if err != nil {
return false
}
newRoot, err := root.PutTable(ctx, tName, newTable)
if err != nil {
return false
}
_ = se.prettyPrintResults(ctx, root.VRW().Format(), sql.Schema{{Name: "updated", Type: sql.Uint64}}, printRowIter)
err = se.sdb.SetRoot(ctx, newRoot)
_ = se.prettyPrintResults(ctx, sql.Schema{{Name: "updated", Type: sql.Uint64}}, printRowIter)
db, err := se.getDB(dbName)
if err != nil {
return false
}
err = db.SetRoot(ctx, newRoot)
if err != nil {
return false
}
@@ -1135,6 +1317,7 @@ func (se *sqlEngine) checkThenDeleteAllRows(ctx *sql.Context, s *sqlparser.Delet
}
}
}
return false
}

62
go/cmd/dolt/commands/sqlserver/server.go
View File

@@ -28,13 +28,14 @@ import (
"vitess.io/vitess/go/mysql"
"github.com/liquidata-inc/dolt/go/cmd/dolt/cli"
"github.com/liquidata-inc/dolt/go/cmd/dolt/commands"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/doltdb"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/env"
dsqle "github.com/liquidata-inc/dolt/go/libraries/doltcore/sqle"
)
// Serve starts a MySQL-compatible server. Returns any errors that were encountered.
func Serve(ctx context.Context, serverConfig *ServerConfig, rootValue *doltdb.RootValue, serverController *ServerController, dEnv *env.DoltEnv) (startError error, closeError error) {
func Serve(ctx context.Context, serverConfig *ServerConfig, serverController *ServerController, dEnv *env.DoltEnv) (startError error, closeError error) {
if serverConfig == nil {
cli.Println("No configuration given, using defaults")
serverConfig = DefaultServerConfig()
@@ -57,9 +58,11 @@ func Serve(ctx context.Context, serverConfig *ServerConfig, rootValue *doltdb.Ro
}()
if startError = serverConfig.Validate(); startError != nil {
cli.PrintErr(startError)
return
}
if serverConfig.LogLevel != LogLevel_Info {
var level logrus.Level
level, startError = logrus.ParseLevel(serverConfig.LogLevel.String())
@@ -77,11 +80,38 @@ func Serve(ctx context.Context, serverConfig *ServerConfig, rootValue *doltdb.Ro
userAuth := auth.NewAudit(auth.NewNativeSingle(serverConfig.User, serverConfig.Password, permissions), auth.NewAuditLog(logrus.StandardLogger()))
sqlEngine := sqle.NewDefault()
db := dsqle.NewDatabase("dolt", rootValue, dEnv.DoltDB, dEnv.RepoState)
sqlEngine.AddDatabase(db)
sqlEngine.AddDatabase(sql.NewInformationSchemaDatabase(sqlEngine.Catalog))
idxDriver := dsqle.NewDoltIndexDriver(db)
var mrEnv env.MultiRepoEnv
var roots map[string]*doltdb.RootValue
if serverConfig.MultiDBDir == "" {
var err error
mrEnv = env.DoltEnvAsMultiEnv(dEnv)
roots, err = mrEnv.GetWorkingRoots(ctx)
if err != nil {
return err, nil
}
} else {
var err error
mrEnv, err = env.LoadMultiEnvFromDir(ctx, env.GetCurrentUserHomeDir, dEnv.FS, serverConfig.MultiDBDir, serverConfig.Version)
if err != nil {
return err, nil
}
roots, err = mrEnv.GetWorkingRoots(ctx)
if err != nil {
return err, nil
}
}
dbs := commands.CollectDBs(mrEnv, roots, dsqle.NewDatabase)
for _, db := range dbs {
sqlEngine.AddDatabase(db)
}
sqlEngine.AddDatabase(sql.NewInformationSchemaDatabase(sqlEngine.Catalog))
hostPort := net.JoinHostPort(serverConfig.Host, strconv.Itoa(serverConfig.Port))
timeout := time.Second * time.Duration(serverConfig.Timeout)
@@ -110,19 +140,27 @@ func Serve(ctx context.Context, serverConfig *ServerConfig, rootValue *doltdb.Ro
sql.WithViewRegistry(vr),
sql.WithSession(doltSess))
ir.RegisterIndexDriver(idxDriver)
dbs := commands.CollectDBs(mrEnv, roots, dsqle.NewDatabase)
for _, db := range dbs {
err := db.SetRoot(sqlCtx, db.GetDefaultRoot())
if err != nil {
return nil, nil, nil, err
}
err = dsqle.RegisterSchemaFragments(sqlCtx, db, db.GetDefaultRoot())
if err != nil {
cli.PrintErr(err)
return nil, nil, nil, err
}
}
sqlCtx.RegisterIndexDriver(dsqle.NewDoltIndexDriver(dbs...))
err = ir.LoadIndexes(sqlCtx, sqlEngine.Catalog.AllDatabases())
if err != nil {
return nil, nil, nil, err
}
err = dsqle.RegisterSchemaFragments(sqlCtx, db, db.GetDefaultRoot())
if startError != nil {
cli.PrintErr(startError)
return nil, nil, nil, err
}
return doltSess, ir, vr, nil
},
)

9
go/cmd/dolt/commands/sqlserver/server_test.go
View File

@@ -24,7 +24,6 @@ import (
"github.com/stretchr/testify/require"
"golang.org/x/net/context"
"github.com/liquidata-inc/dolt/go/cmd/dolt/commands"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/dtestutils"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/env"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/table"
@@ -99,8 +98,6 @@ func TestServerBadArgs(t *testing.T) {
func TestServerGoodParams(t *testing.T) {
env := createEnvWithSeedData(t)
root, verr := commands.GetWorkingWithVErr(env)
require.NoError(t, verr)
tests := []*ServerConfig{
DefaultServerConfig(),
@@ -121,7 +118,7 @@ func TestServerGoodParams(t *testing.T) {
t.Run(test.String(), func(t *testing.T) {
sc := CreateServerController()
go func(config *ServerConfig, sc *ServerController) {
_, _ = Serve(context.Background(), config, root, sc, env)
_, _ = Serve(context.Background(), config, sc, env)
}(test, sc)
err := sc.WaitForStart()
require.NoError(t, err)
@@ -138,14 +135,12 @@ func TestServerGoodParams(t *testing.T) {
func TestServerSelect(t *testing.T) {
env := createEnvWithSeedData(t)
root, verr := commands.GetWorkingWithVErr(env)
require.NoError(t, verr)
serverConfig := DefaultServerConfig().WithLogLevel(LogLevel_Fatal).WithPort(15300)
sc := CreateServerController()
defer sc.StopServer()
go func() {
_, _ = Serve(context.Background(), serverConfig, root, sc, env)
_, _ = Serve(context.Background(), serverConfig, sc, env)
}()
err := sc.WaitForStart()
require.NoError(t, err)

41
go/cmd/dolt/commands/sqlserver/serverconfig.go
View File

@@ -22,6 +22,8 @@ import (
// LogLevel defines the available levels of logging for the server.
type LogLevel string
var CliVersion = "test"
const (
LogLevel_Debug LogLevel = "debug"
LogLevel_Info LogLevel = "info"
@@ -32,25 +34,29 @@ const (
// ServerConfig contains all of the configurable options for the MySQL-compatible server.
type ServerConfig struct {
Host string // The domain that the server will run on. Accepts an IPv4 or IPv6 address, in addition to localhost.
Port int // The port that the server will run on. The valid range is [1024, 65535].
User string // The username that connecting clients must use.
Password string // The password that connecting clients must use.
Timeout int // The read and write timeouts.
ReadOnly bool // Whether the server will only accept read statements or all statements.
LogLevel LogLevel // Specifies the level of logging that the server will use.
Host string // The domain that the server will run on. Accepts an IPv4 or IPv6 address, in addition to localhost.
Port int // The port that the server will run on. The valid range is [1024, 65535].
User string // The username that connecting clients must use.
Password string // The password that connecting clients must use.
Timeout int // The read and write timeouts.
ReadOnly bool // Whether the server will only accept read statements or all statements.
LogLevel LogLevel // Specifies the level of logging that the server will use.
MultiDBDir string // Directory whose children are dolt data repositories
Version string // Dolt cli version
}
// DefaultServerConfig creates a `*ServerConfig` that has all of the options set to their default values.
func DefaultServerConfig() *ServerConfig {
return &ServerConfig{
Host: "localhost",
Port: 3306,
User: "root",
Password: "",
Timeout: 30,
ReadOnly: false,
LogLevel: LogLevel_Info,
Host: "localhost",
Port: 3306,
User: "root",
Password: "",
Timeout: 30,
ReadOnly: false,
LogLevel: LogLevel_Info,
MultiDBDir: "",
Version: CliVersion,
}
}
@@ -119,6 +125,13 @@ func (config *ServerConfig) WithLogLevel(loglevel LogLevel) *ServerConfig {
return config
}
// WithMultiDBDir returns an updated server config with the root directory for a multi-db environment where each of
// the subdirectories is a dolt data repository that should be accessible via sql as a database.
func (config *ServerConfig) WithMultiDBDir(multiDBDir string) *ServerConfig {
config.MultiDBDir = multiDBDir
return config
}
// ConnectionString returns a Data Source Name (DSN) to be used by go clients for connecting to a running server.
func (config *ServerConfig) ConnectionString() string {
return fmt.Sprintf("%v:%v@tcp(%v:%v)/dolt", config.User, config.Password, config.Host, config.Port)

42
go/cmd/dolt/commands/sqlserver/sqlserver.go
View File

@@ -28,13 +28,14 @@ import (
)
const (
hostFlag = "host"
portFlag = "port"
userFlag = "user"
passwordFlag = "password"
timeoutFlag = "timeout"
readonlyFlag = "readonly"
logLevelFlag = "loglevel"
hostFlag = "host"
portFlag = "port"
userFlag = "user"
passwordFlag = "password"
timeoutFlag = "timeout"
readonlyFlag = "readonly"
logLevelFlag = "loglevel"
multiDBDirFlag = "multi-db-dir"
)
var sqlServerDocs = cli.CommandDocumentationContent{
@@ -44,7 +45,7 @@ var sqlServerDocs = cli.CommandDocumentationContent{
Currently, only {{.EmphasisLeft}}SELECT{{.EmphasisRight}} statements are operational, as support for other statements is still being developed.
`,
Synopsis: []string{
"[-H {{.LessThan}}host{{.GreaterThan}}] [-P {{.LessThan}}port{{.GreaterThan}}] [-u {{.LessThan}}user{{.GreaterThan}}] [-p {{.LessThan}}password{{.GreaterThan}}] [-t {{.LessThan}}timeout{{.GreaterThan}}] [-l {{.LessThan}}loglevel{{.GreaterThan}}] [-r]",
"[-H {{.LessThan}}host{{.GreaterThan}}] [-P {{.LessThan}}port{{.GreaterThan}}] [-u {{.LessThan}}user{{.GreaterThan}}] [-p {{.LessThan}}password{{.GreaterThan}}] [-t {{.LessThan}}timeout{{.GreaterThan}}] [-l {{.LessThan}}loglevel{{.GreaterThan}}] [--multi-db-dir {{.LessThan}}directory{{.GreaterThan}}] [-r]",
},
}
@@ -75,6 +76,7 @@ func createArgParser(serverConfig *ServerConfig) *argparser.ArgParser {
ap.SupportsInt(timeoutFlag, "t", "Connection timeout", fmt.Sprintf("Defines the timeout, in seconds, used for connections\nA value of `0` represents an infinite timeout (default `%v`)", serverConfig.Timeout))
ap.SupportsFlag(readonlyFlag, "r", "Disables modification of the database")
ap.SupportsString(logLevelFlag, "l", "Log level", fmt.Sprintf("Defines the level of logging provided\nOptions are: `debug`, `info`, `warning`, `error`, `fatal` (default `%v`)", serverConfig.LogLevel))
ap.SupportsString(multiDBDirFlag, "", "directory", "Defines a directory whose subdirectories should all be dolt data repositories accessible as independent databases.")
return ap
}
@@ -83,6 +85,14 @@ func (cmd SqlServerCmd) EventType() eventsapi.ClientEventType {
return eventsapi.ClientEventType_SQL_SERVER
}
// RequiresRepo indicates that this command does not have to be run from within a dolt data repository directory.
// In this case it is because this command supports the multiDBDirFlag which can pass in a directory. In the event that
// that parameter is not provided there is additional error handling within this command to make sure that this was in
// fact run from within a dolt data repository directory.
func (cmd SqlServerCmd) RequiresRepo() bool {
return false
}
// Exec executes the command
func (cmd SqlServerCmd) Exec(ctx context.Context, commandStr string, args []string, dEnv *env.DoltEnv) int {
return SqlServerImpl(ctx, commandStr, args, dEnv, nil)
@@ -92,16 +102,11 @@ func SqlServerImpl(ctx context.Context, commandStr string, args []string, dEnv *
serverConfig := DefaultServerConfig()
ap := createArgParser(serverConfig)
help, usage := cli.HelpAndUsagePrinters(cli.GetCommandDocumentation(commandStr, sqlServerDocs, ap))
help, _ := cli.HelpAndUsagePrinters(cli.GetCommandDocumentation(commandStr, sqlServerDocs, ap))
apr := cli.ParseArgs(ap, args, help)
args = apr.Args()
root, verr := commands.GetWorkingWithVErr(dEnv)
if verr != nil {
return commands.HandleVErrAndExitCode(verr, usage)
}
if host, ok := apr.GetValue(hostFlag); ok {
serverConfig.Host = host
}
@@ -123,10 +128,17 @@ func SqlServerImpl(ctx context.Context, commandStr string, args []string, dEnv *
if logLevel, ok := apr.GetValue(logLevelFlag); ok {
serverConfig.LogLevel = LogLevel(logLevel)
}
if multiDBDir, ok := apr.GetValue(multiDBDirFlag); ok {
serverConfig.MultiDBDir = multiDBDir
} else {
if !cli.CheckEnvIsValid(dEnv) {
return 2
}
}
cli.PrintErrf("Starting server on port %d.", serverConfig.Port)
if startError, closeError := Serve(ctx, serverConfig, root, serverController, dEnv); startError != nil || closeError != nil {
if startError, closeError := Serve(ctx, serverConfig, serverController, dEnv); startError != nil || closeError != nil {
if startError != nil {
cli.PrintErrln(startError)
}

3
go/cmd/dolt/dolt.go
View File

@@ -51,7 +51,7 @@ var doltCommand = cli.NewSubCommandHandler("dolt", "it's git for data", []cli.Co
commands.AddCmd{},
commands.ResetCmd{},
commands.CommitCmd{},
commands.SqlCmd{},
commands.SqlCmd{VersionStr: Version},
sqlserver.SqlServerCmd{},
commands.LogCmd{},
commands.DiffCmd{},
@@ -79,6 +79,7 @@ var doltCommand = cli.NewSubCommandHandler("dolt", "it's git for data", []cli.Co
func init() {
dumpDocsCommand.DoltCommand = doltCommand
sqlserver.CliVersion = Version
}
const chdirFlag = "--chdir"

4
go/go.mod
View File

@@ -87,10 +87,10 @@ require (
replace github.com/liquidata-inc/dolt/go/gen/proto/dolt/services/eventsapi => ./gen/proto/dolt/services/eventsapi
replace github.com/src-d/go-mysql-server => github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200330231002-2ac5a85cf8d6
replace github.com/src-d/go-mysql-server => github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200407175239-21fb18d4d9fd
//replace github.com/src-d/go-mysql-server => ../../go-mysql-server
replace vitess.io/vitess => github.com/liquidata-inc/vitess v0.0.0-20200318153456-e0b079da3f54
replace vitess.io/vitess => github.com/liquidata-inc/vitess v0.0.0-20200407071440-54a487aaf7d9
go 1.13

37
go/go.sum
View File

@@ -361,14 +361,20 @@ github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200330231002-2ac5a85cf8d6 h1:iKET+xfMh3NaiiIbrMBLi+MJ9hwmm++7DBtPGfarf50=
github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200330231002-2ac5a85cf8d6/go.mod h1:TCTrDbzIA05e8zV3SW+nsjc1LCR58GRSOIcF32lJ+Qc=
github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200403150612-34c67410dcd4 h1:UIksBT7bRENT38ErKSz+auGLc7a5tDpCHwNhuajoJbU=
github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200403150612-34c67410dcd4/go.mod h1:TCTrDbzIA05e8zV3SW+nsjc1LCR58GRSOIcF32lJ+Qc=
github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200403171307-83ac7e7158e2 h1:l4mXLvgHMoihWuEqcmcJKEvQtAccHxhsKwkVn/okoxc=
github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200403171307-83ac7e7158e2/go.mod h1:TCTrDbzIA05e8zV3SW+nsjc1LCR58GRSOIcF32lJ+Qc=
github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200407175239-21fb18d4d9fd h1:SGGh7+XPqPYw3LaIK4VUvy/81Za1Y3p29lh4WDMtXh0=
github.com/liquidata-inc/go-mysql-server v0.5.1-0.20200407175239-21fb18d4d9fd/go.mod h1:xu1cUi3vfWVJZ/9mQl9f8sdfJGobnS7kIucM3lfWIPk=
github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0 h1:phMgajKClMUiIr+hF2LGt8KRuUa2Vd2GI1sNgHgSXoU=
github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0/go.mod h1:YC1rI9k5gx8D02ljlbxDfZe80s/iq8bGvaaQsvR+qxs=
github.com/liquidata-inc/mmap-go v1.0.3 h1:2LndAeAtup9rpvUmu4wZSYCsjCQ0Zpc+NqE+6+PnT7g=
github.com/liquidata-inc/mmap-go v1.0.3/go.mod h1:w0doE7jfkuDEZyxb/zD3VWnRaQBYx1uDTS816kH8HoY=
github.com/liquidata-inc/sqllogictest/go v0.0.0-20200320151923-b11801f10e15 h1:H3RwcYfzkdW4kFh7znTUopcX3XZqnFXm6pcmxSy0mNo=
github.com/liquidata-inc/sqllogictest/go v0.0.0-20200320151923-b11801f10e15/go.mod h1:kKRVtyuomkqz15YFRpS0OT8kpsU8y/F3jyiZtvALdKU=
github.com/liquidata-inc/vitess v0.0.0-20200318153456-e0b079da3f54 h1:LR/OEhgIYVQuo5a/lxr8Ps76AZ1FNWUgNANfKCA0XSQ=
github.com/liquidata-inc/vitess v0.0.0-20200318153456-e0b079da3f54/go.mod h1:vn/QvIl/1+N6+qjheejcLt8jmX2kQSQwFinzZuoY1VY=
github.com/liquidata-inc/vitess v0.0.0-20200407071440-54a487aaf7d9 h1:eaE6IFxMviaDSNFaKlTbNPA/+0Vhj/XgV6lG2SaoAWM=
github.com/liquidata-inc/vitess v0.0.0-20200407071440-54a487aaf7d9/go.mod h1:vn/QvIl/1+N6+qjheejcLt8jmX2kQSQwFinzZuoY1VY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
@@ -387,11 +393,14 @@ github.com/mattn/go-runewidth v0.0.1/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1/go.mod h1:vuvdOZLJuf5HmJAJrKV64MmozrSsk+or0PB5dzdfspg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -419,11 +428,14 @@ github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/uuid v0.0.0-20160824210600-b984ec7fa9ff/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
github.com/pilosa/pilosa v1.4.0/go.mod h1:NSTtTprtb5MSgCs4mcNqeQ2JdIMpInOi4DEImxGJeTs=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -441,10 +453,12 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_golang v1.4.1 h1:FFSuS004yOQEtDdTq+TAOLP5xUq63KqAFYyOi8zA+Y8=
github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
@@ -452,6 +466,7 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -459,9 +474,11 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -471,9 +488,12 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sanity-io/litter v1.2.0 h1:DGJO0bxH/+C2EukzOSBmAlxmkhVMGqzvcx/rvySYw9M=
github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4=
github.com/satori/go.uuid v0.0.0-20160713180306-0aa62d5ddceb/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -496,6 +516,7 @@ github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -514,6 +535,7 @@ github.com/src-d/go-oniguruma v1.1.0 h1:EG+Nm5n2JqWUaCjtM0NtutPxU7ZN5Tp50GWrrV8b
github.com/src-d/go-oniguruma v1.1.0/go.mod h1:chVbff8kcVtmrhxtZ3yBVLLquXbzCS6DrxQaAK/CeqM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -525,13 +547,18 @@ github.com/tchap/go-patricia v0.0.0-20160729071656-dd168db6051b/go.mod h1:bmLyhP
github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE=
github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8=
github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY=
github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR83gdUHXjRJvjoBh1yACsM=
github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-lib v2.0.0+incompatible h1:iMSCV0rmXEogjNWPh2D0xk9YVKvrtGoHJNe9ebLu/pw=
github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
@@ -543,6 +570,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/z-division/go-zookeeper v0.0.0-20190128072838-6d7457066b9b/go.mod h1:JNALoWa+nCXR8SmgLluHcBNVJgyejzpKPZk9pX2yXXE=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.3.0 h1:ew6uUIeJOo+qdUUv7LxFCUhtWmVv7ZV/Xuy4FAUsw2E=
@@ -558,6 +586,7 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo=
@@ -648,6 +677,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -792,6 +822,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@@ -818,8 +849,10 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
modernc.org/mathutil v1.1.0 h1:z3/dTcIoU+Ql+ovBW6FaGAAAFJL48ZqABhGYCkbRpKE=
modernc.org/mathutil v1.1.0/go.mod h1:Jip3gBlE32vJMsD4RJ5qzniC0pvRnCxrlwmKPZrrLXI=
modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/strutil v1.1.0 h1:+1/yCzZxY2pZwwrsbH+4T7BQMoLQ9QiBshRC9eicYsc=
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

9
go/libraries/doltcore/dbfactory/file.go
View File

@@ -43,7 +43,14 @@ type FileFactory struct {
// CreateDB creates an local filesys backed database
func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]string) (datas.Database, error) {
path := urlObj.Host + urlObj.Path
path, err := url.PathUnescape(urlObj.Path)
if err != nil {
return nil, err
}
path = filepath.FromSlash(path)
path = urlObj.Host + path
info, err := os.Stat(path)

3
go/libraries/doltcore/env/actions/commit.go vendored
View File

@@ -16,11 +16,10 @@ package actions
import (
"context"
"errors"
"sort"
"time"
"github.com/pkg/errors"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/diff"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/doltdb"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/env"

55
go/libraries/doltcore/env/environment.go vendored
View File

@@ -17,6 +17,7 @@ package env
import (
"context"
"crypto/tls"
"errors"
"fmt"
"os"
"path/filepath"
@@ -25,7 +26,6 @@ import (
"time"
"unicode"
"github.com/pkg/errors"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
@@ -136,7 +136,8 @@ func (dEnv *DoltEnv) HasDoltDir() bool {
}
func (dEnv *DoltEnv) HasDoltDataDir() bool {
return dEnv.hasDoltDataDir("./")
exists, isDir := dEnv.FS.Exists(dbfactory.DoltDataDir)
return exists && isDir
}
func (dEnv *DoltEnv) HasDoltTempTableDir() bool {
@@ -145,21 +146,27 @@ func (dEnv *DoltEnv) HasDoltTempTableDir() bool {
return ex
}
func mustAbs(dEnv *DoltEnv, path ...string) string {
absPath, err := dEnv.FS.Abs(filepath.Join(path...))
if err != nil {
panic(err)
}
return absPath
}
// GetDoltDir returns the path to the .dolt directory
func (dEnv *DoltEnv) GetDoltDir() string {
if !dEnv.HasDoltDataDir() {
panic("No dolt dir")
}
return filepath.Join("./", dbfactory.DoltDir)
return mustAbs(dEnv, dbfactory.DoltDir)
}
func (dEnv *DoltEnv) hasDoltDir(path string) bool {
exists, isDir := dEnv.FS.Exists(filepath.Join(path, dbfactory.DoltDir))
return exists && isDir
}
func (dEnv *DoltEnv) hasDoltDataDir(path string) bool {
exists, isDir := dEnv.FS.Exists(filepath.Join(path, dbfactory.DoltDataDir))
exists, isDir := dEnv.FS.Exists(mustAbs(dEnv, dbfactory.DoltDir))
return exists && isDir
}
@@ -249,19 +256,32 @@ func (dEnv *DoltEnv) InitRepoWithNoData(ctx context.Context, nbf *types.NomsBinF
}
func (dEnv *DoltEnv) createDirectories(dir string) (string, error) {
doltDir := filepath.Join(dir, dbfactory.DoltDir)
if dEnv.hasDoltDir(doltDir) {
absPath, err := dEnv.FS.Abs(dir)
if err != nil {
return "", err
}
exists, isDir := dEnv.FS.Exists(absPath)
if !exists {
return "", fmt.Errorf("'%s' does not exist so could not create '%s", absPath, dbfactory.DoltDataDir)
} else if !isDir {
return "", fmt.Errorf("'%s' exists but it's a file not a directory", absPath)
}
if dEnv.hasDoltDir(dir) {
return "", ErrPreexistingDoltDir
}
doltDataDir := filepath.Join(doltDir, dbfactory.DataDir)
err := dEnv.FS.MkDirs(doltDataDir)
absDataDir := filepath.Join(absPath, dbfactory.DoltDataDir)
err = dEnv.FS.MkDirs(absDataDir)
if err != nil {
return "", fmt.Errorf("unable to make directory %s within the working directory", dbfactory.DoltDataDir)
return "", fmt.Errorf("unable to make directory '%s', cause: %s", absDataDir, err.Error())
}
return doltDir, nil
return filepath.Join(absPath, dbfactory.DoltDir), nil
}
func (dEnv *DoltEnv) configureRepo(doltDir string) error {
@@ -329,6 +349,7 @@ func (dEnv *DoltEnv) initializeRepoState(ctx context.Context) error {
return ErrStateUpdate
}
dEnv.RSLoadErr = nil
return nil
}
@@ -657,7 +678,7 @@ func (dEnv *DoltEnv) FindCreds(credsDir, pubKeyOrId string) (string, error) {
return "", ErrNotACred
}
path := filepath.Join(credsDir, pubKeyOrId+creds.JWKFileExtension)
path := mustAbs(dEnv, credsDir, pubKeyOrId+creds.JWKFileExtension)
exists, isDir := dEnv.FS.Exists(path)
if isDir {
@@ -766,7 +787,7 @@ func (dEnv *DoltEnv) GetUserHomeDir() (string, error) {
}
func (dEnv *DoltEnv) TempTableFilesDir() string {
return filepath.Join(dEnv.GetDoltDir(), tempTablesDir)
return mustAbs(dEnv, dEnv.GetDoltDir(), tempTablesDir)
}
func (dEnv *DoltEnv) GetAllValidDocDetails() (docs []doltdb.DocDetails, err error) {

245
go/libraries/doltcore/env/multi_repo_env.go vendored Normal file
View File

@@ -0,0 +1,245 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package env
import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"strings"
"unicode"
"github.com/liquidata-inc/dolt/go/libraries/utils/earl"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/dbfactory"
"github.com/liquidata-inc/dolt/go/libraries/doltcore/doltdb"
"github.com/liquidata-inc/dolt/go/libraries/utils/filesys"
)
// EnvNameAndPath is a simple tuple of the name of an environment and the path to where it is on disk
type EnvNameAndPath struct {
// Name is the name of the environment and is used as the identifier when accessing a given environment
Name string
// Path is the path on disk to where the environment lives
Path string
}
// MultiRepoEnv is a type used to store multiple environments which can be retrieved by name
type MultiRepoEnv map[string]*DoltEnv
// AddEnv adds an environment to the MultiRepoEnv by name
func (mrEnv MultiRepoEnv) AddEnv(name string, dEnv *DoltEnv) {
mrEnv[name] = dEnv
}
// Iter iterates over all environments in the MultiRepoEnv
func (mrEnv MultiRepoEnv) Iter(cb func(name string, dEnv *DoltEnv) (stop bool, err error)) error {
for name, dEnv := range mrEnv {
stop, err := cb(name, dEnv)
if err != nil {
return err
}
if stop {
break
}
}
return nil
}
// GetWorkingRoots returns a map with entries for each environment name with a value equal to the working root
// for that environment
func (mrEnv MultiRepoEnv) GetWorkingRoots(ctx context.Context) (map[string]*doltdb.RootValue, error) {
roots := make(map[string]*doltdb.RootValue)
err := mrEnv.Iter(func(name string, dEnv *DoltEnv) (stop bool, err error) {
root, err := dEnv.WorkingRoot(ctx)
if err != nil {
return true, err
}
roots[name] = root
return false, nil
})
if err != nil {
return nil, err
}
return roots, err
}
func getRepoRootDir(path, pathSeparator string) string {
if pathSeparator != "/" {
path = strings.ReplaceAll(path, pathSeparator, "/")
}
// filepath.Clean does not work with cross platform paths. So can't test a windows path on a mac
tokens := strings.Split(path, "/")
for i := len(tokens) - 1; i >= 0; i-- {
if tokens[i] == "" {
tokens = append(tokens[:i], tokens[i+1:]...)
}
}
if len(tokens) == 0 {
return ""
}
if tokens[len(tokens)-1] == dbfactory.DataDir && tokens[len(tokens)-2] == dbfactory.DoltDir {
tokens = tokens[:len(tokens)-2]
}
if len(tokens) == 0 {
return ""
}
name := tokens[len(tokens)-1]
// handles drive letters. fine with a folder containing a colon having the default name
if strings.IndexRune(name, ':') != -1 {
return ""
}
return name
}
// DoltEnvAsMultiEnv returns a MultiRepoEnv which wraps the DoltEnv and names it based on the directory DoltEnv refers to
func DoltEnvAsMultiEnv(dEnv *DoltEnv) MultiRepoEnv {
dbName := "dolt"
u, err := earl.Parse(dEnv.urlStr)
if err == nil {
if u.Scheme == dbfactory.FileScheme {
path, err := url.PathUnescape(u.Path)
if err == nil {
path, err = dEnv.FS.Abs(path)
if err == nil {
dirName := getRepoRootDir(path, string(os.PathSeparator))
if dirName != "" {
dbName = dirToDBName(dirName)
}
}
}
}
}
mrEnv := make(MultiRepoEnv)
mrEnv.AddEnv(dbName, dEnv)
return mrEnv
}
// LoadMultiEnv takes a variable list of EnvNameAndPath objects loads each of the environments, and returns a new
// MultiRepoEnv
func LoadMultiEnv(ctx context.Context, hdp HomeDirProvider, fs filesys.Filesys, version string, envNamesAndPaths ...EnvNameAndPath) (MultiRepoEnv, error) {
nameToPath := make(map[string]string)
for _, nameAndPath := range envNamesAndPaths {
existingPath, ok := nameToPath[nameAndPath.Name]
if ok {
if existingPath == nameAndPath.Path {
continue
}
return nil, fmt.Errorf("databases at paths '%s' and '%s' both attempted to load with the name '%s'", existingPath, nameAndPath.Path, nameAndPath.Name)
}
nameToPath[nameAndPath.Name] = nameAndPath.Path
}
mrEnv := make(MultiRepoEnv)
for name, path := range nameToPath {
absPath, err := fs.Abs(path)
if err != nil {
return nil, err
}
fsForEnv, err := filesys.LocalFilesysWithWorkingDir(absPath)
if err != nil {
return nil, err
}
urlStr := earl.FileUrlFromPath(filepath.Join(absPath, dbfactory.DoltDataDir), os.PathSeparator)
dEnv := Load(ctx, hdp, fsForEnv, urlStr, version)
if dEnv.RSLoadErr != nil {
return nil, fmt.Errorf("error loading environment '%s' at path '%s': %s", name, absPath, dEnv.RSLoadErr.Error())
} else if dEnv.DBLoadError != nil {
return nil, fmt.Errorf("error loading environment '%s' at path '%s': %s", name, absPath, dEnv.DBLoadError.Error())
} else if dEnv.CfgLoadErr != nil {
return nil, fmt.Errorf("error loading environment '%s' at path '%s': %s", name, absPath, dEnv.CfgLoadErr.Error())
}
mrEnv.AddEnv(name, dEnv)
}
return mrEnv, nil
}
// LoadMultiEnvFromDir looks at each subfolder of the given path as a Dolt repository and attempts to return a MultiRepoEnv
// with initialized environments for each of those subfolder data repositories. subfolders whose name starts with '.' are
// skipped.
func LoadMultiEnvFromDir(ctx context.Context, hdp HomeDirProvider, fs filesys.Filesys, path, version string) (MultiRepoEnv, error) {
var envNamesAndPaths []EnvNameAndPath
err := fs.Iter(path, false, func(path string, size int64, isDir bool) (stop bool) {
if isDir {
dirName := filepath.Base(path)
if dirName[0] == '.' {
return false
}
name := dirToDBName(dirName)
envNamesAndPaths = append(envNamesAndPaths, EnvNameAndPath{Name: name, Path: path})
}
return false
})
if err != nil {
return nil, err
}
return LoadMultiEnv(ctx, hdp, fs, version, envNamesAndPaths...)
}
func dirToDBName(dirName string) string {
dbName := strings.TrimSpace(dirName)
dbName = strings.Map(func(r rune) rune {
if unicode.IsSpace(r) || r == '-' {
return '_'
}
return r
}, dbName)
newDBName := strings.ReplaceAll(dbName, "__", "_")
for dbName != newDBName {
dbName = newDBName
newDBName = strings.ReplaceAll(dbName, "__", "_")
}
return dbName
}

172
go/libraries/doltcore/env/multi_repo_env_test.go vendored Normal file
View File

@@ -0,0 +1,172 @@
// Copyright 2020 Liquidata, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package env
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/liquidata-inc/dolt/go/libraries/utils/earl"
"github.com/liquidata-inc/dolt/go/libraries/utils/filesys"
"github.com/liquidata-inc/dolt/go/libraries/utils/test"
"github.com/liquidata-inc/dolt/go/store/types"
)
func TestDirToDBName(t *testing.T) {
tests := map[string]string{
"irs": "irs",
"corona-virus": "corona_virus",
" fake - name ": "fake_name",
}
for dirName, expected := range tests {
actual := dirToDBName(dirName)
assert.Equal(t, expected, actual)
}
}
func TestGetRepoRootDir(t *testing.T) {
tests := []struct {
path string
sep string
expected string
}{
{``, `/`, ``},
{``, `\`, ``},
{`/`, `/`, ``},
{`C:\`, `\`, ``},
{`.dolt/noms`, `/`, ``},
{`.dolt\noms`, `\`, ``},
{`name/.dolt/noms`, `/`, `name`},
{`name\.dolt\noms`, `\`, `name`},
{`name/.dolt/noms/`, `/`, `name`},
{`name\.dolt\noms\`, `\`, `name`},
{`/var/folders/w6/mhtq880n6y55xxm3_2kn0bs80000gn/T/dolt-repo-76581`, `/`, `dolt-repo-76581`},
{`/var/folders/w6/mhtq880n6y55xxm3_2kn0bs80000gn/T/dolt-repo-76581/.dolt/noms`, `/`, `dolt-repo-76581`},
{`/Users/u/name/.dolt/noms`, `/`, `name`},
{`C:\files\name\.dolt\noms`, `\`, `name`},
{`/Users/u/name/.dolt/noms/`, `/`, `name`},
{`C:\files\name\.dolt\noms\`, `\`, `name`},
{`//Users////u//name//.dolt/noms/////`, `/`, `name`},
{`C:\\files\\\\name\\.dolt\noms\\\\\\`, `\`, `name`},
}
for _, test := range tests {
actual := getRepoRootDir(test.path, test.sep)
assert.Equal(t, test.expected, actual, "For '%s' expected '%s' got '%s'", test.path, test.expected, actual)
}
}
func initRepoWithRelativePath(t *testing.T, envPath string, hdp HomeDirProvider) *DoltEnv {
err := filesys.LocalFS.MkDirs(envPath)
require.NoError(t, err)
fs, err := filesys.LocalFilesysWithWorkingDir(envPath)
require.NoError(t, err)
urlStr := earl.FileUrlFromPath(filepath.Join(envPath, ".dolt", "noms"), os.PathSeparator)
dEnv := Load(context.Background(), hdp, fs, urlStr, "test")
cfg, _ := dEnv.Config.GetConfig(GlobalConfig)
cfg.SetStrings(map[string]string{
UserNameKey: name,
UserEmailKey: email,
})
err = dEnv.InitRepo(context.Background(), types.Format_7_18, name, email)
require.NoError(t, err)
return dEnv
}
func TestDoltEnvAsMultiEnv(t *testing.T) {
rootPath, err := test.ChangeToTestDir("TestDoltEnvAsMultiEnv")
require.NoError(t, err)
hdp := func() (string, error) { return rootPath, nil }
envPath := filepath.Join(rootPath, " test---name _ 123")
dEnv := initRepoWithRelativePath(t, envPath, hdp)
mrEnv := DoltEnvAsMultiEnv(dEnv)
assert.Len(t, mrEnv, 1)
for k, v := range mrEnv {
assert.Equal(t, "test_name_123", k)
assert.Equal(t, dEnv, v)
}
}
func initMultiEnv(t *testing.T, testName string, names []string) (string, HomeDirProvider, map[string]*DoltEnv) {
rootPath, err := test.ChangeToTestDir(testName)
require.NoError(t, err)
hdp := func() (string, error) { return rootPath, nil }
envs := make(map[string]*DoltEnv)
for _, name := range names {
envPath := filepath.Join(rootPath, name)
envs[name] = initRepoWithRelativePath(t, envPath, hdp)
}
return rootPath, hdp, envs
}
func TestLoadMultiEnv(t *testing.T) {
names := []string{"env 1", " env 2", "env-3"}
rootPath, hdp, _ := initMultiEnv(t, "TestLoadMultiEnv", names)
envNamesAndPaths := make([]EnvNameAndPath, len(names))
for i, name := range names {
envNamesAndPaths[i] = EnvNameAndPath{name, filepath.Join(rootPath, name)}
}
mrEnv, err := LoadMultiEnv(context.Background(), hdp, filesys.LocalFS, "test", envNamesAndPaths...)
require.NoError(t, err)
for _, name := range names {
_, ok := mrEnv[name]
assert.True(t, ok)
}
}
func TestLoadMultiEnvFromDir(t *testing.T) {
dirNameToDBName := map[string]string{
"env 1": "env_1",
" env 2": "env_2",
"env-3": "env_3",
}
names := make([]string, 0, len(dirNameToDBName))
for k := range dirNameToDBName {
names = append(names, k)
}
rootPath, hdp, envs := initMultiEnv(t, "TestLoadMultiEnvFromDir", names)
mrEnv, err := LoadMultiEnvFromDir(context.Background(), hdp, filesys.LocalFS, rootPath, "test")
require.NoError(t, err)
assert.Len(t, mrEnv, len(names))
for _, dirName := range names {
dbName := dirNameToDBName[dirName]
_, ok := envs[dirName]
require.True(t, ok)
_, ok = mrEnv[dbName]
require.True(t, ok)
}
}

6
go/libraries/doltcore/sqle/database.go
View File

@@ -146,10 +146,16 @@ func (db Database) Name() string {
return db.name
}
// GetDefaultRoot returns the default root of the database that is used by new sessions.
func (db Database) GetDefaultRoot() *doltdb.RootValue {
return db.defRoot
}
// GetDoltDB gets the underlying DoltDB of the Database
func (db Database) GetDoltDB() *doltdb.DoltDB {
return db.ddb
}
// GetTableInsensitive is used when resolving tables in queries. It returns a best-effort case-insensitive match for
// the table name given.
func (db Database) GetTableInsensitive(ctx *sql.Context, tblName string) (sql.Table, bool, error) {

18
go/libraries/doltcore/sqle/indexes.go
View File

@@ -29,11 +29,16 @@ import (
// IndexDriver implementation. Not ready for prime time.
type DoltIndexDriver struct {
db Database
dbs map[string]Database
}
func NewDoltIndexDriver(database Database) *DoltIndexDriver {
return &DoltIndexDriver{database}
func NewDoltIndexDriver(dbs ...Database) *DoltIndexDriver {
nameToDB := make(map[string]Database)
for _, db := range dbs {
nameToDB[db.Name()] = db
}
return &DoltIndexDriver{nameToDB}
}
func (*DoltIndexDriver) ID() string {
@@ -53,11 +58,12 @@ func (i *DoltIndexDriver) Delete(sql.Index, sql.PartitionIter) error {
}
func (i *DoltIndexDriver) LoadAll(ctx *sql.Context, db, table string) ([]sql.Index, error) {
if db != i.db.name {
database, ok := i.dbs[db]
if !ok {
panic("Unexpected db: " + db)
}
root, err := i.db.GetRoot(ctx)
root, err := database.GetRoot(ctx)
if err != nil {
return nil, err
@@ -79,7 +85,7 @@ func (i *DoltIndexDriver) LoadAll(ctx *sql.Context, db, table string) ([]sql.Ind
return nil, err
}
return []sql.Index{&doltIndex{sch, table, i.db, i}}, nil
return []sql.Index{&doltIndex{sch, table, database, i}}, nil
}
type doltIndex struct {

13
go/libraries/doltcore/sqle/logictest/dolt/doltharness.go
View File

@@ -140,17 +140,19 @@ func innerInit(h *DoltHarness, dEnv *env.DoltEnv) error {
sql.WithViewRegistry(h.viewReg),
sql.WithSession(h.sess))
for _, db := range h.engine.Catalog.AllDatabases() {
dbs := h.engine.Catalog.AllDatabases()
dsqlDBs := make([]dsql.Database, len(dbs))
for i, db := range dbs {
dsqlDB := db.(dsql.Database)
defRoot := dsqlDB.GetDefaultRoot()
dsqlDBs[i] = dsqlDB
defRoot := dsqlDB.GetDefaultRoot()
err := dsqlDB.SetRoot(ctx, defRoot)
if err != nil {
return err
}
ctx.RegisterIndexDriver(dsql.NewDoltIndexDriver(dsqlDB))
err = dsql.RegisterSchemaFragments(ctx, dsqlDB, defRoot)
if err != nil {
@@ -158,8 +160,13 @@ func innerInit(h *DoltHarness, dEnv *env.DoltEnv) error {
}
}
ctx.RegisterIndexDriver(dsql.NewDoltIndexDriver(dsqlDBs...))
err = ctx.LoadIndexes(ctx, h.engine.Catalog.AllDatabases())
if len(dbs) == 1 {
h.sess.SetCurrentDatabase(dbs[0].Name())
}
if err != nil {
return err
}

7
go/libraries/doltcore/sqle/testutil.go
View File

@@ -88,12 +88,14 @@ func ExecuteSql(dEnv *env.DoltEnv, root *doltdb.RootValue, statements string) (*
// NewTestSQLCtx returns a new *sql.Context with a default DoltSession, a new IndexRegistry, and a new ViewRegistry
func NewTestSQLCtx(ctx context.Context) *sql.Context {
return sql.NewContext(
sqlCtx := sql.NewContext(
ctx,
sql.WithSession(DefaultDoltSession()),
sql.WithIndexRegistry(sql.NewIndexRegistry()),
sql.WithViewRegistry(sql.NewViewRegistry()),
)
).WithCurrentDB("dolt")
return sqlCtx
}
// NewTestEngine creates a new default engine, and a *sql.Context and initializes indexes and schema fragments.
@@ -102,6 +104,7 @@ func NewTestEngine(ctx context.Context, db Database, root *doltdb.RootValue) (*s
engine.AddDatabase(db)
sqlCtx := NewTestSQLCtx(ctx)
sqlCtx.SetCurrentDatabase(db.Name())
err := db.SetRoot(sqlCtx, root)
if err != nil {

3
go/libraries/utils/argparser/option.go
View File

@@ -15,9 +15,8 @@
package argparser
import (
"errors"
"strconv"
"github.com/pkg/errors"
)
type OptionType int

3
go/libraries/utils/argparser/parser.go
View File

@@ -15,9 +15,8 @@
package argparser
import (
"errors"
"strings"
"github.com/pkg/errors"
)
const (

23
go/libraries/utils/earl/url.go
View File

@@ -59,11 +59,17 @@ func Parse(urlStr string) (*url.URL, error) {
return nil, err
}
// url.parse doesn't handle file paths that begin with . correctly
if u.Scheme == "file" && strings.HasPrefix(u.Host, ".") {
u.Path = u.Host + u.Path
u.Host = ""
}
// if Path is e.g. "/C$/" for a network location, it should instead be "C:/"
if len(u.Path) >= 3 && u.Path[0] == '/' && u.Path[1] >= 'A' && u.Path[1] <= 'Z' && u.Path[2] == '$' {
u.Path = u.Path[1:2] + ":" + u.Path[3:]
} else if !osutil.StartsWithWindowsVolume(u.Path) { // normalize some
if len(u.Path) == 0 || u.Path[0] != '/' {
if len(u.Path) == 0 || (u.Path[0] != '/' && u.Path[0] != '.') {
u.Path = "/" + u.Path
}
}
@@ -92,3 +98,18 @@ func parse(urlStr string) (*url.URL, error) {
return url.Parse(urlStr)
}
// FileUrlFromPath returns a url for the given path with the "file" scheme i.e. file://...
func FileUrlFromPath(path string, separator rune) string {
if osutil.StartsWithWindowsVolume(path) {
path = "/" + path
}
if separator != '/' {
path = strings.ReplaceAll(path, string(separator), "/")
}
u := &url.URL{Scheme: "file", Path: path}
urlStr := u.String()
return urlStr
}

63
go/libraries/utils/earl/url_test.go
View File

@@ -16,8 +16,9 @@ package earl
import (
"net/url"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
func TestParse(t *testing.T) {
@@ -224,6 +225,62 @@ func TestParse(t *testing.T) {
},
false,
},
{
FileUrlFromPath(`C:\Users\name\datasets`, '\\'),
url.URL{
Scheme: "file",
Path: "C:/Users/name/datasets",
},
false,
},
{
FileUrlFromPath(`./.dolt/noms`, '/'),
url.URL{
Scheme: "file",
Path: "./.dolt/noms",
},
false,
},
{
FileUrlFromPath(`./.dolt\noms`, '\\'),
url.URL{
Scheme: "file",
Path: "./.dolt/noms",
},
false,
},
{
FileUrlFromPath(`.dolt/noms`, '/'),
url.URL{
Scheme: "file",
Path: ".dolt/noms",
},
false,
},
{
FileUrlFromPath(`.dolt\noms`, '\\'),
url.URL{
Scheme: "file",
Path: ".dolt/noms",
},
false,
},
{
FileUrlFromPath(`./test/.dolt/noms`, '/'),
url.URL{
Scheme: "file",
Path: "./test/.dolt/noms",
},
false,
},
{
FileUrlFromPath(`./test\.dolt\noms`, '\\'),
url.URL{
Scheme: "file",
Path: "./test/.dolt/noms",
},
false,
},
}
for _, test := range tests {
@@ -231,8 +288,8 @@ func TestParse(t *testing.T) {
if (err != nil) != test.expectErr {
t.Error("input:", test.urlStr, "got error:", err != nil, "expected error:", test.expectErr, "result:", actualUrl, "err:", err)
} else if err == nil && !reflect.DeepEqual(actualUrl, &test.expectedUrl) {
t.Error(actualUrl, "!=", &test.expectedUrl)
} else if err == nil {
assert.Equal(t, &test.expectedUrl, actualUrl)
}
}
}

3
go/libraries/utils/filesys/fs.go
View File

@@ -16,10 +16,9 @@ package filesys
import (
"encoding/json"
"errors"
"io"
"time"
"github.com/pkg/errors"
)
var ErrIsDir = errors.New("operation not valid on a directory")

116
go/libraries/utils/filesys/localfs.go
View File

@@ -16,6 +16,7 @@ package filesys
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
@@ -26,10 +27,39 @@ import (
// LocalFS is the machines local filesystem
var LocalFS = &localFS{}
type localFS struct{}
type localFS struct {
cwd string
}
// LocalFilesysWithWorkingDir returns a new Filesys implementation backed by the local filesystem with the supplied
// working directory. Path relative operations occur relative to this directory.
func LocalFilesysWithWorkingDir(cwd string) (Filesys, error) {
absCWD, err := filepath.Abs(cwd)
if err != nil {
return nil, err
}
stat, err := os.Stat(absCWD)
if err != nil {
return nil, err
} else if !stat.IsDir() {
return nil, fmt.Errorf("'%s' is not a valid directory", absCWD)
}
return &localFS{absCWD}, nil
}
// Exists will tell you if a file or directory with a given path already exists, and if it does is it a directory
func (fs *localFS) Exists(path string) (exists bool, isDir bool) {
var err error
path, err = fs.Abs(path)
if err != nil {
return false, false
}
stat, err := os.Stat(path)
if err != nil {
@@ -43,6 +73,13 @@ var errStopMarker = errors.New("stop")
// Iter iterates over the files and subdirectories within a given directory (Optionally recursively.
func (fs *localFS) Iter(path string, recursive bool, cb FSIterCB) error {
var err error
path, err = fs.Abs(path)
if err != nil {
return err
}
if !recursive {
info, err := ioutil.ReadDir(path)
@@ -65,7 +102,14 @@ func (fs *localFS) Iter(path string, recursive bool, cb FSIterCB) error {
}
func (fs *localFS) iter(dir string, cb FSIterCB) error {
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
var err error
dir, err = fs.Abs(dir)
if err != nil {
return err
}
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if dir != path {
stop := cb(path, info.Size(), info.IsDir())
@@ -85,6 +129,13 @@ func (fs *localFS) iter(dir string, cb FSIterCB) error {
// OpenForRead opens a file for reading
func (fs *localFS) OpenForRead(fp string) (io.ReadCloser, error) {
var err error
fp, err = fs.Abs(fp)
if err != nil {
return nil, err
}
if exists, isDir := fs.Exists(fp); !exists {
return nil, os.ErrNotExist
} else if isDir {
@@ -96,24 +147,52 @@ func (fs *localFS) OpenForRead(fp string) (io.ReadCloser, error) {
// ReadFile reads the entire contents of a file
func (fs *localFS) ReadFile(fp string) ([]byte, error) {
var err error
fp, err = fs.Abs(fp)
if err != nil {
return nil, err
}
return ioutil.ReadFile(fp)
}
// OpenForWrite opens a file for writing. The file will be created if it does not exist, and if it does exist
// it will be overwritten.
func (fs *localFS) OpenForWrite(fp string) (io.WriteCloser, error) {
var err error
fp, err = fs.Abs(fp)
if err != nil {
return nil, err
}
return os.OpenFile(fp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
}
// WriteFile writes the entire data buffer to a given file. The file will be created if it does not exist,
// and if it does exist it will be overwritten.
func (fs *localFS) WriteFile(fp string, data []byte) error {
var err error
fp, err = fs.Abs(fp)
if err != nil {
return err
}
return ioutil.WriteFile(fp, data, os.ModePerm)
}
// MkDirs creates a folder and all the parent folders that are necessary to create it.
func (fs *localFS) MkDirs(path string) error {
_, err := os.Stat(path)
var err error
path, err = fs.Abs(path)
if err != nil {
return err
}
_, err = os.Stat(path)
if err != nil {
return os.MkdirAll(path, os.ModePerm)
@@ -124,6 +203,13 @@ func (fs *localFS) MkDirs(path string) error {
// DeleteFile will delete a file at the given path
func (fs *localFS) DeleteFile(path string) error {
var err error
path, err = fs.Abs(path)
if err != nil {
return err
}
if exists, isDir := fs.Exists(path); exists && !isDir {
if isDir {
return ErrIsDir
@@ -138,6 +224,13 @@ func (fs *localFS) DeleteFile(path string) error {
// Delete will delete an empty directory, or a file. If trying delete a directory that is not empty you can set force to
// true in order to delete the dir and all of it's contents
func (fs *localFS) Delete(path string, force bool) error {
var err error
path, err = fs.Abs(path)
if err != nil {
return err
}
if !force {
return os.Remove(path)
} else {
@@ -165,11 +258,26 @@ func (fs *localFS) MoveFile(srcPath, destPath string) error {
// converts a path to an absolute path. If it's already an absolute path the input path will be returned unaltered
func (fs *localFS) Abs(path string) (string, error) {
return filepath.Abs(path)
if filepath.IsAbs(path) {
return path, nil
}
if fs.cwd == "" {
return filepath.Abs(path)
} else {
return filepath.Join(fs.cwd, path), nil
}
}
// LastModified gets the last modified timestamp for a file or directory at a given path
func (fs *localFS) LastModified(path string) (t time.Time, exists bool) {
var err error
path, err = fs.Abs(path)
if err != nil {
return time.Time{}, false
}
stat, err := os.Stat(path)
if err != nil {

2
go/libraries/utils/filesys/lock.go
View File

@@ -15,10 +15,10 @@
package filesys
import (
"errors"
"sync/atomic"
"github.com/juju/fslock"
"github.com/pkg/errors"
)
const unlockedStateValue int32 = 0