mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-04-28 20:49:43 -05:00
Code Issues Packages Projects Releases 159 Wiki Activity

Merge remote-tracking branch 'origin/main' into andy/new-format-query-plans

Browse Source
This commit is contained in:
Andy Arthur
2022-06-10 12:47:09 -07:00
parent c7d8ee7f16 92050790f4
commit 7991df4bc8
14 changed files with 306 additions and 440 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/cmd/dolt/commands/engine/sqlengine.go
+25 -47
View File
@@ -25,7 +25,6 @@ import (
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/go-mysql-server/sql/analyzer"
"github.com/dolthub/go-mysql-server/sql/information_schema"
"github.com/dolthub/go-mysql-server/sql/mysql_db"
"github.com/dolthub/vitess/go/vt/sqlparser"
"github.com/dolthub/dolt/go/cmd/dolt/cli"
@@ -48,18 +47,22 @@ type SqlEngine struct {
resultFormat PrintResultFormat
}
type SqlEngineConfig struct {
InitialDb string
IsReadOnly bool
PrivFilePath string
ServerUser string
ServerPass string
Autocommit bool
}
// NewSqlEngine returns a SqlEngine
func NewSqlEngine(
ctx context.Context,
mrEnv *env.MultiRepoEnv,
format PrintResultFormat,
initialDb string,
isReadOnly bool,
mysqlDbFilePath string,
privFilePath string,
serverUser string,
serverPass string,
autocommit bool) (*SqlEngine, error) {
config *SqlEngineConfig,
) (*SqlEngine, error) {
parallelism := runtime.GOMAXPROCS(0)
@@ -80,54 +83,29 @@ func NewSqlEngine(
b := env.GetDefaultInitBranch(mrEnv.Config())
pro := dsqle.NewDoltDatabaseProvider(b, mrEnv.FileSystem(), all...)
// Set mysql.db file path from server
mysql_file_handler.SetMySQLDbFilePath(mysqlDbFilePath)
// Load in MySQL Db from file, if it exists
data, err := mysql_file_handler.LoadData()
// Load in privileges from file, if it exists
persister := mysql_file_handler.NewPersister(config.PrivFilePath)
data, err := persister.LoadData()
if err != nil {
return nil, err
}
// Use privilege file iff mysql.db file DNE
var users []*mysql_db.User
var roles []*mysql_db.RoleEdge
// Create temporary users if no privileges in config
var tempUsers []gms.TemporaryUser
if len(data) == 0 {
// Set privilege file path from server
if privFilePath != "" {
mysql_file_handler.SetPrivilegeFilePath(privFilePath)
}
// Load privileges from privilege file
users, roles, err = mysql_file_handler.LoadPrivileges()
if err != nil {
return nil, err
}
// Create temporary users if no privileges in config
if len(users) == 0 && len(serverUser) > 0 {
tempUsers = append(tempUsers, gms.TemporaryUser{
Username: serverUser,
Password: serverPass,
})
}
if len(data) == 0 && len(config.ServerUser) > 0 {
tempUsers = append(tempUsers, gms.TemporaryUser{
Username: config.ServerUser,
Password: config.ServerPass,
})
}
// Set up engine
engine := gms.New(analyzer.NewBuilder(pro).WithParallelism(parallelism).Build(), &gms.Config{IsReadOnly: isReadOnly, TemporaryUsers: tempUsers}).WithBackgroundThreads(bThreads)
engine := gms.New(analyzer.NewBuilder(pro).WithParallelism(parallelism).Build(), &gms.Config{IsReadOnly: config.IsReadOnly, TemporaryUsers: tempUsers}).WithBackgroundThreads(bThreads)
engine.Analyzer.Catalog.MySQLDb.SetPersister(persister)
// Load MySQL Db information
if err = engine.Analyzer.Catalog.MySQLDb.LoadData(sql.NewEmptyContext(), data); err != nil {
return nil, err
}
// Load Privilege data iff mysql db didn't exist
if len(data) == 0 {
if err = engine.Analyzer.Catalog.MySQLDb.LoadPrivilegeData(sql.NewEmptyContext(), users, roles); err != nil {
return nil, err
}
}
// Set persist callbacks
engine.Analyzer.Catalog.MySQLDb.SetPersistCallback(mysql_file_handler.SaveData)
if dbg, ok := os.LookupEnv("DOLT_SQL_DEBUG_LOG"); ok && strings.ToLower(dbg) == "true" {
engine.Analyzer.Debug = true
@@ -160,15 +138,15 @@ func NewSqlEngine(
}
// TODO: this should just be the session default like it is with MySQL
err = sess.SetSessionVariable(sql.NewContext(ctx), sql.AutoCommitSessionVar, autocommit)
err = sess.SetSessionVariable(sql.NewContext(ctx), sql.AutoCommitSessionVar, config.Autocommit)
if err != nil {
return nil, err
}
return &SqlEngine{
dbs: nameToDB,
contextFactory: newSqlContext(sess, initialDb),
dsessFactory: newDoltSession(pro, mrEnv.Config(), autocommit),
contextFactory: newSqlContext(sess, config.InitialDb),
dsessFactory: newDoltSession(pro, mrEnv.Config(), config.Autocommit),
engine: engine,
resultFormat: format,
}, nil
go/cmd/dolt/commands/sql.go
+68 -49
View File
@@ -82,17 +82,18 @@ By default this command uses the dolt database in the current working directory,
}
const (
QueryFlag = "query"
FormatFlag = "result-format"
saveFlag = "save"
executeFlag = "execute"
listSavedFlag = "list-saved"
messageFlag = "message"
BatchFlag = "batch"
multiDBDirFlag = "multi-db-dir"
continueFlag = "continue"
fileInputFlag = "file"
welcomeMsg = `# Welcome to the DoltSQL shell.
QueryFlag = "query"
FormatFlag = "result-format"
saveFlag = "save"
executeFlag = "execute"
listSavedFlag = "list-saved"
messageFlag = "message"
BatchFlag = "batch"
multiDBDirFlag = "multi-db-dir"
continueFlag = "continue"
fileInputFlag = "file"
privilegeFilePathFlag = "privilege-file"
welcomeMsg = `# Welcome to the DoltSQL shell.
# Statements must be terminated with ';'.
# "exit" or "quit" (or Ctrl-D) to exit.`
)
@@ -143,6 +144,7 @@ func (cmd SqlCmd) ArgParser() *argparser.ArgParser {
ap.SupportsString(multiDBDirFlag, "", "directory", "Defines a directory whose subdirectories should all be dolt data repositories accessible as independent databases within ")
ap.SupportsFlag(continueFlag, "c", "continue running queries on an error. Used for batch mode only.")
ap.SupportsString(fileInputFlag, "", "input file", "Execute statements from the file given")
ap.SupportsString(privilegeFilePathFlag, "", "Privilege File", "Points to the file that privileges will be loaded from, in addition to being overwritten when privileges have been modified.")
return ap
}
@@ -172,6 +174,8 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
}
privsFp, _ := apr.GetValue(privilegeFilePathFlag)
// We need a username and password for many SQL commands, so set defaults if they don't exist
dEnv.Config.SetFailsafes(env.DefaultFailsafeConfig)
@@ -204,9 +208,9 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
if query, queryOK := apr.GetValue(QueryFlag); queryOK {
return queryMode(ctx, mrEnv, initialRoots, apr, query, currentDb, format, usage)
} else if savedQueryName, exOk := apr.GetValue(executeFlag); exOk {
return savedQueryMode(ctx, mrEnv, initialRoots, savedQueryName, currentDb, format, usage)
return savedQueryMode(ctx, mrEnv, initialRoots, savedQueryName, currentDb, format, usage, privsFp)
} else if apr.Contains(listSavedFlag) {
return listSavedQueriesMode(ctx, mrEnv, initialRoots, currentDb, format, usage)
return listSavedQueriesMode(ctx, mrEnv, initialRoots, currentDb, format, usage, privsFp)
} else {
// Run in either batch mode for piped input, or shell mode for interactive
isTty := false
@@ -233,17 +237,17 @@ func (cmd SqlCmd) Exec(ctx context.Context, commandStr string, args []string, dE
}
if isTty {
verr := execShell(ctx, mrEnv, format, currentDb)
verr := execShell(ctx, mrEnv, format, currentDb, privsFp)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
}
} else if runInBatchMode {
verr := execBatch(ctx, continueOnError, mrEnv, input, format, currentDb)
verr := execBatch(ctx, continueOnError, mrEnv, input, format, currentDb, privsFp)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
}
} else {
verr := execMultiStatements(ctx, continueOnError, mrEnv, input, format, currentDb)
verr := execMultiStatements(ctx, continueOnError, mrEnv, input, format, currentDb, privsFp)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
}
@@ -260,6 +264,7 @@ func listSavedQueriesMode(
currentDb string,
format engine.PrintResultFormat,
usage cli.UsagePrinter,
privsFp string,
) int {
hasQC, err := initialRoots[currentDb].HasTable(ctx, doltdb.DoltQueryCatalogTableName)
@@ -273,7 +278,7 @@ func listSavedQueriesMode(
}
query := "SELECT * FROM " + doltdb.DoltQueryCatalogTableName
return HandleVErrAndExitCode(execQuery(ctx, mrEnv, query, format, currentDb), usage)
return HandleVErrAndExitCode(execQuery(ctx, mrEnv, query, format, currentDb, privsFp), usage)
}
func savedQueryMode(
@@ -284,6 +289,7 @@ func savedQueryMode(
currentDb string,
format engine.PrintResultFormat,
usage cli.UsagePrinter,
privsFp string,
) int {
sq, err := dtables.RetrieveFromQueryCatalog(ctx, initialRoots[currentDb], savedQueryName)
@@ -292,7 +298,7 @@ func savedQueryMode(
}
cli.PrintErrf("Executing saved query '%s':\n%s\n", savedQueryName, sq.Query)
return HandleVErrAndExitCode(execQuery(ctx, mrEnv, sq.Query, format, currentDb), usage)
return HandleVErrAndExitCode(execQuery(ctx, mrEnv, sq.Query, format, currentDb, privsFp), usage)
}
func queryMode(
@@ -315,8 +321,9 @@ func queryMode(
_, continueOnError := apr.GetValue(continueFlag)
privsFp, _ := apr.GetValue(privilegeFilePathFlag)
if saveName != "" {
verr := execQuery(ctx, mrEnv, query, format, currentDb)
verr := execQuery(ctx, mrEnv, query, format, currentDb, privsFp)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
}
@@ -335,13 +342,13 @@ func queryMode(
}
} else if batchMode {
batchInput := strings.NewReader(query)
verr := execBatch(ctx, continueOnError, mrEnv, batchInput, format, currentDb)
verr := execBatch(ctx, continueOnError, mrEnv, batchInput, format, currentDb, privsFp)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
}
} else {
input := strings.NewReader(query)
verr := execMultiStatements(ctx, continueOnError, mrEnv, input, format, currentDb)
verr := execMultiStatements(ctx, continueOnError, mrEnv, input, format, currentDb, privsFp)
if verr != nil {
return HandleVErrAndExitCode(verr, usage)
}
@@ -376,18 +383,21 @@ func execShell(
mrEnv *env.MultiRepoEnv,
format engine.PrintResultFormat,
initialDb string,
privsFp string,
) errhand.VerboseError {
config := &engine.SqlEngineConfig{
InitialDb: initialDb,
IsReadOnly: false,
PrivFilePath: privsFp,
ServerUser: "root",
ServerPass: "",
Autocommit: true,
}
se, err := engine.NewSqlEngine(
ctx,
mrEnv,
format,
initialDb,
false,
"",
"",
"root",
"",
true,
config,
)
if err != nil {
return errhand.VerboseErrorFromError(err)
@@ -408,18 +418,21 @@ func execBatch(
batchInput io.Reader,
format engine.PrintResultFormat,
initialDb string,
privsFp string,
) errhand.VerboseError {
config := &engine.SqlEngineConfig{
InitialDb: initialDb,
IsReadOnly: false,
PrivFilePath: privsFp,
ServerUser: "root",
ServerPass: "",
Autocommit: true,
}
se, err := engine.NewSqlEngine(
ctx,
mrEnv,
format,
initialDb,
false,
"",
"",
"root",
"",
false,
config,
)
if err != nil {
return errhand.VerboseErrorFromError(err)
@@ -457,18 +470,21 @@ func execMultiStatements(
batchInput io.Reader,
format engine.PrintResultFormat,
initialDb string,
privsFp string,
) errhand.VerboseError {
config := &engine.SqlEngineConfig{
InitialDb: initialDb,
IsReadOnly: false,
PrivFilePath: privsFp,
ServerUser: "root",
ServerPass: "",
Autocommit: true,
}
se, err := engine.NewSqlEngine(
ctx,
mrEnv,
format,
initialDb,
false,
"",
"",
"root",
"",
true,
config,
)
if err != nil {
return errhand.VerboseErrorFromError(err)
@@ -493,18 +509,21 @@ func execQuery(
query string,
format engine.PrintResultFormat,
initialDb string,
privsFp string,
) errhand.VerboseError {
config := &engine.SqlEngineConfig{
InitialDb: initialDb,
IsReadOnly: false,
PrivFilePath: privsFp,
ServerUser: "root",
ServerPass: "",
Autocommit: true,
}
se, err := engine.NewSqlEngine(
ctx,
mrEnv,
format,
initialDb,
false,
"",
"",
"root",
"",
true,
config,
)
if err != nil {
return errhand.VerboseErrorFromError(err)
go/cmd/dolt/commands/sqlserver/server.go
+9 -7
View File
@@ -166,17 +166,19 @@ func Serve(
serverConf.RequireSecureTransport = serverConfig.RequireSecureTransport()
// Create SQL Engine with users
config := &engine.SqlEngineConfig{
InitialDb: "",
IsReadOnly: isReadOnly,
PrivFilePath: serverConfig.PrivilegeFilePath(),
ServerUser: serverConfig.User(),
ServerPass: serverConfig.Password(),
Autocommit: serverConfig.AutoCommit(),
}
sqlEngine, err := engine.NewSqlEngine(
ctx,
mrEnv,
engine.FormatTabular,
"",
isReadOnly,
serverConfig.MySQLDbFilePath(),
serverConfig.PrivilegeFilePath(),
serverConfig.User(),
serverConfig.Password(),
serverConfig.AutoCommit(),
config,
)
if err != nil {
return err, nil
go/cmd/dolt/commands/sqlserver/serverconfig.go
-11
View File
@@ -49,8 +49,6 @@ const (
defaultDataDir = "."
defaultMetricsHost = ""
defaultMetricsPort = -1
defaultMySQLDbFilePath = "mysql.db"
defaultPrivilegeFilePath = "privs.json"
)
const (
@@ -127,8 +125,6 @@ type ServerConfig interface {
// PrivilegeFilePath returns the path to the file which contains all needed privilege information in the form of a
// JSON string.
PrivilegeFilePath() string
// MySQLDbFilePath returns the path to the file which contains the information for a MySQL db.
MySQLDbFilePath() string
}
type commandLineServerConfig struct {
@@ -149,7 +145,6 @@ type commandLineServerConfig struct {
requireSecureTransport bool
persistenceBehavior string
privilegeFilePath string
mysqlDbFilePath string
}
var _ ServerConfig = (*commandLineServerConfig)(nil)
@@ -246,10 +241,6 @@ func (cfg *commandLineServerConfig) PrivilegeFilePath() string {
return cfg.privilegeFilePath
}
func (cfg *commandLineServerConfig) MySQLDbFilePath() string {
return cfg.mysqlDbFilePath
}
// DatabaseNamesAndPaths returns an array of env.EnvNameAndPathObjects corresponding to the databases to be loaded in
// a multiple db configuration. If nil is returned the server will look for a database in the current directory and
// give it a name automatically.
@@ -351,8 +342,6 @@ func DefaultServerConfig() *commandLineServerConfig {
queryParallelism: defaultQueryParallelism,
persistenceBehavior: defaultPersistenceBahavior,
dataDir: defaultDataDir,
privilegeFilePath: defaultPrivilegeFilePath,
mysqlDbFilePath: defaultMySQLDbFilePath,
}
}
go/cmd/dolt/commands/sqlserver/yaml_config.go
-8
View File
@@ -118,7 +118,6 @@ type YAMLConfig struct {
DataDirStr *string `yaml:"data_dir"`
MetricsConfig MetricsYAMLConfig `yaml:"metrics"`
PrivilegeFile *string `yaml:"privilege_file"`
MySQLDbFile *string `yaml:"mysql_db_file"`
}
var _ ServerConfig = YAMLConfig{}
@@ -325,13 +324,6 @@ func (cfg YAMLConfig) PrivilegeFilePath() string {
return ""
}
func (cfg YAMLConfig) MySQLDbFilePath() string {
if cfg.MySQLDbFile != nil {
return *cfg.MySQLDbFile
}
return ""
}
// QueryParallelism returns the parallelism that should be used by the go-mysql-server analyzer
func (cfg YAMLConfig) QueryParallelism() int {
if cfg.PerformanceConfig.QueryParallelism == nil {
go/go.mod
+1 -1
View File
@@ -68,7 +68,7 @@ require (
)
require (
github.com/dolthub/go-mysql-server v0.11.1-0.20220609220909-37da4c17aedd
github.com/dolthub/go-mysql-server v0.11.1-0.20220609230927-97a1fefc9b78
github.com/google/flatbuffers v2.0.6+incompatible
github.com/gosuri/uilive v0.0.4
github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6
go/go.sum
+2 -2
View File
@@ -178,8 +178,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
github.com/dolthub/go-mysql-server v0.11.1-0.20220609220909-37da4c17aedd h1:xGs/MdwUyrkWz0Btqy2tFcpU1RhRc3soo3xqu0F+mTs=
github.com/dolthub/go-mysql-server v0.11.1-0.20220609220909-37da4c17aedd/go.mod h1:gvDEMITJQDVYDLR4XtcqEZx6rawTvMh2veM1bPsJC3I=
github.com/dolthub/go-mysql-server v0.11.1-0.20220609230927-97a1fefc9b78 h1:VBgmYhLRY34r5Sfe/0X45yNU26rJ+62xJKkrfnbhPQU=
github.com/dolthub/go-mysql-server v0.11.1-0.20220609230927-97a1fefc9b78/go.mod h1:gvDEMITJQDVYDLR4XtcqEZx6rawTvMh2veM1bPsJC3I=
github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371 h1:oyPHJlzumKta1vnOQqUnfdz+pk3EmnHS3Nd0cCT0I2g=
github.com/dolthub/ishell v0.0.0-20220112232610-14e753f0f371/go.mod h1:dhGBqcCEfK5kuFmeO5+WOx3hqc1k3M29c1oS/R7N4ms=
github.com/dolthub/jsonpath v0.0.0-20210609232853-d49537a30474 h1:xTrR+l5l+1Lfq0NvhiEsctylXinUMFhhsqaEcl414p8=
go/libraries/doltcore/mvdata/engine_table_reader.go
+9 -7
View File
@@ -51,17 +51,19 @@ func NewSqlEngineReader(ctx context.Context, dEnv *env.DoltEnv, tableName string
return true, nil
})
config := &engine.SqlEngineConfig{
InitialDb: dbName,
IsReadOnly: false,
PrivFilePath: "",
ServerUser: "root",
ServerPass: "",
Autocommit: true,
}
se, err := engine.NewSqlEngine(
ctx,
mrEnv,
engine.FormatCsv,
dbName,
false,
"",
"",
"root",
"",
false,
config,
)
if err != nil {
return nil, err
go/libraries/doltcore/mvdata/engine_table_writer.go
+9 -7
View File
@@ -78,17 +78,19 @@ func NewSqlEngineTableWriter(ctx context.Context, dEnv *env.DoltEnv, createTable
})
// Simplest path would have our import path be a layer over load data
config := &engine.SqlEngineConfig{
InitialDb: dbName,
IsReadOnly: false,
PrivFilePath: "",
ServerUser: "root",
ServerPass: "",
Autocommit: true,
}
se, err := engine.NewSqlEngine(
ctx,
mrEnv,
engine.FormatCsv,
dbName,
false,
"",
"",
"root",
"",
false,
config,
)
if err != nil {
return nil, err
go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go
+3
View File
@@ -24,6 +24,7 @@ import (
"github.com/dolthub/go-mysql-server/enginetest/queries"
"github.com/dolthub/go-mysql-server/enginetest/scriptgen/setup"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/go-mysql-server/sql/mysql_db"
"github.com/dolthub/go-mysql-server/sql/plan"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -288,7 +289,9 @@ func TestDoltUserPrivileges(t *testing.T) {
User: "root",
Address: "localhost",
})
engine.Analyzer.Catalog.MySQLDb.AddRootAccount()
engine.Analyzer.Catalog.MySQLDb.SetPersister(&mysql_db.NoopPersister{})
for _, statement := range script.SetUpScript {
if sh, ok := interface{}(harness).(enginetest.SkippingHarness); ok {
go/libraries/doltcore/sqle/mysql_file_handler/file_handler.go
+55 -83
View File
@@ -15,7 +15,6 @@
package mysql_file_handler
import (
"encoding/json"
"errors"
"io/ioutil"
"os"
@@ -25,94 +24,83 @@ import (
"github.com/dolthub/go-mysql-server/sql/mysql_db"
)
const defaultMySQLFilePath = "mysql.db"
type Persister struct {
privsFilePath string
fileMutex *sync.Mutex
}
var fileMutex = &sync.Mutex{}
var mysqlDbFilePath string
var privsFilePath string
var _ mysql_db.MySQLDbPersistence = &Persister{}
// privDataJson is used to marshal/unmarshal the privilege data to/from JSON.
type privDataJson struct {
Users []*mysql_db.User
Roles []*mysql_db.RoleEdge
func NewPersister(fp string) *Persister {
// Create file if it does not exist, panic if something goes wrong
if len(fp) > 0 {
_, err := os.Stat(fp)
if err != nil && errors.Is(err, os.ErrNotExist) {
err = ioutil.WriteFile(fp, []byte{}, 0644)
}
if err != nil {
panic(err)
}
}
return &Persister{
privsFilePath: fp,
fileMutex: &sync.Mutex{},
}
}
func (p *Persister) ValidateCanPersist() error {
if len(p.privsFilePath) == 0 {
return errors.New("no privilege file specified, to persist users/grants run with --privilege-file=<file_path>")
}
return nil
}
func (p *Persister) Persist(ctx *sql.Context, data []byte) error {
p.fileMutex.Lock()
defer p.fileMutex.Unlock()
if err := p.ValidateCanPersist(); err != nil {
return err
}
return ioutil.WriteFile(p.privsFilePath, data, 0777)
}
// SetPrivilegeFilePath sets the file path that will be used for loading privileges.
func SetPrivilegeFilePath(fp string) {
// TODO: this is probably not needed
func (p Persister) SetPrivilegeFilePath(fp string) {
// do nothing for empty file path
if len(fp) == 0 {
return
}
fileMutex.Lock()
defer fileMutex.Unlock()
p.fileMutex.Lock()
defer p.fileMutex.Unlock()
// Panic if some strange unknown failure occurs (not just that it doesn't exist)
// Create file if it does not exist, panic if something goes wrong
_, err := os.Stat(fp)
if err != nil && !errors.Is(err, os.ErrNotExist) {
panic(err)
if err != nil && errors.Is(err, os.ErrNotExist) {
err = ioutil.WriteFile(fp, []byte{}, 0644)
}
privsFilePath = fp
}
// SetMySQLDbFilePath sets the file path that will be used for saving and loading MySQL Db tables.
func SetMySQLDbFilePath(fp string) {
// look for default mysql db file path if none specified
if len(fp) == 0 {
fp = defaultMySQLFilePath
}
fileMutex.Lock()
defer fileMutex.Unlock()
// Panic if some strange unknown failure occurs (not just that it doesn't exist)
_, err := os.Stat(fp)
if err != nil && !errors.Is(err, os.ErrNotExist) {
panic(err)
}
mysqlDbFilePath = fp
}
// LoadPrivileges reads the file previously set on the file path and returns the privileges and role connections. If the
// file path has not been set, returns an empty slice for both, but does not error. This is so that the logic path can
// retain the calls regardless of whether a user wants privileges to be loaded or persisted.
func LoadPrivileges() ([]*mysql_db.User, []*mysql_db.RoleEdge, error) {
// return nil for empty path
if len(privsFilePath) == 0 {
return nil, nil, nil
}
fileMutex.Lock()
defer fileMutex.Unlock()
// read from privsFilePath, error if something other than not-exists
fileContents, err := ioutil.ReadFile(privsFilePath)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, nil, err
}
if len(fileContents) == 0 {
return nil, nil, nil
}
data := &privDataJson{}
err = json.Unmarshal(fileContents, data)
if err != nil {
return nil, nil, err
panic(err)
}
return data.Users, data.Roles, nil
p.privsFilePath = fp
}
// LoadData reads the mysql.db file, returns nil if empty or not found
func LoadData() ([]byte, error) {
// use default mysql db file path if none specified
if len(mysqlDbFilePath) == 0 {
mysqlDbFilePath = defaultMySQLFilePath
func (p Persister) LoadData() ([]byte, error) {
// do nothing if no filepath specified
if len(p.privsFilePath) == 0 {
return nil, nil
}
fileMutex.Lock()
defer fileMutex.Unlock()
p.fileMutex.Lock()
defer p.fileMutex.Unlock()
// read from mysqldbFilePath, error if something other than not-exists
buf, err := ioutil.ReadFile(mysqlDbFilePath)
buf, err := ioutil.ReadFile(p.privsFilePath)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
@@ -122,19 +110,3 @@ func LoadData() ([]byte, error) {
return buf, nil
}
var _ mysql_db.PersistCallback = SaveData
// SaveData writes the provided []byte (in valid flatbuffer format) to the mysql db file
func SaveData(ctx *sql.Context, data []byte) error {
fileMutex.Lock()
defer fileMutex.Unlock()
// use default if empty filepath
if len(mysqlDbFilePath) == 0 {
mysqlDbFilePath = defaultMySQLFilePath
}
// should create file if it doesn't exist
return ioutil.WriteFile(mysqlDbFilePath, data, 0777)
}
integration-tests/bats/sql-privs.bats
+21 -192
View File
@@ -31,14 +31,10 @@ teardown() {
teardown_common
}
@test "sql-privs: no privs.json and no mysql.db, create mysql.db" {
@test "sql-privs: no mysql.db, throws error when attempting to persist" {
skiponwindows "redirecting SQL to sql-client returns nothing after welcome messages"
cd repo1
# remove/replace mysql.db and privs.json if they exist
rm -f mysql.db
rm -f privs.json
start_sql_server repo1
# expect only dolt user
@@ -53,137 +49,54 @@ teardown() {
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '+------+' ]
# create user
# create user, expect error
run create_user
[ "$status" -eq 0 ]
[[ "$output" =~ "no privilege file specified, to persist users/grants run with --privilege-file=<file_path>" ]] || false
# expect dolt and new_user
# expect dolt user and new_user
run show_users
[ "$status" -eq 0 ]
[ "${lines[0]}" = '# Welcome to the Dolt MySQL client.' ]
[ "${lines[1]}" = "# Statements must be terminated with ';'." ]
[ "${lines[2]}" = '# "exit" or "quit" (or Ctrl-D) to exit.' ]
[ "${lines[3]}" = '+----------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+----------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '| new_user |' ]
[ "${lines[8]}" = '+----------+' ]
# check that mysql.db file exists, and privs.json doesn't
run ls
[[ "$output" =~ "mysql.db" ]] || false
! [[ "$output" =~ "privs.json" ]] || false
[ "${lines[3]}" = '+------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '+------+' ]
# restart server
stop_sql_server
start_sql_server repo1
# check for new_user
run show_users
[ "${lines[0]}" = '# Welcome to the Dolt MySQL client.' ]
[ "${lines[1]}" = "# Statements must be terminated with ';'." ]
[ "${lines[2]}" = '# "exit" or "quit" (or Ctrl-D) to exit.' ]
[ "${lines[3]}" = '+----------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+----------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '| new_user |' ]
[ "${lines[8]}" = '+----------+' ]
# remove mysql.db and privs.json if they exist
rm -f mysql.db
rm -f privs.json
# leave the directory
cd ..
}
@test "sql-privs: has privs.json and no mysql.db, read from privs.json and create mysql.db" {
skiponwindows "redirecting SQL to sql-client returns nothing after welcome messages"
cd repo1
# remove/replace mysql.db and privs.json if they exist
rm -f mysql.db
rm -f privs.json
cp $BATS_TEST_DIRNAME/privs.json .
start_sql_server repo1
# expect dolt and privs_user
# expect only dolt user
run show_users
[ "$status" -eq 0 ]
[ "${lines[0]}" = '# Welcome to the Dolt MySQL client.' ]
[ "${lines[1]}" = "# Statements must be terminated with ';'." ]
[ "${lines[2]}" = '# "exit" or "quit" (or Ctrl-D) to exit.' ]
[ "${lines[3]}" = '+------------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+------------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '| privs_user |' ]
[ "${lines[8]}" = '+------------+' ]
# create user
run create_user
[ "$status" -eq 0 ]
# expect dolt, privs_user, and new_user
run show_users
[ "${lines[0]}" = '# Welcome to the Dolt MySQL client.' ]
[ "${lines[1]}" = "# Statements must be terminated with ';'." ]
[ "${lines[2]}" = '# "exit" or "quit" (or Ctrl-D) to exit.' ]
[ "${lines[3]}" = '+------------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+------------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '| new_user |' ]
[ "${lines[8]}" = '| privs_user |' ]
[ "${lines[9]}" = '+------------+' ]
# new user didn't persist to privs.json
run cat privs.json
! [[ "$output" =~ "new_user" ]] || false
# check that mysql.db and privs.json exist
run ls
[[ "$output" =~ "mysql.db" ]] || false
[[ "$output" =~ "privs.json" ]] || false
# restart server
stop_sql_server
start_sql_server repo1
# expect dolt, privs_user, and new_user
run show_users
[ "${lines[0]}" = '# Welcome to the Dolt MySQL client.' ]
[ "${lines[1]}" = "# Statements must be terminated with ';'." ]
[ "${lines[2]}" = '# "exit" or "quit" (or Ctrl-D) to exit.' ]
[ "${lines[3]}" = '+------------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+------------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '| new_user |' ]
[ "${lines[8]}" = '| privs_user |' ]
[ "${lines[9]}" = '+------------+' ]
[ "${lines[3]}" = '+------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '+------+' ]
# remove mysql.db and privs.json if they exist
rm -f mysql.db
rm -f privs.json
# leave the directory
cd ..
}
@test "sql-privs: no privs.json and has mysql.db, read from mysql.db" {
@test "sql-privs: has mysql.db, reads from mysql.db" {
skiponwindows "redirecting SQL to sql-client returns nothing after welcome messages"
cd repo1
# remove/replace mysql.db and privs.json if they exist
# remove/replace mysql.db if they exist
rm -f mysql.db
rm -f privs.json
cp $BATS_TEST_DIRNAME/mysql.db .
start_sql_server repo1
start_sql_server_with_args --privilege-file=mysql.db repo1
# expect dolt and mysql_user
run show_users
@@ -215,14 +128,8 @@ teardown() {
[ "${lines[8]}" = '| new_user |' ]
[ "${lines[9]}" = '+------------+' ]
# check that mysql.db exists, and privs.json doesn't
run ls
[[ "$output" =~ "mysql.db" ]] || false
! [[ "$output" =~ "privs.json" ]] || false
# restart server
stop_sql_server
start_sql_server repo1
start_sql_server_with_args --privilege-file=mysql.db repo1
# expect dolt, new_user, and mysql_user
run show_users
@@ -237,86 +144,8 @@ teardown() {
[ "${lines[8]}" = '| new_user |' ]
[ "${lines[9]}" = '+------------+' ]
# remove mysql.db and privs.json if they exist
# remove mysql.db if they exist
rm -f mysql.db
rm -f privs.json
# leave the directory
cd ..
}
@test "sql-privs: has privs.json and has mysql.db, only reads from mysql.db" {
skiponwindows "redirecting SQL to sql-client returns nothing after welcome messages"
cd repo1
# remove/replace mysql.db and privs.json if they exist
rm -f mysql.db
rm -f privs.json
cp $BATS_TEST_DIRNAME/privs.json .
cp $BATS_TEST_DIRNAME/mysql.db .
start_sql_server repo1
# expect dolt and mysql_user
run show_users
[ "$status" -eq 0 ]
[ "${lines[0]}" = '# Welcome to the Dolt MySQL client.' ]
[ "${lines[1]}" = "# Statements must be terminated with ';'." ]
[ "${lines[2]}" = '# "exit" or "quit" (or Ctrl-D) to exit.' ]
[ "${lines[3]}" = '+------------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+------------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '| mysql_user |' ]
[ "${lines[8]}" = '+------------+' ]
# create user
run create_user
[ "$status" -eq 0 ]
# expect dolt, new_user, and mysql_user
run show_users
[ "${lines[0]}" = '# Welcome to the Dolt MySQL client.' ]
[ "${lines[1]}" = "# Statements must be terminated with ';'." ]
[ "${lines[2]}" = '# "exit" or "quit" (or Ctrl-D) to exit.' ]
[ "${lines[3]}" = '+------------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+------------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '| mysql_user |' ]
[ "${lines[8]}" = '| new_user |' ]
[ "${lines[9]}" = '+------------+' ]
# new user didn't persist to privs.json
run cat privs.json
! [[ "$output" =~ "new_user" ]] || false
# check that mysql.db and privs.json exist
run ls
[[ "$output" =~ "mysql.db" ]] || false
[[ "$output" =~ "privs.json" ]] || false
# restart server
stop_sql_server
start_sql_server repo1
# expect dolt, new_user, and mysql_user
run show_users
[ "${lines[0]}" = '# Welcome to the Dolt MySQL client.' ]
[ "${lines[1]}" = "# Statements must be terminated with ';'." ]
[ "${lines[2]}" = '# "exit" or "quit" (or Ctrl-D) to exit.' ]
[ "${lines[3]}" = '+------------+' ]
[ "${lines[4]}" = '| User |' ]
[ "${lines[5]}" = '+------------+' ]
[ "${lines[6]}" = '| dolt |' ]
[ "${lines[7]}" = '| mysql_user |' ]
[ "${lines[8]}" = '| new_user |' ]
[ "${lines[9]}" = '+------------+' ]
# remove mysql.db and privs.json if they exist
rm -f mysql.db
rm -f privs.json
# leave the directory
cd ..
integration-tests/bats/sql-shell.bats
+37 -13
View File
@@ -42,14 +42,10 @@ teardown() {
[[ "$output" =~ "+---------------------" ]] || false
}
@test "sql-shell: dolt sql shell has mysql db and can create users" {
# there does not exist a mysql.db file
run ls
! [[ "$output" =~ "mysql.db" ]] || false
@test "sql-shell: dolt sql shell has mysql db" {
# mysql database exists and has privilege tables
run dolt sql <<< "show tables from mysql;"
[ "$status" -eq "0" ]
[ "$status" -eq 0 ]
[[ "$output" =~ "user" ]] || false
[[ "$output" =~ "role_edges" ]] || false
@@ -58,21 +54,49 @@ teardown() {
[[ "$output" =~ "root" ]] || false
! [[ "$output" =~ "new_user" ]] || false
# create a new user
# create a new user, fails when no privilege file is specified
run dolt sql <<< "create user new_user;"
[ "$status" -eq "0" ]
[ "$status" -eq 1 ]
[[ "$output" =~ "no privilege file specified, to persist users/grants run with --privilege-file=<file_path>" ]] || false
# there should now be a mysql.db file
run ls
[[ "$output" =~ "mysql.db" ]] || false
# there shouldn't be any mysql.db files
run ls .dolt
! [[ "$output" =~ "mysql.db" ]] || false
# show users, expect root and new_user
# remove mysql.db just in case
rm -f .dolt/mysql.db
}
@test "sql-shell: dolt sql shell can create users" {
# remove existing privs.db
rm -f privs.db
# mysql database exists and has privilege tables
run dolt sql <<< "show tables from mysql;"
[ "$status" -eq 0 ]
[[ "$output" =~ "user" ]] || false
[[ "$output" =~ "role_edges" ]] || false
# show users, expect root user
run dolt sql <<< "select user from mysql.user;"
[[ "$output" =~ "root" ]] || false
! [[ "$output" =~ "new_user" ]] || false
# create a new user
run dolt sql --privilege-file=privs.db <<< "create user new_user;"
[ "$status" -eq 0 ]
# there is now a privs.db file
run ls
[[ "$output" =~ "privs.db" ]] || false
# show users, expect root user and new_user
run dolt sql --privilege-file=privs.db <<< "select user from mysql.user;"
[[ "$output" =~ "root" ]] || false
[[ "$output" =~ "new_user" ]] || false
# remove mysql.db just in case
rm -f mysql.db
rm -f privs.db
}
@test "sql-shell: bad sql in sql shell should error" {
integration-tests/bats/sql.bats
+67 -13
View File
@@ -39,14 +39,10 @@ teardown() {
teardown_common
}
@test "sql: dolt sql -q has mysql db and can create users" {
# there does not exist a mysql.db file
run ls
! [[ "$output" =~ "mysql.db" ]] || false
@test "sql: dolt sql -q without privilege file doesn't persist" {
# mysql database exists and has privilege tables
run dolt sql -q "show tables from mysql;"
[ "$status" -eq "0" ]
[ "$status" -eq 0 ]
[[ "$output" =~ "user" ]] || false
[[ "$output" =~ "role_edges" ]] || false
@@ -55,21 +51,79 @@ teardown() {
[[ "$output" =~ "root" ]] || false
! [[ "$output" =~ "new_user" ]] || false
# create a new user
# create a new user, fails
run dolt sql -q "create user new_user;"
[ "$status" -eq "0" ]
[ "$status" -eq 1 ]
[[ "$output" =~ "no privilege file specified, to persist users/grants run with --privilege-file=<file_path>" ]] || false
# there should now be a mysql.db file
# there shouldn't be a mysql.db file
run ls
[[ "$output" =~ "mysql.db" ]] || false
! [[ "$output" =~ "privs.db" ]] || false
# show users, expect root and new_user
# show users, expect just root user
run dolt sql -q "select user from mysql.user;"
[[ "$output" =~ "root" ]] || false
! [[ "$output" =~ "new_user" ]] || false
# remove privs.db just in case
rm -f privs.db
}
@test "sql: dolt sql -q with privilege file persists" {
# mysql database exists and has privilege tables
run dolt sql --privilege-file=privs.db -q "show tables from mysql;"
[ "$status" -eq 0 ]
[[ "$output" =~ "user" ]] || false
[[ "$output" =~ "role_edges" ]] || false
# show users, expect just root user
run dolt sql --privilege-file=privs.db -q "select user from mysql.user;"
[[ "$output" =~ "root" ]] || false
! [[ "$output" =~ "new_user" ]] || false
# create a new user, fails
run dolt sql --privilege-file=privs.db -q "create user new_user;"
[ "$status" -eq 0 ]
# show users, expect just root user
run dolt sql --privilege-file=privs.db -q "select user from mysql.user;"
[[ "$output" =~ "root" ]] || false
[[ "$output" =~ "new_user" ]] || false
# remove mysql.db just in case
rm -f mysql.db
# there should now be a mysql.db file
run ls
[[ "$output" =~ "privs.db" ]] || false
# show users, expect root and new_user
run dolt sql --privilege-file=privs.db -q "select user from mysql.user;"
[[ "$output" =~ "root" ]] || false
[[ "$output" =~ "new_user" ]] || false
# remove mysql.db
rm -f privs.db
}
@test "sql: dolt sql -q create database and specify privilege file" {
run dolt sql --privilege-file=privs.db -q "create database inner_db;"
[ "$status" -eq 0 ]
run dolt sql --privilege-file=privs.db -q "create user new_user;"
[ "$status" -eq 0 ]
run dolt sql --privilege-file=privs.db -q "select user from mysql.user;"
[ "$status" -eq 0 ]
[[ "$output" =~ "root" ]] || false
[[ "$output" =~ "new_user" ]] || false
cd inner_db
run dolt sql --privilege-file=../privs.db -q "select user from mysql.user;"
[ "$status" -eq 0 ]
[[ "$output" =~ "root" ]] || false
[[ "$output" =~ "new_user" ]] || false
cd ..
rm -f privs.db
}
@test "sql: errors do not write incomplete rows" {