Merge pull request #3827 from dolthub/zachmu/caching

Added table, index, and view caching for performance

go/cmd/dolt/commands/diff.go

@@ -452,6 +452,7 @@ func printShowCreateTableDiff(ctx context.Context, td diff.TableDelta) errhand.V
 
 	var fromCreateStmt = ""
 	if td.FromTable != nil {
+		// TODO: use UserSpaceDatabase for these, no reason for this separate database implementation
 		sqlDb := sqle.NewSingleTableDatabase(td.FromName, fromSch, td.FromFks, td.FromFksParentSch)
 		sqlCtx, engine, _ := sqle.PrepareCreateTableStmt(ctx, sqlDb)
 		fromCreateStmt, err = sqle.GetCreateTableStmt(sqlCtx, engine, td.FromName)

go/cmd/dolt/commands/filter-branch.go

@@ -39,7 +39,6 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
 	"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
-	"github.com/dolthub/dolt/go/libraries/utils/config"
 	"github.com/dolthub/dolt/go/store/hash"
 )
 
@@ -226,19 +225,6 @@ func processFilterQuery(ctx context.Context, dEnv *env.DoltEnv, cm *doltdb.Commi
 // we set manually with the one at the working set of the HEAD being rebased.
 // Some functionality will not work on this kind of engine, e.g. many DOLT_ functions.
 func rebaseSqlEngine(ctx context.Context, dEnv *env.DoltEnv, cm *doltdb.Commit) (*sql.Context, *engine.SqlEngine, error) {
-	sess := dsess.DefaultSession().NewDoltSession(config.NewMapConfig(make(map[string]string)))
-
-	sqlCtx := sql.NewContext(ctx, sql.WithSession(sess))
-	err := sqlCtx.SetSessionVariable(sqlCtx, sql.AutoCommitSessionVar, false)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	err = sqlCtx.SetSessionVariable(sqlCtx, dsess.TransactionsDisabledSysVar, true)
-	if err != nil {
-		return nil, nil, err
-	}
-
 	opts := editor.Options{Deaf: dEnv.DbEaFactory(), Tempdir: dEnv.TempTableFilesDir()}
 	db := dsqle.NewDatabase(dbName, dEnv.DbData(), opts)
 
@@ -250,6 +236,19 @@ func rebaseSqlEngine(ctx context.Context, dEnv *env.DoltEnv, cm *doltdb.Commit)
 	b := env.GetDefaultInitBranch(dEnv.Config)
 	pro := dsqle.NewDoltDatabaseProvider(b, mrEnv.FileSystem(), db)
 
+	sess := dsess.DefaultSession(pro)
+
+	sqlCtx := sql.NewContext(ctx, sql.WithSession(sess))
+	err = sqlCtx.SetSessionVariable(sqlCtx, sql.AutoCommitSessionVar, false)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	err = sqlCtx.SetSessionVariable(sqlCtx, dsess.TransactionsDisabledSysVar, true)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	parallelism := runtime.GOMAXPROCS(0)
 	azr := analyzer.NewBuilder(pro).WithParallelism(parallelism).Build()
 

go/cmd/dolt/commands/indexcmds/cat.go

@@ -35,7 +35,6 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/table/untyped/csv"
 	"github.com/dolthub/dolt/go/libraries/doltcore/table/untyped/tabular"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
-	"github.com/dolthub/dolt/go/libraries/utils/config"
 	"github.com/dolthub/dolt/go/libraries/utils/iohelp"
 	"github.com/dolthub/dolt/go/store/types"
 )
@@ -159,7 +158,7 @@ func (cmd CatCmd) prettyPrintResults(ctx context.Context, doltSch schema.Schema,
 	}
 	defer wr.Close(ctx)
 
-	sess := dsess.DefaultSession().NewDoltSession(config.NewMapConfig(make(map[string]string)))
+	sess := dsess.DefaultSession(dsess.EmptyDatabaseProvider())
 	sqlCtx := sql.NewContext(ctx, sql.WithSession(sess))
 
 	rowItr, err := getRowIter(ctx, doltSch, sqlSch, idx)

go/libraries/doltcore/doltdb/root_val.go

@@ -1564,9 +1564,14 @@ func (r fbRvStorage) nomsValue() types.Value {
 }
 
 type DataCacheKey struct {
-	rv *RootValue
+	hash.Hash
 }
 
-func NewDataCacheKey(rv *RootValue) DataCacheKey {
-	return DataCacheKey{rv}
+func NewDataCacheKey(rv *RootValue) (DataCacheKey, error) {
+	hash, err := rv.HashOf()
+	if err != nil {
+		return DataCacheKey{}, err
+	}
+
+	return DataCacheKey{hash}, nil
 }

go/libraries/doltcore/sqle/database.go

@@ -181,6 +181,11 @@ func NewDatabase(name string, dbData env.DbData, editOpts editor.Options) Databa
 
 // GetInitialDBState returns the InitialDbState for |db|.
 func GetInitialDBState(ctx context.Context, db SqlDatabase) (dsess.InitialDbState, error) {
+	switch db := db.(type) {
+	case *UserSpaceDatabase, *SingleTableInfoDatabase:
+		return getInitialDBStateForUserSpaceDb(ctx, db)
+	}
+
 	rsr := db.DbData().Rsr
 	ddb := db.DbData().Ddb
 
@@ -282,12 +287,16 @@ func (db Database) GetTableInsensitive(ctx *sql.Context, tblName string) (sql.Ta
 		return nil, false, err
 	}
 
-	head, err := ds.GetHeadCommit(ctx, db.Name())
+	tbl, ok, err := db.getTableInsensitive(ctx, nil, ds, root, tblName)
 	if err != nil {
 		return nil, false, err
 	}
 
-	return db.getTableInsensitive(ctx, head, root, tblName)
+	if !ok {
+		return nil, false, nil
+	}
+
+	return tbl, true, nil
 }
 
 // GetTableInsensitiveAsOf implements sql.VersionedDatabase
@@ -299,7 +308,9 @@ func (db Database) GetTableInsensitiveAsOf(ctx *sql.Context, tableName string, a
 		return nil, false, nil
 	}
 
-	table, ok, err := db.getTableInsensitive(ctx, head, root, tableName)
+	sess := dsess.DSessFromSess(ctx.Session)
+
+	table, ok, err := db.getTableInsensitive(ctx, head, sess, root, tableName)
 	if err != nil {
 		return nil, false, err
 	}
@@ -337,13 +348,21 @@ func (db Database) GetTableInsensitiveAsOf(ctx *sql.Context, tableName string, a
 	}
 }
 
-func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, root *doltdb.RootValue, tblName string) (sql.Table, bool, error) {
+func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ds *dsess.DoltSession, root *doltdb.RootValue, tblName string) (sql.Table, bool, error) {
 	lwrName := strings.ToLower(tblName)
 
 	// TODO: these tables that cache a root value at construction time should not, they need to get it from the session
 	//  at runtime
 	switch {
 	case strings.HasPrefix(lwrName, doltdb.DoltDiffTablePrefix):
+		if head == nil {
+			var err error
+			head, err = ds.GetHeadCommit(ctx, db.Name())
+			if err != nil {
+				return nil, false, err
+			}
+		}
+
 		suffix := tblName[len(doltdb.DoltDiffTablePrefix):]
 		dt, err := dtables.NewDiffTable(ctx, suffix, db.ddb, root, head)
 		if err != nil {
@@ -368,6 +387,15 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ro
 		if !ok {
 			return nil, false, nil
 		}
+
+		if head == nil {
+			var err error
+			head, err = ds.GetHeadCommit(ctx, db.Name())
+			if err != nil {
+				return nil, false, err
+			}
+		}
+
 		return NewHistoryTable(baseTable.(*AlterableDoltTable).DoltTable, db.ddb, head), true, nil
 
 	case strings.HasPrefix(lwrName, doltdb.DoltConfTablePrefix):
@@ -391,8 +419,24 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ro
 	found := false
 	switch lwrName {
 	case doltdb.LogTableName:
+		if head == nil {
+			var err error
+			head, err = ds.GetHeadCommit(ctx, db.Name())
+			if err != nil {
+				return nil, false, err
+			}
+		}
+
 		dt, found = dtables.NewLogTable(ctx, db.ddb, head), true
 	case doltdb.DiffTableName:
+		if head == nil {
+			var err error
+			head, err = ds.GetHeadCommit(ctx, db.Name())
+			if err != nil {
+				return nil, false, err
+			}
+		}
+
 		dt, found = dtables.NewUnscopedDiffTable(ctx, db.ddb, head), true
 	case doltdb.TableOfTablesInConflictName:
 		dt, found = dtables.NewTableOfTablesInConflict(ctx, db.name, db.ddb), true
@@ -409,7 +453,7 @@ func (db Database) getTableInsensitive(ctx *sql.Context, head *doltdb.Commit, ro
 	case doltdb.StatusTableName:
 		sess := dsess.DSessFromSess(ctx.Session)
 		adapter := dsess.NewSessionStateAdapter(
-			sess.Session, db.name,
+			sess, db.name,
 			map[string]env.Remote{},
 			map[string]env.BranchConfig{},
 			map[string]env.Remote{})
@@ -498,6 +542,7 @@ func resolveAsOfCommitRef(ctx *sql.Context, ddb *doltdb.DoltDB, head ref.DoltRef
 	}
 
 	cs, err := doltdb.NewCommitSpec(commitRef)
+
 	if err != nil {
 		return nil, nil, err
 	}
@@ -531,15 +576,33 @@ func (db Database) GetTableNamesAsOf(ctx *sql.Context, time interface{}) ([]stri
 	return filterDoltInternalTables(tblNames), nil
 }
 
-// getTable gets the table with the exact name given at the root value given. The database caches tables for all root
-// values to avoid doing schema lookups on every table lookup, which are expensive.
+// getTable returns the user table with the given name from the root given
 func (db Database) getTable(ctx *sql.Context, root *doltdb.RootValue, tableName string) (sql.Table, bool, error) {
+	sess := dsess.DSessFromSess(ctx.Session)
+	dbState, ok, err := sess.LookupDbState(ctx, db.name)
+	if err != nil {
+		return nil, false, err
+	}
+	if !ok {
+		return nil, false, fmt.Errorf("no state for database %s", db.name)
+	}
+
+	key, err := doltdb.NewDataCacheKey(root)
+	if err != nil {
+		return nil, false, err
+	}
+
+	cachedTable, ok := dbState.SessionCache().GetCachedTable(key, tableName)
+	if ok {
+		return cachedTable, true, nil
+	}
+
 	tableNames, err := getAllTableNames(ctx, root)
 	if err != nil {
 		return nil, true, err
 	}
 
-	tableName, ok := sql.GetTableNameInsensitive(tableName, tableNames)
+	tableName, ok = sql.GetTableNameInsensitive(tableName, tableNames)
 	if !ok {
 		return nil, false, nil
 	}
@@ -571,6 +634,8 @@ func (db Database) getTable(ctx *sql.Context, root *doltdb.RootValue, tableName
 		table = &AlterableDoltTable{WritableDoltTable{DoltTable: readonlyTable, db: db}}
 	}
 
+	dbState.SessionCache().CacheTable(key, tableName, table)
+
 	return table, true, nil
 }
 
@@ -875,11 +940,28 @@ func (db Database) GetView(ctx *sql.Context, viewName string) (string, bool, err
 		return view, true, nil
 	}
 
+	key, err := doltdb.NewDataCacheKey(root)
+	if err != nil {
+		return "", false, err
+	}
+
+	ds := dsess.DSessFromSess(ctx.Session)
+	dbState, _, err := ds.LookupDbState(ctx, db.name)
+	if err != nil {
+		return "", false, err
+	}
+
+	if dbState.SessionCache().ViewsCached(key) {
+		view, ok := dbState.SessionCache().GetCachedView(key, viewName)
+		return view, ok, nil
+	}
+
 	tbl, ok, err := db.GetTableInsensitive(ctx, doltdb.SchemasTableName)
 	if err != nil {
 		return "", false, err
 	}
 	if !ok {
+		dbState.SessionCache().CacheViews(key, nil, nil)
 		return "", false, nil
 	}
 
@@ -888,13 +970,23 @@ func (db Database) GetView(ctx *sql.Context, viewName string) (string, bool, err
 		return "", false, err
 	}
 
-	for _, fragment := range fragments {
+	found := false
+	viewDef := ""
+	viewNames := make([]string, len(fragments))
+	viewDefs := make([]string, len(fragments))
+	for i, fragment := range fragments {
 		if strings.ToLower(fragment.name) == strings.ToLower(viewName) {
-			return fragment.fragment, true, nil
+			found = true
+			viewDef = fragments[i].fragment
 		}
+
+		viewNames[i] = fragments[i].name
+		viewDefs[i] = fragments[i].fragment
 	}
 
-	return "", false, nil
+	dbState.SessionCache().CacheViews(key, viewNames, viewDefs)
+
+	return viewDef, found, nil
 }
 
 // AllViews implements sql.ViewDatabase
@@ -1091,3 +1183,54 @@ func (db Database) GetAllTemporaryTables(ctx *sql.Context) ([]sql.Table, error)
 	sess := dsess.DSessFromSess(ctx.Session)
 	return sess.GetAllTemporaryTables(ctx, db.Name())
 }
+
+// TODO: this is a hack to make user space DBs appear to the analyzer as full DBs with state etc., but the state is
+//  really skeletal. We need to reexamine the DB / session initialization to make this simpler -- most of these things
+//  aren't needed at initialization time and for most code paths.
+func getInitialDBStateForUserSpaceDb(ctx context.Context, db SqlDatabase) (dsess.InitialDbState, error) {
+	return dsess.InitialDbState{
+		Db: db,
+		DbData: env.DbData{
+			Rsw: noopRepoStateWriter{},
+		},
+	}, nil
+}
+
+// noopRepoStateWriter is a minimal implementation of RepoStateWriter that does nothing
+type noopRepoStateWriter struct{}
+
+func (n noopRepoStateWriter) UpdateStagedRoot(ctx context.Context, newRoot *doltdb.RootValue) error {
+	return nil
+}
+
+func (n noopRepoStateWriter) UpdateWorkingRoot(ctx context.Context, newRoot *doltdb.RootValue) error {
+	return nil
+}
+
+func (n noopRepoStateWriter) SetCWBHeadRef(ctx context.Context, marshalableRef ref.MarshalableRef) error {
+	return nil
+}
+
+func (n noopRepoStateWriter) AddRemote(name string, url string, fetchSpecs []string, params map[string]string) error {
+	return nil
+}
+
+func (n noopRepoStateWriter) AddBackup(name string, url string, fetchSpecs []string, params map[string]string) error {
+	return nil
+}
+
+func (n noopRepoStateWriter) RemoveRemote(ctx context.Context, name string) error {
+	return nil
+}
+
+func (n noopRepoStateWriter) RemoveBackup(ctx context.Context, name string) error {
+	return nil
+}
+
+func (n noopRepoStateWriter) TempTableFilesDir() string {
+	return ""
+}
+
+func (n noopRepoStateWriter) UpdateBranch(name string, new env.BranchConfig) error {
+	return nil
+}

go/libraries/doltcore/sqle/dfunctions/dolt_branch.go

@@ -237,7 +237,7 @@ func removeBranchRevisionDatabase(ctx *sql.Context, revisionDbName string) error
 		return fmt.Errorf("unexpected session type: %T", ctx.Session)
 	}
 
-	provider := doltsess.Session.Provider()
+	provider := doltsess.Provider()
 	if provider, ok := provider.(dsess.RevisionDatabaseProvider); ok {
 		err := provider.DropRevisionDb(ctx, revisionDbName)
 		// Try to remove any branch-qualified database, but don't error if it isn't found

go/libraries/doltcore/sqle/dsess/database_session_state.go

@@ -57,12 +57,20 @@ type DatabaseSessionState struct {
 
 	TblStats map[string]sql.TableStatistics
 
+	sessionCache *SessionCache
+
 	// Same as InitialDbState.Err, this signifies that this
 	// DatabaseSessionState is invalid. LookupDbState returning a
 	// DatabaseSessionState with Err != nil will return that err.
 	Err error
 }
 
+func NewEmptyDatabaseSessionState() *DatabaseSessionState {
+	return &DatabaseSessionState{
+		sessionCache: newSessionCache(),
+	}
+}
+
 func (d DatabaseSessionState) GetRoots() doltdb.Roots {
 	if d.WorkingSet == nil {
 		return doltdb.Roots{
@@ -78,6 +86,10 @@ func (d DatabaseSessionState) GetRoots() doltdb.Roots {
 	}
 }
 
+func (d *DatabaseSessionState) SessionCache() *SessionCache {
+	return d.sessionCache
+}
+
 func (d DatabaseSessionState) EditOpts() editor.Options {
 	return d.WriteSession.GetOptions()
 }

go/libraries/doltcore/sqle/dsess/dolt_session.go

@@ -1,256 +0,0 @@
-// Copyright 2021 Dolthub, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package dsess
-
-import (
-	"errors"
-	"fmt"
-	"strconv"
-	"sync"
-
-	"github.com/dolthub/go-mysql-server/sql"
-
-	"github.com/dolthub/dolt/go/cmd/dolt/cli"
-	"github.com/dolthub/dolt/go/libraries/doltcore/env"
-	"github.com/dolthub/dolt/go/libraries/utils/config"
-)
-
-var ErrSessionNotPeristable = errors.New("session is not persistable")
-
-type DoltSession struct {
-	*Session
-	globalsConf config.ReadWriteConfig
-	mu          *sync.Mutex
-}
-
-var _ sql.Session = (*DoltSession)(nil)
-var _ sql.PersistableSession = (*DoltSession)(nil)
-
-// NewDoltSession creates a DoltSession object from a standard sql.Session and 0 or more Database objects.
-func NewDoltSession(ctx *sql.Context, sqlSess *sql.BaseSession, pro RevisionDatabaseProvider, conf config.ReadWriteConfig, dbs ...InitialDbState) (*DoltSession, error) {
-	sess, err := NewSession(ctx, sqlSess, pro, conf, dbs...)
-	if err != nil {
-		return nil, err
-	}
-
-	globals := config.NewPrefixConfig(conf, env.SqlServerGlobalsPrefix)
-	return sess.NewDoltSession(globals), nil
-}
-
-// PersistGlobal implements sql.PersistableSession
-func (s *DoltSession) PersistGlobal(sysVarName string, value interface{}) error {
-	if s.globalsConf == nil {
-		return ErrSessionNotPeristable
-	}
-
-	sysVar, _, err := validatePersistableSysVar(sysVarName)
-	if err != nil {
-		return err
-	}
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return setPersistedValue(s.globalsConf, sysVar.Name, value)
-}
-
-// RemovePersistedGlobal implements sql.PersistableSession
-func (s *DoltSession) RemovePersistedGlobal(sysVarName string) error {
-	if s.globalsConf == nil {
-		return ErrSessionNotPeristable
-	}
-
-	sysVar, _, err := validatePersistableSysVar(sysVarName)
-	if err != nil {
-		return err
-	}
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.globalsConf.Unset([]string{sysVar.Name})
-}
-
-// RemoveAllPersistedGlobals implements sql.PersistableSession
-func (s *DoltSession) RemoveAllPersistedGlobals() error {
-	if s.globalsConf == nil {
-		return ErrSessionNotPeristable
-	}
-
-	allVars := make([]string, s.globalsConf.Size())
-	i := 0
-	s.globalsConf.Iter(func(k, v string) bool {
-		allVars[i] = k
-		i++
-		return false
-	})
-
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	return s.globalsConf.Unset(allVars)
-}
-
-// RemoveAllPersistedGlobals implements sql.PersistableSession
-func (s *DoltSession) GetPersistedValue(k string) (interface{}, error) {
-	if s.globalsConf == nil {
-		return nil, ErrSessionNotPeristable
-	}
-
-	return getPersistedValue(s.globalsConf, k)
-}
-
-// SystemVariablesInConfig returns a list of System Variables associated with the session
-func (s *DoltSession) SystemVariablesInConfig() ([]sql.SystemVariable, error) {
-	if s.globalsConf == nil {
-		return nil, ErrSessionNotPeristable
-	}
-
-	return SystemVariablesInConfig(s.globalsConf)
-}
-
-// validatePersistedSysVar checks whether a system variable exists and is dynamic
-func validatePersistableSysVar(name string) (sql.SystemVariable, interface{}, error) {
-	sysVar, val, ok := sql.SystemVariables.GetGlobal(name)
-	if !ok {
-		return sql.SystemVariable{}, nil, sql.ErrUnknownSystemVariable.New(name)
-	}
-	if !sysVar.Dynamic {
-		return sql.SystemVariable{}, nil, sql.ErrSystemVariableReadOnly.New(name)
-	}
-	return sysVar, val, nil
-}
-
-// getPersistedValue reads and converts a config value to the associated SystemVariable type
-func getPersistedValue(conf config.ReadableConfig, k string) (interface{}, error) {
-	v, err := conf.GetString(k)
-	if err != nil {
-		return nil, err
-	}
-
-	_, value, err := validatePersistableSysVar(k)
-	if err != nil {
-		return nil, err
-	}
-
-	var res interface{}
-	switch value.(type) {
-	case int8:
-		var tmp int64
-		tmp, err = strconv.ParseInt(v, 10, 8)
-		res = int8(tmp)
-	case int, int16, int32, int64:
-		res, err = strconv.ParseInt(v, 10, 64)
-	case uint, uint8, uint16, uint32, uint64:
-		res, err = strconv.ParseUint(v, 10, 64)
-	case float32, float64:
-		res, err = strconv.ParseFloat(v, 64)
-	case bool:
-		return nil, sql.ErrInvalidType.New(value)
-	case string:
-		return v, nil
-	default:
-		return nil, sql.ErrInvalidType.New(value)
-	}
-
-	if err != nil {
-		return nil, err
-	}
-
-	return res, nil
-}
-
-// setPersistedValue casts and persists a key value pair assuming thread safety
-func setPersistedValue(conf config.WritableConfig, key string, value interface{}) error {
-	switch v := value.(type) {
-	case int:
-		return config.SetInt(conf, key, int64(v))
-	case int8:
-		return config.SetInt(conf, key, int64(v))
-	case int16:
-		return config.SetInt(conf, key, int64(v))
-	case int32:
-		return config.SetInt(conf, key, int64(v))
-	case int64:
-		return config.SetInt(conf, key, v)
-	case uint:
-		return config.SetUint(conf, key, uint64(v))
-	case uint8:
-		return config.SetUint(conf, key, uint64(v))
-	case uint16:
-		return config.SetUint(conf, key, uint64(v))
-	case uint32:
-		return config.SetUint(conf, key, uint64(v))
-	case uint64:
-		return config.SetUint(conf, key, v)
-	case float32:
-		return config.SetFloat(conf, key, float64(v))
-	case float64:
-		return config.SetFloat(conf, key, v)
-	case string:
-		return config.SetString(conf, key, v)
-	case bool:
-		return sql.ErrInvalidType.New(v)
-	default:
-		return sql.ErrInvalidType.New(v)
-	}
-}
-
-// SystemVariablesInConfig returns system variables from the persisted config
-func SystemVariablesInConfig(conf config.ReadableConfig) ([]sql.SystemVariable, error) {
-	allVars := make([]sql.SystemVariable, conf.Size())
-	i := 0
-	var err error
-	var sysVar sql.SystemVariable
-	var def interface{}
-	conf.Iter(func(k, v string) bool {
-		def, err = getPersistedValue(conf, k)
-		if err != nil {
-			err = fmt.Errorf("key: '%s'; %w", k, err)
-			return true
-		}
-		// getPeristedVal already checked for errors
-		sysVar, _, _ = sql.SystemVariables.GetGlobal(k)
-		sysVar.Default = def
-		allVars[i] = sysVar
-		i++
-		return false
-	})
-	if err != nil {
-		return nil, err
-	}
-	return allVars, nil
-}
-
-var initMu = sync.Mutex{}
-
-func InitPersistedSystemVars(dEnv *env.DoltEnv) error {
-	initMu.Lock()
-	defer initMu.Unlock()
-
-	var globals config.ReadWriteConfig
-	if localConf, ok := dEnv.Config.GetConfig(env.LocalConfig); ok {
-		globals = config.NewPrefixConfig(localConf, env.SqlServerGlobalsPrefix)
-	} else if globalConf, ok := dEnv.Config.GetConfig(env.GlobalConfig); ok {
-		globals = config.NewPrefixConfig(globalConf, env.SqlServerGlobalsPrefix)
-	} else {
-		cli.Println("warning: no local or global Dolt configuration found; session is not persistable")
-		globals = config.NewMapConfig(make(map[string]string))
-	}
-
-	persistedGlobalVars, err := SystemVariablesInConfig(globals)
-	if err != nil {
-		return err
-	}
-	sql.SystemVariables.AddSystemVariables(persistedGlobalVars)
-	return nil
-}

go/libraries/doltcore/sqle/dsess/dolt_session_test.go

@@ -25,23 +25,22 @@ import (
 )
 
 func TestDoltSessionInit(t *testing.T) {
-	sess := DefaultSession()
+	dsess := DefaultSession(EmptyDatabaseProvider())
 	conf := config.NewMapConfig(make(map[string]string))
-	dsess := sess.NewDoltSession(conf)
 	assert.Equal(t, conf, dsess.globalsConf)
 }
 
 func TestNewPersistedSystemVariables(t *testing.T) {
-	sess := DefaultSession()
+	dsess := DefaultSession(EmptyDatabaseProvider())
 	conf := config.NewMapConfig(map[string]string{"max_connections": "1000"})
-	dsess := sess.NewDoltSession(conf)
+	dsess = dsess.WithGlobals(conf)
+
 	sysVars, err := dsess.SystemVariablesInConfig()
 	assert.NoError(t, err)
 
 	maxConRes := sysVars[0]
 	assert.Equal(t, "max_connections", maxConRes.Name)
 	assert.Equal(t, int64(1000), maxConRes.Default)
-
 }
 
 func TestValidatePeristableSystemVar(t *testing.T) {

go/libraries/doltcore/sqle/dsess/session_cache.go

@@ -0,0 +1,153 @@
+// Copyright 2022 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dsess
+
+import (
+	"strings"
+
+	"github.com/dolthub/go-mysql-server/sql"
+
+	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
+)
+
+// SessionCache caches various pieces of expensive to compute information to speed up future lookups in the session.
+// No methods are thread safe.
+type SessionCache struct {
+	indexes map[doltdb.DataCacheKey]map[string][]sql.Index
+	tables  map[doltdb.DataCacheKey]map[string]sql.Table
+	views   map[doltdb.DataCacheKey]map[string]string
+}
+
+func newSessionCache() *SessionCache {
+	return &SessionCache{}
+}
+
+// CacheTableIndexes caches all indexes for the table with the name given
+func (c *SessionCache) CacheTableIndexes(key doltdb.DataCacheKey, table string, indexes []sql.Index) {
+	table = strings.ToLower(table)
+
+	if c.indexes == nil {
+		c.indexes = make(map[doltdb.DataCacheKey]map[string][]sql.Index)
+	}
+
+	tableIndexes, ok := c.indexes[key]
+	if !ok {
+		tableIndexes = make(map[string][]sql.Index)
+		c.indexes[key] = tableIndexes
+	}
+
+	tableIndexes[table] = indexes
+}
+
+// GetTableIndexesCache returns the cached index information for the table named, and whether the cache was present
+func (c *SessionCache) GetTableIndexesCache(key doltdb.DataCacheKey, table string) ([]sql.Index, bool) {
+	table = strings.ToLower(table)
+
+	if c.indexes == nil {
+		return nil, false
+	}
+
+	tableIndexes, ok := c.indexes[key]
+	if !ok {
+		return nil, false
+	}
+
+	indexes, ok := tableIndexes[table]
+	return indexes, ok
+}
+
+// CacheTable caches a sql.Table implementation for the table named
+func (c *SessionCache) CacheTable(key doltdb.DataCacheKey, tableName string, table sql.Table) {
+	tableName = strings.ToLower(tableName)
+
+	if c.tables == nil {
+		c.tables = make(map[doltdb.DataCacheKey]map[string]sql.Table)
+	}
+
+	tablesForKey, ok := c.tables[key]
+	if !ok {
+		tablesForKey = make(map[string]sql.Table)
+		c.tables[key] = tablesForKey
+	}
+
+	tablesForKey[tableName] = table
+}
+
+// ClearTableCache removes all cache info for all tables at all cache keys
+func (c *SessionCache) ClearTableCache() {
+	c.tables = make(map[doltdb.DataCacheKey]map[string]sql.Table)
+}
+
+// GetCachedTable returns the cached sql.Table for the table named, and whether the cache was present
+func (c *SessionCache) GetCachedTable(key doltdb.DataCacheKey, tableName string) (sql.Table, bool) {
+	tableName = strings.ToLower(tableName)
+
+	if c.tables == nil {
+		return nil, false
+	}
+
+	tablesForKey, ok := c.tables[key]
+	if !ok {
+		return nil, false
+	}
+
+	table, ok := tablesForKey[tableName]
+	return table, ok
+}
+
+// CacheViews caches all views in a database for the cache key given
+func (c *SessionCache) CacheViews(key doltdb.DataCacheKey, viewNames []string, viewDefs []string) {
+	if c.views == nil {
+		c.views = make(map[doltdb.DataCacheKey]map[string]string)
+	}
+
+	viewsForKey, ok := c.views[key]
+	if !ok {
+		viewsForKey = make(map[string]string)
+		c.views[key] = viewsForKey
+	}
+
+	for i := range viewNames {
+		viewName := strings.ToLower(viewNames[i])
+		viewsForKey[viewName] = viewDefs[i]
+	}
+}
+
+// ViewsCached returns whether this cache has been initialized with the set of views yet
+func (c *SessionCache) ViewsCached(key doltdb.DataCacheKey) bool {
+	if c.views == nil {
+		return false
+	}
+
+	_, ok := c.views[key]
+	return ok
+}
+
+// GetCachedView returns the cached view named, and whether the cache was present
+func (c *SessionCache) GetCachedView(key doltdb.DataCacheKey, viewName string) (string, bool) {
+	viewName = strings.ToLower(viewName)
+
+	if c.views == nil {
+		return "", false
+	}
+
+	viewsForKey, ok := c.views[key]
+	if !ok {
+		return "", false
+	}
+
+	table, ok := viewsForKey[viewName]
+	return table, ok
+}

go/libraries/doltcore/sqle/dsess/session_state_adapter.go

@@ -28,7 +28,7 @@ import (
 // SessionStateAdapter is an adapter for env.RepoStateReader in SQL contexts, getting information about the repo state
 // from the session.
 type SessionStateAdapter struct {
-	session  *Session
+	session  *DoltSession
 	dbName   string
 	remotes  map[string]env.Remote
 	backups  map[string]env.Remote
@@ -75,7 +75,7 @@ var _ env.RepoStateReader = SessionStateAdapter{}
 var _ env.RepoStateWriter = SessionStateAdapter{}
 var _ env.RootsProvider = SessionStateAdapter{}
 
-func NewSessionStateAdapter(session *Session, dbName string, remotes map[string]env.Remote, branches map[string]env.BranchConfig, backups map[string]env.Remote) SessionStateAdapter {
+func NewSessionStateAdapter(session *DoltSession, dbName string, remotes map[string]env.Remote, branches map[string]env.BranchConfig, backups map[string]env.Remote) SessionStateAdapter {
 	if branches == nil {
 		branches = make(map[string]env.BranchConfig)
 	}

go/libraries/doltcore/sqle/enginetest/dolt_engine_test.go

@@ -978,7 +978,7 @@ func TestPersist(t *testing.T) {
 	require.True(t, ok)
 	globals := config.NewPrefixConfig(localConf, env.SqlServerGlobalsPrefix)
 	newPersistableSession := func(ctx *sql.Context) sql.PersistableSession {
-		session := ctx.Session.(*dsess.DoltSession).Session.NewDoltSession(globals)
+		session := ctx.Session.(*dsess.DoltSession).WithGlobals(globals)
 		err := session.RemoveAllPersistedGlobals()
 		require.NoError(t, err)
 		return session

go/libraries/doltcore/sqle/index/dolt_index_test.go

@@ -34,7 +34,6 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/index"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
-	"github.com/dolthub/dolt/go/libraries/utils/config"
 )
 
 type indexComp int
@@ -1398,8 +1397,7 @@ INSERT INTO types VALUES (1, 4, '2020-05-14 12:00:03', 1.1, 'd', 1.1, 'a,c', '00
 }
 
 func NewTestSQLCtx(ctx context.Context) *sql.Context {
-	session := dsess.DefaultSession()
-	s := session.NewDoltSession(config.NewMapConfig(make(map[string]string)))
+	s := dsess.DefaultSession(dsess.EmptyDatabaseProvider())
 	sqlCtx := sql.NewContext(
 		ctx,
 		sql.WithSession(s),

go/libraries/doltcore/sqle/show_create_table.go

@@ -27,12 +27,20 @@ import (
 
 // These functions cannot be in the sqlfmt package as the reliance on the sqle package creates a circular reference.
 
-func PrepareCreateTableStmt(ctx context.Context, sqlDb sql.Database) (*sql.Context, *sqle.Engine, *dsess.Session) {
-	sess := dsess.DefaultSession()
-	sqlCtx := sql.NewContext(ctx, sql.WithSession(sess))
-
+func PrepareCreateTableStmt(ctx context.Context, sqlDb SqlDatabase) (*sql.Context, *sqle.Engine, *dsess.DoltSession) {
 	pro := NewDoltDatabaseProvider(env.DefaultInitBranch, nil, sqlDb)
 	engine := sqle.NewDefault(pro)
+
+	sess := dsess.DefaultSession(pro)
+	sqlCtx := sql.NewContext(ctx, sql.WithSession(sess))
+	dbState, err := GetInitialDBState(ctx, sqlDb)
+	if err != nil {
+		// TODO
+		return nil, nil, nil
+	}
+
+	sess.AddDB(sqlCtx, dbState)
+
 	sqlCtx.SetCurrentDatabase(sqlDb.Name())
 	return sqlCtx, engine, sess
 }

go/libraries/doltcore/sqle/single_table_info_db.go

@@ -21,8 +21,10 @@ import (
 	"github.com/dolthub/go-mysql-server/sql"
 
 	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
+	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
+	"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
 )
 
 // SingleTableInfoDatabase is intended to allow a sole schema to make use of any display functionality in `go-mysql-server`.
@@ -37,6 +39,7 @@ type SingleTableInfoDatabase struct {
 }
 
 var _ doltReadOnlyTableInterface = (*SingleTableInfoDatabase)(nil)
+var _ SqlDatabase = (*SingleTableInfoDatabase)(nil)
 
 func NewSingleTableDatabase(tableName string, sch schema.Schema, foreignKeys []doltdb.ForeignKey, parentSchs map[string]schema.Schema) *SingleTableInfoDatabase {
 	return &SingleTableInfoDatabase{
@@ -210,6 +213,26 @@ func (db *SingleTableInfoDatabase) PrimaryKeySchema() sql.PrimaryKeySchema {
 	return sqlSch
 }
 
+func (db *SingleTableInfoDatabase) GetRoot(context *sql.Context) (*doltdb.RootValue, error) {
+	return nil, nil
+}
+
+func (db *SingleTableInfoDatabase) DbData() env.DbData {
+	panic("SingleTableInfoDatabase doesn't have DbData")
+}
+
+func (db *SingleTableInfoDatabase) StartTransaction(ctx *sql.Context, tCharacteristic sql.TransactionCharacteristic) (sql.Transaction, error) {
+	panic("SingleTableInfoDatabase cannot start transaction")
+}
+
+func (db *SingleTableInfoDatabase) Flush(context *sql.Context) error {
+	panic("SingleTableInfoDatabase cannot Flush")
+}
+
+func (db *SingleTableInfoDatabase) EditOptions() editor.Options {
+	return editor.Options{}
+}
+
 // fmtIndex is used for CREATE TABLE statements only.
 type fmtIndex struct {
 	id  string

go/libraries/doltcore/sqle/tables.go

@@ -248,7 +248,12 @@ func (t *DoltTable) DataCacheKey(ctx *sql.Context) (doltdb.DataCacheKey, bool, e
 	if err != nil {
 		return doltdb.DataCacheKey{}, false, err
 	}
-	return doltdb.NewDataCacheKey(r), true, nil
+	key, err := doltdb.NewDataCacheKey(r)
+	if err != nil {
+		return doltdb.DataCacheKey{}, false, err
+	}
+
+	return key, true, nil
 }
 
 func (t *DoltTable) workingRoot(ctx *sql.Context) (*doltdb.RootValue, error) {
@@ -267,12 +272,46 @@ func (t *DoltTable) getRoot(ctx *sql.Context) (*doltdb.RootValue, error) {
 
 // GetIndexes implements sql.IndexedTable
 func (t *DoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) {
+	key, tableIsCacheable, err := t.DataCacheKey(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if !tableIsCacheable {
+		tbl, err := t.DoltTable(ctx)
+		if err != nil {
+			return nil, err
+		}
+		return index.DoltIndexesFromTable(ctx, t.db.Name(), t.tableName, tbl)
+	}
+
+	sess := dsess.DSessFromSess(ctx.Session)
+	dbState, ok, err := sess.LookupDbState(ctx, t.db.Name())
+	if err != nil {
+		return nil, err
+	}
+
+	if !ok {
+		return nil, fmt.Errorf("couldn't find db state for database %s", t.db.Name())
+	}
+
+	indexes, ok := dbState.SessionCache().GetTableIndexesCache(key, t.Name())
+	if ok {
+		return indexes, nil
+	}
+
 	tbl, err := t.DoltTable(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	return index.DoltIndexesFromTable(ctx, t.db.Name(), t.tableName, tbl)
+	indexes, err = index.DoltIndexesFromTable(ctx, t.db.Name(), t.tableName, tbl)
+	if err != nil {
+		return nil, err
+	}
+
+	dbState.SessionCache().CacheTableIndexes(key, t.Name(), indexes)
+	return indexes, nil
 }
 
 // HasIndex returns whether the given index is present in the table
@@ -1577,11 +1616,7 @@ func (t *AlterableDoltTable) ModifyColumn(ctx *sql.Context, columnName string, c
 		return err
 	}
 
-	return nil
-	// TODO: we can't make this update right now because renames happen in two passes if you rename a column mentioned in
-	//  a default value, and one of those two passes will have the old name for the column. Fix this by not analyzing
-	//  column defaults in NewDoltTable.
-	// return t.updateFromRoot(ctx, newRoot)
+	return t.updateFromRoot(ctx, newRoot)
 }
 
 // getFirstAutoIncrementValue returns the next auto increment value for a table that just acquired one through an
@@ -2315,6 +2350,10 @@ func (t *AlterableDoltTable) dropIndex(ctx *sql.Context, indexName string) (*dol
 	return newTable, tblSch, nil
 }
 
+// updateFromRoot updates the table using data and schema in the root given. This is necessary for some schema change
+// statements that take place in multiple steps (e.g. adding a foreign key may create an index, then add a constraint).
+// We can't update the session's working set until the statement boundary, so we have to do it here.
+// TODO: eliminate this pattern, store all table data and schema in the session rather than in these objects.
 func (t *AlterableDoltTable) updateFromRoot(ctx *sql.Context, root *doltdb.RootValue) error {
 	updatedTableSql, ok, err := t.db.getTable(ctx, root, t.tableName)
 	if err != nil {
@@ -2330,6 +2369,17 @@ func (t *AlterableDoltTable) updateFromRoot(ctx *sql.Context, root *doltdb.RootV
 		updatedTable = updatedTableSql.(*AlterableDoltTable)
 	}
 	t.WritableDoltTable.DoltTable = updatedTable.WritableDoltTable.DoltTable
+
+	// When we update this table we need to also clear any cached versions of the object, since they may now have
+	// incorrect schema information
+	sess := dsess.DSessFromSess(ctx.Session)
+	dbState, ok, err := sess.LookupDbState(ctx, t.db.name)
+	if !ok {
+		return fmt.Errorf("no db state found for %s", t.db.name)
+	}
+
+	dbState.SessionCache().ClearTableCache()
+
 	return nil
 }
 

go/libraries/doltcore/sqle/user_space_database.go

@@ -97,5 +97,5 @@ func (db *UserSpaceDatabase) Flush(ctx *sql.Context) error {
 }
 
 func (db *UserSpaceDatabase) EditOptions() editor.Options {
-	panic("UserSpaceDatabase does not have edit options")
+	return editor.Options{}
 }

go/libraries/doltcore/table/untyped/sqlexport/sqlwriter.go

@@ -48,7 +48,6 @@ type SqlExportWriter struct {
 
 // OpenSQLExportWriter returns a new SqlWriter for the table with the writer given.
 func OpenSQLExportWriter(ctx context.Context, wr io.WriteCloser, root *doltdb.RootValue, tableName string, autocommitOff bool, sch schema.Schema, editOpts editor.Options) (*SqlExportWriter, error) {
-
 	allSchemas, err := root.GetAllSchemas(ctx)
 	if err != nil {
 		return nil, err