Added global caching for databases to avoid expensive lookups on names

go/libraries/doltcore/sqle/database_provider.go

@@ -1086,9 +1086,8 @@ func (p DoltDatabaseProvider) SessionDatabase(ctx *sql.Context, name string) (ds
 	revisionQualifiedName := name
 	usingDefaultBranch := false
 	head := ""
+	sess := dsess.DSessFromSess(ctx.Session)
 	if !isRevisionDbName {
-		sess := dsess.DSessFromSess(ctx.Session)
-
 		var err error
 		head, ok, err = sess.CurrentHead(ctx, baseName)
 		if err != nil {
@@ -1108,7 +1107,7 @@ func (p DoltDatabaseProvider) SessionDatabase(ctx *sql.Context, name string) (ds
 
 		revisionQualifiedName = baseName + dsess.DbRevisionDelimiter + head
 	}
-
+	
 	db, ok, err := p.databaseForRevision(ctx, revisionQualifiedName, name)
 	if err != nil {
 		if sql.ErrDatabaseNotFound.Is(err) && usingDefaultBranch {

go/libraries/doltcore/sqle/dsess/database_session_state.go

@@ -72,10 +72,13 @@ type DatabaseSessionState struct {
 	checkedOutRevSpec string
 	// heads records the in-memory DB state for every branch head accessed by the session
 	heads map[string]*branchState
-	// caches records the session-caches for every branch head accessed by the session
+	// globalCache records cache information for the entire session to speed up reads when nothing has changed since the
+	// last transaction
+	globalCache *SessionCache
+	// headCache records the session-caches for every branch head accessed by the session
 	// This is managed separately from the branch states themselves because it persists across transactions (which is
 	// safe because it's keyed by immutable hashes)
-	caches map[string]*SessionCache
+	headCache map[string]*SessionCache
 	// globalState is the global state of this session (shared by all sessions for a particular db)
 	globalState globalstate.GlobalState
 	// tmpFileDir is the directory to use for temporary files for this database
@@ -89,8 +92,9 @@ type DatabaseSessionState struct {
 
 func NewEmptyDatabaseSessionState() *DatabaseSessionState {
 	return &DatabaseSessionState{
-		heads: make(map[string]*branchState),
-		caches: make(map[string]*SessionCache),
+		heads:     make(map[string]*branchState),
+		headCache: make(map[string]*SessionCache),
+		globalCache: newSessionCache(),
 	}
 }
 
@@ -137,9 +141,9 @@ func (dbState *DatabaseSessionState) NewEmptyBranchState(head string) *branchSta
 	}
 
 	dbState.heads[head] = b
-	_, ok := dbState.caches[head]
+	_, ok := dbState.headCache[head]
 	if !ok {
-		dbState.caches[head] = newSessionCache()
+		dbState.headCache[head] = newSessionCache()
 	}
 
 	return b
@@ -160,7 +164,7 @@ func (bs *branchState) WriteSession() writer.WriteSession {
 }
 
 func (bs *branchState) SessionCache() *SessionCache {
-	return bs.dbState.caches[bs.head]
+	return bs.dbState.headCache[bs.head]
 }
 
 func (bs branchState) EditOpts() editor.Options {

go/libraries/doltcore/sqle/dsess/session.go

@@ -146,10 +146,10 @@ func (d *DoltSession) lookupDbState(ctx *sql.Context, dbName string) (*branchSta
 	baseName, rev = SplitRevisionDbName(dbName)
 
 	d.mu.Lock()
-	dbState, ok := d.dbStates[baseName]
+	dbState, dbStateFound := d.dbStates[baseName]
 	d.mu.Unlock()
 
-	if ok {
+	if dbStateFound {
 		// If we got an unqualified name, use the current working set head
 		if rev == "" {
 			rev = dbState.currRevSpec
@@ -174,24 +174,45 @@ func (d *DoltSession) lookupDbState(ctx *sql.Context, dbName string) (*branchSta
 		revisionQualifiedName = revisionDbName(baseName, rev)
 	}
 
-	database, ok, err := d.provider.SessionDatabase(ctx, revisionQualifiedName)
-	if err != nil {
-		return nil, false, err
+	// Try to get the session from the cache before going to the provider
+	tx, usingDoltTransaction := d.GetTransaction().(*DoltTransaction)
+	var database SqlDatabase
+	if usingDoltTransaction && dbStateFound {
+		nomsRoot, ok := tx.GetInitialRoot(baseName)
+		if ok {
+			database, ok = dbState.globalCache.GetCachedRevisionDb(doltdb.DataCacheKey{Hash: nomsRoot}, revisionQualifiedName)
+		}
 	}
 
-	if !ok {
-		return nil, false, nil
-	}
+	if database == nil {
+		var err error
+		var ok bool
+		database, ok, err = d.provider.SessionDatabase(ctx, revisionQualifiedName)
+		if err != nil {
+			return nil, false, err
+		}
+		if !ok {
+			return nil, false, nil
+		}
 
+		if usingDoltTransaction {
+			nomsRoot, ok := tx.GetInitialRoot(baseName)
+			if ok {
+				dbState.globalCache.CacheRevisionDb(doltdb.DataCacheKey{Hash: nomsRoot}, revisionQualifiedName, database)
+			}
+		}
+	}
+	
 	// Add the initial state to the session for future reuse
-	if err = d.addDB(ctx, database); err != nil {
+	if err := d.addDB(ctx, database); err != nil {
 		return nil, false, err
 	}
 
 	d.mu.Lock()
-	dbState, ok = d.dbStates[baseName]
+	dbState, dbStateFound = d.dbStates[baseName]
 	d.mu.Unlock()
-	if !ok {
+	if !dbStateFound {
+		// should be impossible
 		return nil, false, sql.ErrDatabaseNotFound.New(dbName)
 	}
 

go/libraries/doltcore/sqle/dsess/session_cache.go

@@ -24,12 +24,16 @@ import (
 )
 
 // SessionCache caches various pieces of expensive to compute information to speed up future lookups in the session.
-// No methods are thread safe.
 type SessionCache struct {
 	indexes map[doltdb.DataCacheKey]map[string][]sql.Index
 	tables  map[doltdb.DataCacheKey]map[string]sql.Table
 	views   map[doltdb.DataCacheKey]map[string]sql.ViewDefinition
-
+	
+	// unlike the other caches, the database cache is keyed by noms root hash, not a rootValue hash. Keys in the 
+	// secondary map are revision specifier strings
+	revisionDbs map[doltdb.DataCacheKey]map[string]SqlDatabase
+	initialDbStates map[doltdb.DataCacheKey]map[string]InitialDbState
+	
 	mu sync.RWMutex
 }
 
@@ -193,3 +197,88 @@ func (c *SessionCache) GetCachedViewDefinition(key doltdb.DataCacheKey, viewName
 	table, ok := viewsForKey[viewName]
 	return table, ok
 }
+
+// GetCachedRevisionDb returns the cached revision database named, and whether the cache was present
+func (c *SessionCache) GetCachedRevisionDb(key doltdb.DataCacheKey, revisionDbName string) (SqlDatabase, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	
+	if c.revisionDbs == nil {
+		return nil, false
+	}
+
+	dbsForKey, ok := c.revisionDbs[key]
+	if !ok {
+		return nil, false
+	}
+
+	db, ok := dbsForKey[revisionDbName]
+	return db, ok
+}
+
+// CacheRevisionDb caches the revision database named
+func (c *SessionCache) CacheRevisionDb(key doltdb.DataCacheKey, revisionDbName string, database SqlDatabase) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.revisionDbs == nil {
+		c.revisionDbs = make(map[doltdb.DataCacheKey]map[string]SqlDatabase)
+	}
+	
+	if len(c.revisionDbs) > maxCachedKeys {
+		for k := range c.revisionDbs {
+			delete(c.revisionDbs, k)
+		}
+	}
+
+	dbsForKey, ok := c.revisionDbs[key]
+	if !ok {
+		dbsForKey = make(map[string]SqlDatabase)
+		c.revisionDbs[key] = dbsForKey
+	}
+
+	dbsForKey[revisionDbName] = database
+}
+
+// GetCachedInitialDbState returns the cached initial state for the revision database named, and whether the cache 
+// was present
+func (c *SessionCache) GetCachedInitialDbState(key doltdb.DataCacheKey, revisionDbName string) (InitialDbState, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.initialDbStates == nil {
+		return InitialDbState{}, false
+	}
+
+	dbsForKey, ok := c.initialDbStates[key]
+	if !ok {
+		return InitialDbState{}, false
+	}
+
+	db, ok := dbsForKey[revisionDbName]
+	return db, ok
+}
+
+// CacheInitialDbState caches the initials state for the revision database named
+func (c *SessionCache) CacheInitialDbState(key doltdb.DataCacheKey, revisionDbName string, state InitialDbState) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.initialDbStates == nil {
+		c.initialDbStates = make(map[doltdb.DataCacheKey]map[string]InitialDbState)
+	}
+
+	if len(c.initialDbStates) > maxCachedKeys {
+		for k := range c.initialDbStates {
+			delete(c.initialDbStates, k)
+		}
+	}
+
+	dbsForKey, ok := c.initialDbStates[key]
+	if !ok {
+		dbsForKey = make(map[string]InitialDbState)
+		c.initialDbStates[key] = dbsForKey
+	}
+
+	dbsForKey[revisionDbName] = state
+}