More db caching by name. Much faster, some bugs still

go/libraries/doltcore/sqle/database_provider.go

@@ -714,7 +714,6 @@ func (p DoltDatabaseProvider) invalidateDbStateInAllSessions(ctx *sql.Context, n
 }
 
 func (p DoltDatabaseProvider) databaseForRevision(ctx *sql.Context, revisionQualifiedName string, requestedName string) (dsess.SqlDatabase, bool, error) {
-	// TODO: use session cache to get database
 	if !strings.Contains(revisionQualifiedName, dsess.DbRevisionDelimiter) {
 		return nil, false, nil
 	}
@@ -722,6 +721,16 @@ func (p DoltDatabaseProvider) databaseForRevision(ctx *sql.Context, revisionQual
 	parts := strings.SplitN(revisionQualifiedName, dsess.DbRevisionDelimiter, 2)
 	baseName, rev := parts[0], parts[1]
 
+	// Look in the session cache for this DB before doing any IO to figure out what's being asked for
+	sess := dsess.DSessFromSess(ctx.Session)
+	dbCache, ok := sess.DatabaseCache(ctx, baseName)
+	if ok {
+		db, ok := dbCache.GetCachedRevisionDb(revisionQualifiedName)
+		if ok {
+			return db, true, nil
+		}
+	}
+
 	p.mu.RLock()
 	srcDb, ok := p.databases[formatDbMapKeyName(baseName)]
 	p.mu.RUnlock()

go/libraries/doltcore/sqle/dsess/database_session_state.go

@@ -72,9 +72,9 @@ type DatabaseSessionState struct {
 	checkedOutRevSpec string
 	// heads records the in-memory DB state for every branch head accessed by the session
 	heads map[string]*branchState
-	// globalCache records cache information for the entire session to speed up reads when nothing has changed since the
-	// last transaction
-	globalCache *SessionCache
+	// databaseCache records database name resolution for the session to speed up database resolution when nothing has
+	// changed since the last transaction
+	databaseCache *DatabaseCache
 	// headCache records the session-caches for every branch head accessed by the session
 	// This is managed separately from the branch states themselves because it persists across transactions (which is
 	// safe because it's keyed by immutable hashes)
@@ -92,9 +92,9 @@ type DatabaseSessionState struct {
 
 func newEmptyDatabaseSessionState() *DatabaseSessionState {
 	return &DatabaseSessionState{
-		heads:     make(map[string]*branchState),
-		headCache: make(map[string]*SessionCache),
-		globalCache: newSessionCache(),
+		heads:         make(map[string]*branchState),
+		headCache:     make(map[string]*SessionCache),
+		databaseCache: newDatabaseCache(),
 	}
 }
 

go/libraries/doltcore/sqle/dsess/session.go

@@ -175,13 +175,9 @@ func (d *DoltSession) lookupDbState(ctx *sql.Context, dbName string) (*branchSta
 	}
 
 	// Try to get the session from the cache before going to the provider
-	tx, usingDoltTransaction := d.GetTransaction().(*DoltTransaction)
 	var database SqlDatabase
-	if usingDoltTransaction && dbStateFound {
-		nomsRoot, ok := tx.GetInitialRoot(baseName)
-		if ok {
-			database, _ = dbState.globalCache.GetCachedRevisionDb(doltdb.DataCacheKey{Hash: nomsRoot}, revisionQualifiedName)
-		}
+	if dbStateFound {
+		database, _ = dbState.databaseCache.GetCachedRevisionDb(revisionQualifiedName)
 	}
 
 	if database == nil {
@@ -1141,7 +1137,7 @@ func (d *DoltSession) addDB(ctx *sql.Context, db SqlDatabase) error {
 	if usingDoltTransaction {
 		nomsRoot, ok := tx.GetInitialRoot(baseName)
 		if ok && sessionStateExists {
-			dbState, dbStateCached = sessionState.globalCache.GetCachedInitialDbState(doltdb.DataCacheKey{Hash: nomsRoot}, revisionQualifiedName)
+			dbState, dbStateCached = sessionState.databaseCache.GetCachedInitialDbState(doltdb.DataCacheKey{Hash: nomsRoot}, revisionQualifiedName)
 		}
 	}
 	
@@ -1187,8 +1183,8 @@ func (d *DoltSession) addDB(ctx *sql.Context, db SqlDatabase) error {
 	if !dbStateCached && usingDoltTransaction {
 		nomsRoot, ok := tx.GetInitialRoot(baseName)
 		if ok {
-			sessionState.globalCache.CacheInitialDbState(doltdb.DataCacheKey{Hash: nomsRoot}, revisionQualifiedName, dbState)
-			sessionState.globalCache.CacheRevisionDb(doltdb.DataCacheKey{Hash: nomsRoot}, revisionQualifiedName, db)
+			sessionState.databaseCache.CacheInitialDbState(doltdb.DataCacheKey{Hash: nomsRoot}, revisionQualifiedName, dbState)
+			sessionState.databaseCache.CacheRevisionDb(revisionQualifiedName, db)
 		}
 	}
 
@@ -1247,6 +1243,16 @@ func (d *DoltSession) addDB(ctx *sql.Context, db SqlDatabase) error {
 	return nil
 }
 
+func (d *DoltSession) DatabaseCache(ctx *sql.Context, dbName string) (*DatabaseCache, bool) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	dbState, ok := d.dbStates[dbName]
+	if !ok {
+		return nil, false
+	}
+	return dbState.databaseCache, true
+}
+
 func (d *DoltSession) AddTemporaryTable(ctx *sql.Context, db string, tbl sql.Table) {
 	d.tempTables[strings.ToLower(db)] = append(d.tempTables[strings.ToLower(db)], tbl)
 }

go/libraries/doltcore/sqle/dsess/session_cache.go

@@ -29,11 +29,19 @@ type SessionCache struct {
 	tables  map[doltdb.DataCacheKey]map[string]sql.Table
 	views   map[doltdb.DataCacheKey]map[string]sql.ViewDefinition
 	
-	// unlike the other caches, the database cache is keyed by noms root hash, not a rootValue hash. Keys in the 
-	// secondary map are revision specifier strings
-	revisionDbs map[doltdb.DataCacheKey]map[string]SqlDatabase
+	mu sync.RWMutex
+}
+
+// DatabaseCache stores databases and their initial states, offloading the compute / IO involved in resolving a 
+// database name to a particular database. This is safe only because the database objects themselves don't have any 
+// handles to data or state, but always defer to the session. Keys in the secondary map are revision specifier strings
+type DatabaseCache struct {
+	// revisionDbs caches databases by name. The name is always lower case and revision qualified
+	revisionDbs map[string]SqlDatabase
+	// initialDbStates caches the initial state of databases by name for a given noms root, which is the primary key.
+	// The secondary key is the lower-case revision-qualified database name.
 	initialDbStates map[doltdb.DataCacheKey]map[string]InitialDbState
-	
+
 	mu sync.RWMutex
 }
 
@@ -43,6 +51,10 @@ func newSessionCache() *SessionCache {
 	return &SessionCache{}
 }
 
+func newDatabaseCache() *DatabaseCache {
+	return &DatabaseCache{}
+}
+
 // CacheTableIndexes caches all indexes for the table with the name given
 func (c *SessionCache) CacheTableIndexes(key doltdb.DataCacheKey, table string, indexes []sql.Index) {
 	c.mu.Lock()
@@ -199,30 +211,25 @@ func (c *SessionCache) GetCachedViewDefinition(key doltdb.DataCacheKey, viewName
 }
 
 // GetCachedRevisionDb returns the cached revision database named, and whether the cache was present
-func (c *SessionCache) GetCachedRevisionDb(key doltdb.DataCacheKey, revisionDbName string) (SqlDatabase, bool) {
+func (c *DatabaseCache) GetCachedRevisionDb(revisionDbName string) (SqlDatabase, bool) {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
 	
 	if c.revisionDbs == nil {
 		return nil, false
 	}
-
-	dbsForKey, ok := c.revisionDbs[key]
-	if !ok {
-		return nil, false
-	}
-
-	db, ok := dbsForKey[revisionDbName]
+	
+	db, ok := c.revisionDbs[revisionDbName]
 	return db, ok
 }
 
 // CacheRevisionDb caches the revision database named
-func (c *SessionCache) CacheRevisionDb(key doltdb.DataCacheKey, revisionDbName string, database SqlDatabase) {
+func (c *DatabaseCache) CacheRevisionDb(revisionDbName string, database SqlDatabase) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
 	if c.revisionDbs == nil {
-		c.revisionDbs = make(map[doltdb.DataCacheKey]map[string]SqlDatabase)
+		c.revisionDbs = make(map[string]SqlDatabase)
 	}
 	
 	if len(c.revisionDbs) > maxCachedKeys {
@@ -230,19 +237,13 @@ func (c *SessionCache) CacheRevisionDb(key doltdb.DataCacheKey, revisionDbName s
 			delete(c.revisionDbs, k)
 		}
 	}
-
-	dbsForKey, ok := c.revisionDbs[key]
-	if !ok {
-		dbsForKey = make(map[string]SqlDatabase)
-		c.revisionDbs[key] = dbsForKey
-	}
-
-	dbsForKey[revisionDbName] = database
+	
+	c.revisionDbs[revisionDbName] = database
 }
 
 // GetCachedInitialDbState returns the cached initial state for the revision database named, and whether the cache 
 // was present
-func (c *SessionCache) GetCachedInitialDbState(key doltdb.DataCacheKey, revisionDbName string) (InitialDbState, bool) {
+func (c *DatabaseCache) GetCachedInitialDbState(key doltdb.DataCacheKey, revisionDbName string) (InitialDbState, bool) {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
 
@@ -260,7 +261,7 @@ func (c *SessionCache) GetCachedInitialDbState(key doltdb.DataCacheKey, revision
 }
 
 // CacheInitialDbState caches the initials state for the revision database named
-func (c *SessionCache) CacheInitialDbState(key doltdb.DataCacheKey, revisionDbName string, state InitialDbState) {
+func (c *DatabaseCache) CacheInitialDbState(key doltdb.DataCacheKey, revisionDbName string, state InitialDbState) {
 	c.mu.Lock()
 	defer c.mu.Unlock()