mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-01-24 03:09:22 -06:00
Code Issues Packages Projects Releases 123 Wiki Activity

go/doltcore/{dbfactory, doltdb, env}: refactor database load/creation to only use chunk journal for local databases

Browse Source
This commit is contained in:
Andy Arthur
2023-03-06 17:00:00 -08:00
parent 66150f0747
commit b7ed9f1ebb
8 changed files with 90 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
go/libraries/doltcore/dbfactory/factory.go
View File

@@ -36,9 +36,6 @@ const (
// FileScheme
FileScheme = "file"
// JournalScheme
JournalScheme = "journal"
// MemScheme
MemScheme = "mem"
@@ -73,7 +70,6 @@ var DBFactories = map[string]DBFactory{
OSSScheme: OSSFactory{},
GSScheme: GSFactory{},
FileScheme: FileFactory{},
JournalScheme: JournalFactory{},
MemScheme: MemFactory{},
LocalBSScheme: LocalBSFactory{},
HTTPScheme: NewDoltRemoteFactory(true),

66
go/libraries/doltcore/dbfactory/file.go
View File

@@ -17,9 +17,11 @@ package dbfactory
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"sync"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/datas"
@@ -28,12 +30,23 @@ import (
"github.com/dolthub/dolt/go/store/types"
)
func init() {
// default to chunk journal unless feature flag is set
if os.Getenv("DOLT_DISABLE_CHUNK_JOURNAL") != "" {
chunkJournalFeatureFlag = false
}
}
var chunkJournalFeatureFlag = true
const (
// DoltDir defines the directory used to hold the dolt repo data within the filesys
DoltDir = ".dolt"
// DataDir is the directory internal to the DoltDir which holds the noms files.
DataDir = "noms"
ChunkJournalParam = "journal"
)
// DoltDataDir is the directory where noms files will be stored
@@ -43,6 +56,33 @@ var DoltDataDir = filepath.Join(DoltDir, DataDir)
type FileFactory struct {
}
type singletonDB struct {
ddb datas.Database
vrw types.ValueReadWriter
ns tree.NodeStore
}
var singletonLock = new(sync.Mutex)
var singletons = make(map[string]singletonDB)
func CloseAllLocalDatabases() (err error) {
singletonLock.Lock()
defer singletonLock.Unlock()
for name, s := range singletons {
if cerr := s.ddb.Close(); cerr != nil {
err = fmt.Errorf("error closing DB %s (%s)", name, cerr)
}
}
return
}
func DeleteFromSingletonCache(path string) error {
singletonLock.Lock()
defer singletonLock.Unlock()
delete(singletons, path)
return nil
}
// PrepareDB creates the directory for the DB if it doesn't exist, and returns an error if a file or symlink is at the
// path given
func (fact FileFactory) PrepareDB(ctx context.Context, nbf *types.NomsBinFormat, u *url.URL, params map[string]interface{}) error {
@@ -71,6 +111,13 @@ func (fact FileFactory) PrepareDB(ctx context.Context, nbf *types.NomsBinFormat,
// CreateDB creates a local filesys backed database
func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
singletonLock.Lock()
defer singletonLock.Unlock()
if s, ok := singletons[urlObj.Path]; ok {
return s.ddb, s.vrw, s.ns, nil
}
path, err := url.PathUnescape(urlObj.Path)
if err != nil {
return nil, nil, nil, err
@@ -84,9 +131,19 @@ func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat,
return nil, nil, nil, err
}
var useJournal bool
if params != nil {
_, useJournal = params[ChunkJournalParam]
}
var newGenSt *nbs.NomsBlockStore
q := nbs.NewUnlimitedMemQuotaProvider()
newGenSt, err = nbs.NewLocalStore(ctx, nbf.VersionString(), path, defaultMemTableSize, q)
if useJournal && chunkJournalFeatureFlag {
newGenSt, err = nbs.NewLocalJournalingStore(ctx, nbf.VersionString(), path, q)
} else {
newGenSt, err = nbs.NewLocalStore(ctx, nbf.VersionString(), path, defaultMemTableSize, q)
}
if err != nil {
return nil, nil, nil, err
}
@@ -116,6 +173,13 @@ func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat,
vrw := types.NewValueStore(st)
ns := tree.NewNodeStore(st)
ddb := datas.NewTypesDatabase(vrw, ns)
singletons[urlObj.Path] = singletonDB{
ddb: ddb,
vrw: vrw,
ns: ns,
}
return ddb, vrw, ns, nil
}

148
go/libraries/doltcore/dbfactory/journal.go
View File

@@ -1,148 +0,0 @@
// Copyright 2019 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dbfactory
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"sync"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/nbs"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/dolt/go/store/types"
)
// JournalFactory is a DBFactory implementation for creating local filesys backed databases
type JournalFactory struct{}
type singletonDB struct {
ddb datas.Database
vrw types.ValueReadWriter
ns tree.NodeStore
}
var singletonLock = new(sync.Mutex)
var singletons = make(map[string]singletonDB)
func CloseAllLocalDatabases() (err error) {
singletonLock.Lock()
defer singletonLock.Unlock()
for name, s := range singletons {
if cerr := s.ddb.Close(); cerr != nil {
err = fmt.Errorf("error closing DB %s (%s)", name, cerr)
}
}
return
}
func DeleteFromSingletonCache(path string) error {
singletonLock.Lock()
defer singletonLock.Unlock()
delete(singletons, path)
return nil
}
// PrepareDB creates the directory for the DB if it doesn't exist, and returns an error if a file or symlink is at the
// path given
func (fact JournalFactory) PrepareDB(ctx context.Context, nbf *types.NomsBinFormat, u *url.URL, params map[string]interface{}) error {
path, err := url.PathUnescape(u.Path)
if err != nil {
return err
}
path = filepath.FromSlash(path)
path = u.Host + path
info, err := os.Stat(path)
if os.IsNotExist(err) {
return os.MkdirAll(path, os.ModePerm)
}
if err != nil {
return err
} else if !info.IsDir() {
return filesys.ErrIsFile
}
return nil
}
// CreateDB creates a local filesys backed database
func (fact JournalFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
singletonLock.Lock()
defer singletonLock.Unlock()
if s, ok := singletons[urlObj.String()]; ok {
return s.ddb, s.vrw, s.ns, nil
}
path, err := url.PathUnescape(urlObj.Path)
if err != nil {
return nil, nil, nil, err
}
path = filepath.FromSlash(path)
path = urlObj.Host + path
err = validateDir(path)
if err != nil {
return nil, nil, nil, err
}
var newGenSt *nbs.NomsBlockStore
q := nbs.NewUnlimitedMemQuotaProvider()
newGenSt, err = nbs.NewLocalJournalingStore(ctx, nbf.VersionString(), path, q)
if err != nil {
return nil, nil, nil, err
}
oldgenPath := filepath.Join(path, "oldgen")
err = validateDir(oldgenPath)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return nil, nil, nil, err
}
err = os.Mkdir(oldgenPath, os.ModePerm)
if err != nil && !errors.Is(err, os.ErrExist) {
return nil, nil, nil, err
}
}
oldGenSt, err := nbs.NewLocalStore(ctx, newGenSt.Version(), oldgenPath, defaultMemTableSize, q)
if err != nil {
return nil, nil, nil, err
}
st := nbs.NewGenerationalCS(oldGenSt, newGenSt)
// metrics?
vrw := types.NewValueStore(st)
ns := tree.NewNodeStore(st)
ddb := datas.NewTypesDatabase(vrw, ns)
singletons[urlObj.String()] = singletonDB{
ddb: ddb,
vrw: vrw,
ns: ns,
}
return ddb, vrw, ns, nil
}

20
go/libraries/doltcore/doltdb/doltdb.go
View File

@@ -19,14 +19,14 @@ import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/libraries/doltcore/ref"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/datas"
"github.com/dolthub/dolt/go/store/datas/pull"
@@ -37,10 +37,6 @@ import (
)
func init() {
// default to chunk journal unless feature flag is set
if os.Getenv("DOLT_DISABLE_CHUNK_JOURNAL") != "" {
LocalDirDoltDB = "file://./" + dbfactory.DoltDataDir
}
types.CreateEditAccForMapEdits = edits.NewAsyncSortedEditsWithDefaults
}
@@ -58,7 +54,7 @@ const (
)
// LocalDirDoltDB stores the db in the current directory
var LocalDirDoltDB = "journal://./" + dbfactory.DoltDataDir
var LocalDirDoltDB = "file://./" + dbfactory.DoltDataDir
// InMemDoltDB stores the DoltDB db in memory and is primarily used for testing
var InMemDoltDB = "mem://"
@@ -94,13 +90,12 @@ func HackDatasDatabaseFromDoltDB(ddb *DoltDB) datas.Database {
// to a newly created in memory database will be used. If the location is LocalDirDoltDB, the directory must exist or
// this returns nil.
func LoadDoltDB(ctx context.Context, nbf *types.NomsBinFormat, urlStr string, fs filesys.Filesys) (*DoltDB, error) {
return LoadDoltDBWithParams(ctx, nbf, urlStr, fs, map[string]interface{}{"journal": true})
return LoadDoltDBWithParams(ctx, nbf, urlStr, fs, nil)
}
func LoadDoltDBWithParams(ctx context.Context, nbf *types.NomsBinFormat, urlStr string, fs filesys.Filesys, params map[string]interface{}) (*DoltDB, error) {
if urlStr == LocalDirDoltDB {
exists, isDir := fs.Exists(dbfactory.DoltDataDir)
if !exists {
return nil, errors.New("missing dolt data directory")
} else if !isDir {
@@ -113,14 +108,17 @@ func LoadDoltDBWithParams(ctx context.Context, nbf *types.NomsBinFormat, urlStr
}
urlStr = fmt.Sprintf("file://%s", filepath.ToSlash(absPath))
if params == nil {
params = make(map[string]any)
}
params[dbfactory.ChunkJournalParam] = struct{}{}
}
db, vrw, ns, err := dbfactory.CreateDB(ctx, nbf, urlStr, params)
if err != nil {
return nil, err
}
return &DoltDB{hooksDatabase{Database: db}, vrw, ns}, nil
}

1
go/libraries/doltcore/env/remotes.go vendored
View File

@@ -457,7 +457,6 @@ func NewPullSpec(_ context.Context, rsr RepoStateReader, remoteName, remoteRefNa
func GetAbsRemoteUrl(fs filesys2.Filesys, cfg config.ReadableConfig, urlArg string) (string, string, error) {
u, err := earl.Parse(urlArg)
if err != nil {
return "", "", err
}

2
go/libraries/doltcore/sqle/database_provider.go
View File

@@ -653,7 +653,7 @@ func (p DoltDatabaseProvider) DropDatabase(ctx *sql.Context, name string) error
}
// If this database is re-created, we don't want to return any cached results.
err = dbfactory.DeleteFromSingletonCache("file://" + dropDbLoc + "/.dolt/noms")
err = dbfactory.DeleteFromSingletonCache(dropDbLoc + "/.dolt/noms")
if err != nil {
return err
}

6
go/store/nbs/journal.go
View File

@@ -32,7 +32,7 @@ const (
)
// chunkJournal is a persistence abstraction for a NomsBlockStore.
// It implemented both manifest and tablePersister, durably writing
// It implements both manifest and tablePersister, durably writing
// both memTable persists and manifest updates to a single file.
type chunkJournal struct {
wr *journalWriter
@@ -72,11 +72,11 @@ func newChunkJournal(ctx context.Context, nbfVers, dir string, m manifest, p *fs
}
// bootstrapJournalWriter initializes the journalWriter, which manages access to the
// journal file for this chunkJournal. The bootstrapping process differed depending
// journal file for this chunkJournal. The bootstrapping process differs depending
// on whether a journal file exists at startup time.
//
// If a journal file does not exist, we create one and commit a root hash record
// which we read from the manifest file.
// containing the root hash we read from the manifest file.
//
// If a journal file does exist, we process its records to build up an index of its
// resident chunks. Processing journal records is potentially accelerated by an index

46
integration-tests/bats/sql-server.bats
View File

@@ -1704,7 +1704,7 @@ s.close()
[ $status -eq 0 ]
skip "Forcefully deleting a database doesn't cause direct panics, but also doesn't stop the server"
run grep "panic" server_log.txt
[ "${#lines[@]}" -eq 0 ]
@@ -1778,12 +1778,10 @@ s.close()
dolt sql-client -P $PORT -u dolt --use-db '' -q "CREATE DATABASE mydb1;"
[ -d mydb1 ]
run dolt sql-client -P $PORT -u dolt --use-db '' -q "DROP DATABASE mydb1;"
[ $status -eq 0 ]
dolt sql-client -P $PORT -u dolt --use-db '' -q "DROP DATABASE mydb1;"
[ ! -d mydb1 ]
run dolt sql-client -P $PORT -u dolt --use-db '' -q "CREATE DATABASE mydb1;"
[ $status -eq 0 ]
dolt sql-client -P $PORT -u dolt --use-db '' -q "CREATE DATABASE mydb1;"
[ -d mydb1 ]
run dolt sql-client -P $PORT -u dolt --use-db '' -q "SHOW DATABASES;"
@@ -1809,47 +1807,27 @@ s.close()
}
@test "sql-server: dolt_clone procedure in empty dir" {
repoDir="$BATS_TMPDIR/dolt-repo-$$"
# make directories outside of the dolt repo
repo1=$(mktemp -d)
cd $repo1
# init and populate repo 1
dolt init
mkdir rem1
cd repo1
dolt sql -q "CREATE TABLE test (pk INT PRIMARY KEY);"
dolt sql -q "INSERT INTO test VALUES (1), (2), (3);"
dolt sql -q "CREATE PROCEDURE test() SELECT 42;"
dolt add -A
dolt commit -m "initial commit"
dolt remote add remote1 file://../rem1
dolt push remote1 main
# verify data
run dolt sql -q "SELECT * FROM test"
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
[[ "$output" =~ "2" ]] || false
[[ "$output" =~ "3" ]] || false
# verify procedure
run dolt sql -q "call test()"
[ "$status" -eq 0 ]
[[ "$output" =~ "42" ]] || false
# make repo 2 directory outside of the dolt repo
repo2=$(mktemp -d)
cd $repo2
# Clone repo 1 into repo 2
run dolt sql -q "call dolt_clone('file://$repo1/.dolt/noms', 'repo1');"
[ "$status" -eq 0 ]
cd ..
dolt sql -q "call dolt_clone('file://./rem1', 'repo3');"
cd repo3
# verify databases
run dolt sql -q "show databases;"
[ "$status" -eq 0 ]
[[ "$output" =~ "repo1" ]] || false
[[ "$output" =~ "repo3" ]] || false
run dolt sql -q "select database();"
[[ "$output" =~ "repo1" ]] || false
[[ "$output" =~ "repo3" ]] || false
# verify data
run dolt sql -q "SELECT * FROM test"