go/doltcore/dbfactory: differentiate journal factory from file factory

go/libraries/doltcore/dbfactory/factory.go

@@ -36,6 +36,9 @@ const (
 	// FileScheme
 	FileScheme = "file"
 
+	// JournalScheme
+	JournalScheme = "journal"
+
 	// MemScheme
 	MemScheme = "mem"
 
@@ -70,6 +73,7 @@ var DBFactories = map[string]DBFactory{
 	OSSScheme:     OSSFactory{},
 	GSScheme:      GSFactory{},
 	FileScheme:    FileFactory{},
+	JournalScheme: JournalFactory{},
 	MemScheme:     MemFactory{},
 	LocalBSScheme: LocalBSFactory{},
 	HTTPScheme:    NewDoltRemoteFactory(true),

go/libraries/doltcore/dbfactory/file.go

@@ -17,11 +17,9 @@ package dbfactory
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net/url"
 	"os"
 	"path/filepath"
-	"sync"
 
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 	"github.com/dolthub/dolt/go/store/datas"
@@ -45,33 +43,6 @@ var DoltDataDir = filepath.Join(DoltDir, DataDir)
 type FileFactory struct {
 }
 
-type singletonDB struct {
-	ddb datas.Database
-	vrw types.ValueReadWriter
-	ns  tree.NodeStore
-}
-
-var singletonLock = new(sync.Mutex)
-var singletons = make(map[string]singletonDB)
-
-func CloseAllLocalDatabases() (err error) {
-	singletonLock.Lock()
-	defer singletonLock.Unlock()
-	for name, s := range singletons {
-		if cerr := s.ddb.Close(); cerr != nil {
-			err = fmt.Errorf("error closing DB %s (%s)", name, cerr)
-		}
-	}
-	return
-}
-
-func DeleteFromSingletonCache(path string) error {
-	singletonLock.Lock()
-	defer singletonLock.Unlock()
-	delete(singletons, path)
-	return nil
-}
-
 // PrepareDB creates the directory for the DB if it doesn't exist, and returns an error if a file or symlink is at the
 // path given
 func (fact FileFactory) PrepareDB(ctx context.Context, nbf *types.NomsBinFormat, u *url.URL, params map[string]interface{}) error {
@@ -100,15 +71,7 @@ func (fact FileFactory) PrepareDB(ctx context.Context, nbf *types.NomsBinFormat,
 
 // CreateDB creates a local filesys backed database
 func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
-	singletonLock.Lock()
-	defer singletonLock.Unlock()
-
-	if s, ok := singletons[urlObj.String()]; ok {
-		return s.ddb, s.vrw, s.ns, nil
-	}
-
 	path, err := url.PathUnescape(urlObj.Path)
-
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -123,12 +86,7 @@ func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat,
 
 	var newGenSt *nbs.NomsBlockStore
 	q := nbs.NewUnlimitedMemQuotaProvider()
-	if nbs.UseJournalStore(path) {
-		newGenSt, err = nbs.NewLocalJournalingStore(ctx, nbf.VersionString(), path, q)
-	} else {
-		newGenSt, err = nbs.NewLocalStore(ctx, nbf.VersionString(), path, defaultMemTableSize, q)
-	}
-
+	newGenSt, err = nbs.NewLocalStore(ctx, nbf.VersionString(), path, defaultMemTableSize, q)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -158,13 +116,6 @@ func (fact FileFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat,
 	vrw := types.NewValueStore(st)
 	ns := tree.NewNodeStore(st)
 	ddb := datas.NewTypesDatabase(vrw, ns)
-
-	singletons[urlObj.String()] = singletonDB{
-		ddb: ddb,
-		vrw: vrw,
-		ns:  ns,
-	}
-
 	return ddb, vrw, ns, nil
 }
 

go/libraries/doltcore/dbfactory/journal.go

@@ -0,0 +1,148 @@
+// Copyright 2019 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dbfactory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/dolthub/dolt/go/libraries/utils/filesys"
+	"github.com/dolthub/dolt/go/store/datas"
+	"github.com/dolthub/dolt/go/store/nbs"
+	"github.com/dolthub/dolt/go/store/prolly/tree"
+	"github.com/dolthub/dolt/go/store/types"
+)
+
+// JournalFactory is a DBFactory implementation for creating local filesys backed databases
+type JournalFactory struct{}
+
+type singletonDB struct {
+	ddb datas.Database
+	vrw types.ValueReadWriter
+	ns  tree.NodeStore
+}
+
+var singletonLock = new(sync.Mutex)
+var singletons = make(map[string]singletonDB)
+
+func CloseAllLocalDatabases() (err error) {
+	singletonLock.Lock()
+	defer singletonLock.Unlock()
+	for name, s := range singletons {
+		if cerr := s.ddb.Close(); cerr != nil {
+			err = fmt.Errorf("error closing DB %s (%s)", name, cerr)
+		}
+	}
+	return
+}
+
+func DeleteFromSingletonCache(path string) error {
+	singletonLock.Lock()
+	defer singletonLock.Unlock()
+	delete(singletons, path)
+	return nil
+}
+
+// PrepareDB creates the directory for the DB if it doesn't exist, and returns an error if a file or symlink is at the
+// path given
+func (fact JournalFactory) PrepareDB(ctx context.Context, nbf *types.NomsBinFormat, u *url.URL, params map[string]interface{}) error {
+	path, err := url.PathUnescape(u.Path)
+	if err != nil {
+		return err
+	}
+
+	path = filepath.FromSlash(path)
+	path = u.Host + path
+
+	info, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		return os.MkdirAll(path, os.ModePerm)
+	}
+
+	if err != nil {
+		return err
+	} else if !info.IsDir() {
+		return filesys.ErrIsFile
+	}
+	return nil
+}
+
+// CreateDB creates a local filesys backed database
+func (fact JournalFactory) CreateDB(ctx context.Context, nbf *types.NomsBinFormat, urlObj *url.URL, params map[string]interface{}) (datas.Database, types.ValueReadWriter, tree.NodeStore, error) {
+	singletonLock.Lock()
+	defer singletonLock.Unlock()
+
+	if s, ok := singletons[urlObj.String()]; ok {
+		return s.ddb, s.vrw, s.ns, nil
+	}
+
+	path, err := url.PathUnescape(urlObj.Path)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	path = filepath.FromSlash(path)
+	path = urlObj.Host + path
+
+	err = validateDir(path)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	var newGenSt *nbs.NomsBlockStore
+	q := nbs.NewUnlimitedMemQuotaProvider()
+	newGenSt, err = nbs.NewLocalJournalingStore(ctx, nbf.VersionString(), path, q)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	oldgenPath := filepath.Join(path, "oldgen")
+	err = validateDir(oldgenPath)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return nil, nil, nil, err
+		}
+
+		err = os.Mkdir(oldgenPath, os.ModePerm)
+		if err != nil && !errors.Is(err, os.ErrExist) {
+			return nil, nil, nil, err
+		}
+	}
+
+	oldGenSt, err := nbs.NewLocalStore(ctx, newGenSt.Version(), oldgenPath, defaultMemTableSize, q)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	st := nbs.NewGenerationalCS(oldGenSt, newGenSt)
+	// metrics?
+
+	vrw := types.NewValueStore(st)
+	ns := tree.NewNodeStore(st)
+	ddb := datas.NewTypesDatabase(vrw, ns)
+
+	singletons[urlObj.String()] = singletonDB{
+		ddb: ddb,
+		vrw: vrw,
+		ns:  ns,
+	}
+
+	return ddb, vrw, ns, nil
+}

go/libraries/doltcore/doltdb/doltdb.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -36,6 +37,10 @@ import (
 )
 
 func init() {
+	// default to chunk journal unless feature flag is set
+	if os.Getenv("DOLT_DISABLE_CHUNK_JOURNAL") != "" {
+		LocalDirDoltDB = "file://./" + dbfactory.DoltDataDir
+	}
 	types.CreateEditAccForMapEdits = edits.NewAsyncSortedEditsWithDefaults
 }
 
@@ -53,7 +58,7 @@ const (
 )
 
 // LocalDirDoltDB stores the db in the current directory
-var LocalDirDoltDB = "file://./" + dbfactory.DoltDataDir
+var LocalDirDoltDB = "journal://./" + dbfactory.DoltDataDir
 
 // InMemDoltDB stores the DoltDB db in memory and is primarily used for testing
 var InMemDoltDB = "mem://"
@@ -89,7 +94,7 @@ func HackDatasDatabaseFromDoltDB(ddb *DoltDB) datas.Database {
 // to a newly created in memory database will be used. If the location is LocalDirDoltDB, the directory must exist or
 // this returns nil.
 func LoadDoltDB(ctx context.Context, nbf *types.NomsBinFormat, urlStr string, fs filesys.Filesys) (*DoltDB, error) {
-	return LoadDoltDBWithParams(ctx, nbf, urlStr, fs, nil)
+	return LoadDoltDBWithParams(ctx, nbf, urlStr, fs, map[string]interface{}{"journal": true})
 }
 
 func LoadDoltDBWithParams(ctx context.Context, nbf *types.NomsBinFormat, urlStr string, fs filesys.Filesys, params map[string]interface{}) (*DoltDB, error) {

go/store/nbs/journal.go

@@ -19,7 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"path/filepath"
 	"sort"
 	"time"
@@ -28,25 +27,6 @@ import (
 	"github.com/dolthub/dolt/go/store/hash"
 )
 
-var chunkJournalFeatureFlag = true
-
-func init() {
-	if os.Getenv("DOLT_DISABLE_CHUNK_JOURNAL") != "" {
-		chunkJournalFeatureFlag = false
-	}
-}
-
-func UseJournalStore(path string) bool {
-	if chunkJournalFeatureFlag {
-		return true
-	}
-	ok, err := fileExists(filepath.Join(path, chunkJournalAddr))
-	if err != nil {
-		panic(err)
-	}
-	return ok
-}
-
 const (
 	chunkJournalName = chunkJournalAddr // todo
 )

go/store/nbs/store.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -486,6 +487,12 @@ func newLocalStore(ctx context.Context, nbfVerStr string, dir string, memTableSi
 	if err := checkDir(dir); err != nil {
 		return nil, err
 	}
+	ok, err := fileExists(filepath.Join(dir, chunkJournalAddr))
+	if err != nil {
+		return nil, err
+	} else if ok {
+		return nil, fmt.Errorf("cannot create NBS store for directory containing chunk journal: %s", dir)
+	}
 
 	m, err := getFileManifest(ctx, dir, asyncFlush)
 	if err != nil {