dolt/go/store/types: format: Add error handling to parsing format version strings.

This commit is contained in:
Aaron Son
2019-07-19 17:16:51 -07:00
parent 2d74d6df38
commit 0d68c7adb1
17 changed files with 87 additions and 79 deletions

View File

@@ -17,7 +17,6 @@ import (
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/argparser"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/earl"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/datas"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
"github.com/liquidata-inc/ld/dolt/go/store/types"
@@ -159,7 +158,7 @@ func createRemote(remoteName, remoteUrl string, params map[string]string) (env.R
r := env.NewRemote(remoteName, remoteUrl, params)
ddb, err := r.GetRemoteDB(context.TODO(), types.GetFormatForVersionString(constants.DefaultNomsBinFormat))
ddb, err := r.GetRemoteDB(context.TODO(), types.Format_Default)
if err != nil {
bdr := errhand.BuildDError("error: failed to get remote db").AddCause(err)

View File

@@ -2,13 +2,13 @@ package commands
import (
"context"
"github.com/fatih/color"
"github.com/liquidata-inc/ld/dolt/go/cmd/dolt/cli"
"github.com/liquidata-inc/ld/dolt/go/cmd/dolt/errhand"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/doltdb"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/env"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/argparser"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/types"
)
@@ -59,8 +59,7 @@ func Init(commandStr string, args []string, dEnv *env.DoltEnv) int {
return 1
}
nbf := types.GetFormatForVersionString(constants.DefaultNomsBinFormat)
err := dEnv.InitRepo(context.Background(), nbf, name, email)
err := dEnv.InitRepo(context.Background(), types.Format_Default, name, email)
if err != nil {
cli.PrintErrln(color.RedString("Failed to initialize directory as a data repo. %s", err.Error()))

View File

@@ -4,6 +4,9 @@ import (
"context"
"crypto/tls"
"fmt"
"path/filepath"
"strings"
"github.com/liquidata-inc/ld/dolt/go/cmd/dolt/errhand"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/creds"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/dbfactory"
@@ -12,14 +15,11 @@ import (
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/schema"
"github.com/liquidata-inc/ld/dolt/go/libraries/doltcore/schema/encoding"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
"github.com/liquidata-inc/ld/dolt/go/store/types"
"github.com/pkg/errors"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"path/filepath"
"strings"
)
const (
@@ -53,7 +53,7 @@ type DoltEnv struct {
func Load(ctx context.Context, hdp HomeDirProvider, fs filesys.Filesys, urlStr string) *DoltEnv {
config, cfgErr := loadDoltCliConfig(hdp, fs)
repoState, rsErr := LoadRepoState(fs)
ddb, dbLoadErr := doltdb.LoadDoltDB(ctx, types.GetFormatForVersionString(constants.DefaultNomsBinFormat), urlStr)
ddb, dbLoadErr := doltdb.LoadDoltDB(ctx, types.Format_Default, urlStr)
dEnv := &DoltEnv{
config,

View File

@@ -8,7 +8,6 @@ import (
"context"
"testing"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/datas"
"github.com/liquidata-inc/ld/dolt/go/store/nbs"
"github.com/liquidata-inc/ld/dolt/go/store/spec"
@@ -28,7 +27,7 @@ type nomsDsTestSuite struct {
func (s *nomsDsTestSuite) TestEmptyNomsDs() {
dir := s.DBDir
cs, err := nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, dir, clienttest.DefaultMemTableSize)
cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), dir, clienttest.DefaultMemTableSize)
s.NoError(err)
ds := datas.NewDatabase(cs)
@@ -42,7 +41,7 @@ func (s *nomsDsTestSuite) TestEmptyNomsDs() {
func (s *nomsDsTestSuite) TestNomsDs() {
dir := s.DBDir
cs, err := nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, dir, clienttest.DefaultMemTableSize)
cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), dir, clienttest.DefaultMemTableSize)
s.NoError(err)
db := datas.NewDatabase(cs)

View File

@@ -9,7 +9,6 @@ import (
"os"
"testing"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/datas"
"github.com/liquidata-inc/ld/dolt/go/store/nbs"
"github.com/liquidata-inc/ld/dolt/go/store/spec"
@@ -27,7 +26,7 @@ type nomsSyncTestSuite struct {
}
func (s *nomsSyncTestSuite) TestSyncValidation() {
cs, err := nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir, clienttest.DefaultMemTableSize)
cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir, clienttest.DefaultMemTableSize)
s.NoError(err)
sourceDB := datas.NewDatabase(cs)
source1, err := sourceDB.GetDataset(context.Background(), "src")
@@ -51,7 +50,7 @@ func (s *nomsSyncTestSuite) TestSyncValidation() {
func (s *nomsSyncTestSuite) TestSync() {
defer s.NoError(os.RemoveAll(s.DBDir2))
cs, err := nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir, clienttest.DefaultMemTableSize)
cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir, clienttest.DefaultMemTableSize)
s.NoError(err)
sourceDB := datas.NewDatabase(cs)
source1, err := sourceDB.GetDataset(context.Background(), "src")
@@ -69,7 +68,7 @@ func (s *nomsSyncTestSuite) TestSync() {
sout, _ := s.MustRun(main, []string{"sync", sourceSpec, sinkDatasetSpec})
s.Regexp("Synced", sout)
cs, err = nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir2, clienttest.DefaultMemTableSize)
cs, err = nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir2, clienttest.DefaultMemTableSize)
s.NoError(err)
db := datas.NewDatabase(cs)
dest, err := db.GetDataset(context.Background(), "dest")
@@ -82,7 +81,7 @@ func (s *nomsSyncTestSuite) TestSync() {
sout, _ = s.MustRun(main, []string{"sync", sourceDataset, sinkDatasetSpec})
s.Regexp("Synced", sout)
cs, err = nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir2, clienttest.DefaultMemTableSize)
cs, err = nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir2, clienttest.DefaultMemTableSize)
s.NoError(err)
db = datas.NewDatabase(cs)
dest, err = db.GetDataset(context.Background(), "dest")
@@ -99,7 +98,7 @@ func (s *nomsSyncTestSuite) TestSync() {
sout, _ = s.MustRun(main, []string{"sync", sourceDataset, sinkDatasetSpec})
s.Regexp("Created", sout)
cs, err = nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir2, clienttest.DefaultMemTableSize)
cs, err = nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir2, clienttest.DefaultMemTableSize)
s.NoError(err)
db = datas.NewDatabase(cs)
dest, err = db.GetDataset(context.Background(), "dest2")
@@ -111,7 +110,7 @@ func (s *nomsSyncTestSuite) TestSync() {
func (s *nomsSyncTestSuite) TestSync_Issue2598() {
defer s.NoError(os.RemoveAll(s.DBDir2))
cs, err := nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir, clienttest.DefaultMemTableSize)
cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir, clienttest.DefaultMemTableSize)
s.NoError(err)
sourceDB := datas.NewDatabase(cs)
// Create dataset "src1", which has a lineage of two commits.
@@ -134,7 +133,7 @@ func (s *nomsSyncTestSuite) TestSync_Issue2598() {
sourceDataset := spec.CreateValueSpecString("nbs", s.DBDir, "src1")
sinkDatasetSpec := spec.CreateValueSpecString("nbs", s.DBDir2, "dest")
sout, _ := s.MustRun(main, []string{"sync", sourceDataset, sinkDatasetSpec})
cs, err = nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir2, clienttest.DefaultMemTableSize)
cs, err = nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir2, clienttest.DefaultMemTableSize)
db := datas.NewDatabase(cs)
dest, err := db.GetDataset(context.Background(), "dest")
s.NoError(err)
@@ -145,7 +144,7 @@ func (s *nomsSyncTestSuite) TestSync_Issue2598() {
sourceDataset2 := spec.CreateValueSpecString("nbs", s.DBDir, "src2")
sinkDatasetSpec2 := spec.CreateValueSpecString("nbs", s.DBDir2, "dest2")
sout, _ = s.MustRun(main, []string{"sync", sourceDataset2, sinkDatasetSpec2})
cs, err = nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir2, clienttest.DefaultMemTableSize)
cs, err = nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir2, clienttest.DefaultMemTableSize)
s.NoError(err)
db = datas.NewDatabase(cs)
dest, err = db.GetDataset(context.Background(), "dest2")
@@ -159,7 +158,7 @@ func (s *nomsSyncTestSuite) TestSync_Issue2598() {
func (s *nomsSyncTestSuite) TestRewind() {
var err error
cs, err := nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir, clienttest.DefaultMemTableSize)
cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir, clienttest.DefaultMemTableSize)
s.NoError(err)
sourceDB := datas.NewDatabase(cs)
src, err := sourceDB.GetDataset(context.Background(), "foo")
@@ -175,7 +174,7 @@ func (s *nomsSyncTestSuite) TestRewind() {
sinkDatasetSpec := spec.CreateValueSpecString("nbs", s.DBDir, "foo")
s.MustRun(main, []string{"sync", sourceSpec, sinkDatasetSpec})
cs, err = nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, s.DBDir, clienttest.DefaultMemTableSize)
cs, err = nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), s.DBDir, clienttest.DefaultMemTableSize)
s.NoError(err)
db := datas.NewDatabase(cs)
dest, err := db.GetDataset(context.Background(), "foo")

View File

@@ -5,17 +5,10 @@
// Package constants collects common constants used in Noms, such as the Noms data format version.
package constants
import "os"
var DefaultNomsBinFormat = "7.18"
func init() {
nbf := os.Getenv("DOLT_DEFAULT_BIN_FORMAT")
if nbf != "" {
DefaultNomsBinFormat = nbf
}
}
const NomsVersion = "7.18"
var NomsGitSHA = "<developer build>"
const Format718String = "7.18"
const FormatLD1String = "__LD_1__"
const FormatDefaultString = "7.18"

View File

@@ -19,10 +19,10 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/dustin/go-humanize"
flag "github.com/juju/gnuflag"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/chunks"
"github.com/liquidata-inc/ld/dolt/go/store/d"
"github.com/liquidata-inc/ld/dolt/go/store/nbs"
"github.com/liquidata-inc/ld/dolt/go/store/types"
"github.com/liquidata-inc/ld/dolt/go/store/util/profile"
"github.com/stretchr/testify/assert"
)
@@ -89,7 +89,7 @@ func main() {
err := os.RemoveAll(dir)
d.PanicIfError(err)
}()
open = func() (chunks.ChunkStore, error) { return nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, dir, bufSize) }
open = func() (chunks.ChunkStore, error) { return nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), dir, bufSize) }
reset = func() {
err := os.RemoveAll(dir)
d.PanicIfError(err)
@@ -118,7 +118,7 @@ func main() {
} else if *toAWS != "" {
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
open = func() (chunks.ChunkStore, error) {
return nbs.NewAWSStore(context.Background(), constants.DefaultNomsBinFormat, dynamoTable, *toAWS, s3Bucket, s3.New(sess), dynamodb.New(sess), bufSize)
return nbs.NewAWSStore(context.Background(), types.Format_Default.VersionString(), dynamoTable, *toAWS, s3Bucket, s3.New(sess), dynamodb.New(sess), bufSize)
}
reset = func() {
ddb := dynamodb.New(sess)
@@ -139,11 +139,11 @@ func main() {
}
} else {
if *useNBS != "" {
open = func() (chunks.ChunkStore, error) { return nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, *useNBS, bufSize) }
open = func() (chunks.ChunkStore, error) { return nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), *useNBS, bufSize) }
} else if *useAWS != "" {
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
open = func() (chunks.ChunkStore, error) {
return nbs.NewAWSStore(context.Background(), constants.DefaultNomsBinFormat, dynamoTable, *useAWS, s3Bucket, s3.New(sess), dynamodb.New(sess), bufSize)
return nbs.NewAWSStore(context.Background(), types.Format_Default.VersionString(), dynamoTable, *useAWS, s3Bucket, s3.New(sess), dynamodb.New(sess), bufSize)
}
}
writeDB = func() {}

View File

@@ -41,7 +41,7 @@ func (suite *BlockStoreSuite) SetupTest() {
var err error
suite.dir, err = ioutil.TempDir("", "")
suite.NoError(err)
suite.store, err = NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, suite.dir, testMemTableSize)
suite.store, err = NewLocalStore(context.Background(), constants.FormatDefaultString, suite.dir, testMemTableSize)
suite.NoError(err)
suite.putCountFn = func() int {
return int(suite.store.putCount)
@@ -59,7 +59,7 @@ func (suite *BlockStoreSuite) TearDownTest() {
func (suite *BlockStoreSuite) TestChunkStoreMissingDir() {
newDir := filepath.Join(suite.dir, "does-not-exist")
_, err := NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, newDir, testMemTableSize)
_, err := NewLocalStore(context.Background(), constants.FormatDefaultString, newDir, testMemTableSize)
suite.Error(err)
}
@@ -68,7 +68,7 @@ func (suite *BlockStoreSuite) TestChunkStoreNotDir() {
_, err := os.Create(existingFile)
suite.NoError(err)
_, err = NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, existingFile, testMemTableSize)
_, err = NewLocalStore(context.Background(), constants.FormatDefaultString, existingFile, testMemTableSize)
suite.Error(err)
}
@@ -252,7 +252,7 @@ func (suite *BlockStoreSuite) TestChunkStoreFlushOptimisticLockFail() {
root, err := suite.store.Root(context.Background())
suite.NoError(err)
interloper, err := NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, suite.dir, testMemTableSize)
interloper, err := NewLocalStore(context.Background(), constants.FormatDefaultString, suite.dir, testMemTableSize)
suite.NoError(err)
err = interloper.Put(context.Background(), c1)
suite.NoError(err)
@@ -298,7 +298,7 @@ func (suite *BlockStoreSuite) TestChunkStoreRebaseOnNoOpFlush() {
input1 := []byte("abc")
c1 := chunks.NewChunk(input1)
interloper, err := NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, suite.dir, testMemTableSize)
interloper, err := NewLocalStore(context.Background(), constants.FormatDefaultString, suite.dir, testMemTableSize)
suite.NoError(err)
err = interloper.Put(context.Background(), c1)
suite.NoError(err)
@@ -334,7 +334,7 @@ func (suite *BlockStoreSuite) TestChunkStorePutWithRebase() {
root, err := suite.store.Root(context.Background())
suite.NoError(err)
interloper, err := NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, suite.dir, testMemTableSize)
interloper, err := NewLocalStore(context.Background(), constants.FormatDefaultString, suite.dir, testMemTableSize)
suite.NoError(err)
err = interloper.Put(context.Background(), c1)
suite.NoError(err)
@@ -419,7 +419,7 @@ func TestBlockStoreConjoinOnCommit(t *testing.T) {
p := newFakeTablePersister()
c := &fakeConjoiner{}
smallTableStore, err := newNomsBlockStore(context.Background(), constants.DefaultNomsBinFormat, mm, p, c, testMemTableSize)
smallTableStore, err := newNomsBlockStore(context.Background(), constants.FormatDefaultString, mm, p, c, testMemTableSize)
assert.NoError(t, err)
root, err := smallTableStore.Root(context.Background())
@@ -460,7 +460,7 @@ func TestBlockStoreConjoinOnCommit(t *testing.T) {
[]cannedConjoin{makeCanned(upstream[:2], upstream[2:], p)},
}
smallTableStore, err := newNomsBlockStore(context.Background(), constants.DefaultNomsBinFormat, makeManifestManager(fm), p, c, testMemTableSize)
smallTableStore, err := newNomsBlockStore(context.Background(), constants.FormatDefaultString, makeManifestManager(fm), p, c, testMemTableSize)
assert.NoError(t, err)
root, err := smallTableStore.Root(context.Background())
@@ -491,7 +491,7 @@ func TestBlockStoreConjoinOnCommit(t *testing.T) {
},
}
smallTableStore, err := newNomsBlockStore(context.Background(), constants.DefaultNomsBinFormat, makeManifestManager(fm), p, c, testMemTableSize)
smallTableStore, err := newNomsBlockStore(context.Background(), constants.FormatDefaultString, makeManifestManager(fm), p, c, testMemTableSize)
assert.NoError(t, err)
root, err := smallTableStore.Root(context.Background())

View File

@@ -10,7 +10,7 @@ import (
"io/ioutil"
"os"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/types"
"github.com/liquidata-inc/ld/dolt/go/store/chunks"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
)
@@ -26,7 +26,7 @@ func NewCache(ctx context.Context) (*NomsBlockCache, error) {
return nil, err
}
store, err := NewLocalStore(ctx, constants.DefaultNomsBinFormat, dir, defaultCacheMemTableSize)
store, err := NewLocalStore(ctx, types.Format_Default.VersionString(), dir, defaultCacheMemTableSize)
if err != nil {
return nil, err

View File

@@ -18,7 +18,6 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/dustin/go-humanize"
flag "github.com/juju/gnuflag"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/datas"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
"github.com/liquidata-inc/ld/dolt/go/store/nbs"
@@ -52,7 +51,7 @@ func main() {
var store *nbs.NomsBlockStore
if *dir != "" {
var err error
store, err = nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, *dir, memTableSize)
store, err = nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), *dir, memTableSize)
d.PanicIfError(err)
*dbName = *dir
@@ -60,7 +59,7 @@ func main() {
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
var err error
store, err = nbs.NewAWSStore(context.Background(), constants.DefaultNomsBinFormat, *table, *dbName, *bucket, s3.New(sess), dynamodb.New(sess), memTableSize)
store, err = nbs.NewAWSStore(context.Background(), types.Format_Default.VersionString(), *table, *dbName, *bucket, s3.New(sess), dynamodb.New(sess), memTableSize)
d.PanicIfError(err)
} else {
log.Fatalf("Must set either --dir or ALL of --table, --bucket and --db\n")

View File

@@ -144,7 +144,7 @@ func TestChunkStoreManifestFirstWriteByOtherProcess(t *testing.T) {
newRoot, chunks, err := interloperWrite(fm, p, []byte("new root"), []byte("hello2"), []byte("goodbye2"), []byte("badbye2"))
assert.NoError(err)
store, err := newNomsBlockStore(context.Background(), constants.DefaultNomsBinFormat, mm, p, inlineConjoiner{defaultMaxTables}, defaultMemTableSize)
store, err := newNomsBlockStore(context.Background(), constants.FormatDefaultString, mm, p, inlineConjoiner{defaultMaxTables}, defaultMemTableSize)
assert.NoError(err)
defer store.Close()
@@ -181,12 +181,12 @@ func TestChunkStoreManifestPreemptiveOptimisticLockFail(t *testing.T) {
p := newFakeTablePersister()
c := inlineConjoiner{defaultMaxTables}
store, err := newNomsBlockStore(context.Background(), constants.DefaultNomsBinFormat, mm, p, c, defaultMemTableSize)
store, err := newNomsBlockStore(context.Background(), constants.FormatDefaultString, mm, p, c, defaultMemTableSize)
assert.NoError(err)
defer store.Close()
// Simulate another goroutine writing a manifest behind store's back.
interloper, err := newNomsBlockStore(context.Background(), constants.DefaultNomsBinFormat, mm, p, c, defaultMemTableSize)
interloper, err := newNomsBlockStore(context.Background(), constants.FormatDefaultString, mm, p, c, defaultMemTableSize)
assert.NoError(err)
defer interloper.Close()
@@ -224,7 +224,7 @@ func TestChunkStoreCommitLocksOutFetch(t *testing.T) {
p := newFakeTablePersister()
c := inlineConjoiner{defaultMaxTables}
store, err := newNomsBlockStore(context.Background(), constants.DefaultNomsBinFormat, mm, p, c, defaultMemTableSize)
store, err := newNomsBlockStore(context.Background(), constants.FormatDefaultString, mm, p, c, defaultMemTableSize)
assert.NoError(err)
defer store.Close()
@@ -265,7 +265,7 @@ func TestChunkStoreSerializeCommits(t *testing.T) {
p := newFakeTablePersister()
c := inlineConjoiner{defaultMaxTables}
store, err := newNomsBlockStore(context.Background(), constants.DefaultNomsBinFormat, manifestManager{upm, mc, l}, p, c, defaultMemTableSize)
store, err := newNomsBlockStore(context.Background(), constants.FormatDefaultString, manifestManager{upm, mc, l}, p, c, defaultMemTableSize)
assert.NoError(err)
defer store.Close()
@@ -275,7 +275,7 @@ func TestChunkStoreSerializeCommits(t *testing.T) {
interloper, err := newNomsBlockStore(
context.Background(),
constants.DefaultNomsBinFormat,
constants.FormatDefaultString,
manifestManager{
updatePreemptManifest{fm, func() { updateCount++ }}, mc, l,
},
@@ -320,7 +320,7 @@ func makeStoreWithFakes(t *testing.T) (fm *fakeManifest, p tablePersister, store
fm = &fakeManifest{}
mm := manifestManager{fm, newManifestCache(0), newManifestLocks()}
p = newFakeTablePersister()
store, err := newNomsBlockStore(context.Background(), constants.DefaultNomsBinFormat, mm, p, inlineConjoiner{defaultMaxTables}, 0)
store, err := newNomsBlockStore(context.Background(), constants.FormatDefaultString, mm, p, inlineConjoiner{defaultMaxTables}, 0)
assert.NoError(t, err)
return
}

View File

@@ -25,7 +25,7 @@ func TestStats(t *testing.T) {
dir, err := ioutil.TempDir("", "")
assert.NoError(err)
store, err := NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, dir, testMemTableSize)
store, err := NewLocalStore(context.Background(), constants.FormatDefaultString, dir, testMemTableSize)
assert.NoError(err)
assert.EqualValues(1, stats(store).OpenLatency.Samples())

View File

@@ -23,7 +23,6 @@ import (
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/liquidata-inc/ld/dolt/go/store/chunks"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/d"
"github.com/liquidata-inc/ld/dolt/go/store/datas"
"github.com/liquidata-inc/ld/dolt/go/store/nbs"
@@ -272,7 +271,7 @@ func (sp Spec) NewChunkStore(ctx context.Context) chunks.ChunkStore {
case "gs":
return parseGCSSpec(ctx, sp.Href(), sp.Options)
case "nbs":
cs, err := nbs.NewLocalStore(ctx, constants.DefaultNomsBinFormat, sp.DatabaseName, 1<<28)
cs, err := nbs.NewLocalStore(ctx, types.Format_Default.VersionString(), sp.DatabaseName, 1<<28)
d.PanicIfError(err)
return cs
case "mem":
@@ -322,7 +321,7 @@ func parseAWSSpec(ctx context.Context, awsURL string, options SpecOptions) chunk
}
sess := session.Must(session.NewSession(awsConfig))
cs, err := nbs.NewAWSStore(ctx, constants.DefaultNomsBinFormat, parts[0], u.Path, parts[1], s3.New(sess), dynamodb.New(sess), 1<<28)
cs, err := nbs.NewAWSStore(ctx, types.Format_Default.VersionString(), parts[0], u.Path, parts[1], s3.New(sess), dynamodb.New(sess), 1<<28)
d.PanicIfError(err)
@@ -344,7 +343,7 @@ func parseGCSSpec(ctx context.Context, gcsURL string, options SpecOptions) chunk
panic("Could not create GCSBlobstore")
}
cs, err := nbs.NewGCSStore(ctx, constants.DefaultNomsBinFormat, bucket, path, gcs, 1<<28)
cs, err := nbs.NewGCSStore(ctx, types.Format_Default.VersionString(), bucket, path, gcs, 1<<28)
d.PanicIfError(err)
@@ -437,7 +436,7 @@ func (sp Spec) createDatabase(ctx context.Context) datas.Database {
return datas.NewDatabase(parseGCSSpec(ctx, sp.Href(), sp.Options))
case "nbs":
os.Mkdir(sp.DatabaseName, 0777)
cs, err := nbs.NewLocalStore(ctx, constants.DefaultNomsBinFormat, sp.DatabaseName, 1<<28)
cs, err := nbs.NewLocalStore(ctx, types.Format_Default.VersionString(), sp.DatabaseName, 1<<28)
d.PanicIfError(err)
return datas.NewDatabase(cs)
case "mem":

View File

@@ -12,7 +12,6 @@ import (
"path/filepath"
"testing"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/chunks"
"github.com/liquidata-inc/ld/dolt/go/store/d"
"github.com/liquidata-inc/ld/dolt/go/store/datas"
@@ -118,7 +117,7 @@ func TestNBSDatabaseSpec(t *testing.T) {
store1 := filepath.Join(tmpDir, "store1")
os.Mkdir(store1, 0777)
func() {
cs, err := nbs.NewLocalStore(context.Background(), constants.DefaultNomsBinFormat, store1, 8*(1<<20))
cs, err := nbs.NewLocalStore(context.Background(), types.Format_Default.VersionString(), store1, 8*(1<<20))
assert.NoError(err)
db := datas.NewDatabase(cs)
defer db.Close()

View File

@@ -1,5 +1,9 @@
package types
import "errors"
import "os"
import "github.com/liquidata-inc/ld/dolt/go/store/constants"
type NomsBinFormat struct {
tag *formatTag
}
@@ -12,26 +16,40 @@ var formatTag_LD_1 = &formatTag{}
var Format_7_18 = &NomsBinFormat{}
var Format_LD_1 = &NomsBinFormat{formatTag_LD_1}
var Format_Default *NomsBinFormat
func isFormat_7_18(nbf *NomsBinFormat) bool {
return nbf.tag == formatTag_7_18
}
func GetFormatForVersionString(s string) *NomsBinFormat {
if s == "7.18" {
return Format_7_18
} else if s == "__LD_1__" {
return Format_LD_1
func GetFormatForVersionString(s string) (*NomsBinFormat, error) {
if s == constants.Format718String {
return Format_7_18, nil
} else if s == constants.FormatLD1String {
return Format_LD_1, nil
} else {
panic("Unsupported ChunkStore.Version() == " + s)
return nil, errors.New("unsupported ChunkStore version " + s)
}
}
func (nbf *NomsBinFormat) VersionString() string {
if nbf.tag == formatTag_7_18 {
return "7.18"
return constants.Format718String
} else if nbf.tag == formatTag_LD_1 {
return "__LD_1__"
return constants.FormatLD1String
} else {
panic("unrecognized NomsBinFormat tag value")
}
}
func init() {
nbfVerStr := os.Getenv("DOLT_DEFAULT_BIN_FORMAT")
if nbfVerStr == "" {
nbfVerStr = constants.FormatDefaultString
}
nbf, err := GetFormatForVersionString(nbfVerStr)
if err != nil {
panic("unrecognized value for DOLT_DEFAULT_BIN_FORMAT " + nbfVerStr)
}
Format_Default = nbf
}

View File

@@ -106,7 +106,11 @@ func newValueStoreWithCacheAndPending(cs chunks.ChunkStore, cacheSize, pendingMa
func (lvs *ValueStore) expectVersion() {
dataVersion := lvs.cs.Version()
lvs.nbf = GetFormatForVersionString(dataVersion)
nbf, err := GetFormatForVersionString(dataVersion)
if err != nil {
panic(err)
}
lvs.nbf = nbf
}
func (lvs *ValueStore) SetEnforceCompleteness(enforce bool) {

View File

@@ -3,8 +3,8 @@ package main
import (
"context"
"github.com/liquidata-inc/ld/dolt/go/libraries/utils/filesys"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/nbs"
"github.com/liquidata-inc/ld/dolt/go/store/types"
"path/filepath"
"sync"
)
@@ -46,7 +46,7 @@ func (cache *DBCache) Get(org, repo string) (*nbs.NomsBlockStore, error) {
return nil, err
}
newCS, err = nbs.NewLocalStore(context.TODO(), constants.DefaultNomsBinFormat, id, defaultMemTableSize)
newCS, err = nbs.NewLocalStore(context.TODO(), types.Format_Default.VersionString(), id, defaultMemTableSize)
if err != nil {
return nil, err