mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-04-21 11:29:51 -05:00
Code Issues Packages Projects Releases 156 Wiki Activity

dolt/go/store/nbs: Make manifest updates thread version to new manifest. Override a new manifest's version from an environment variable.

Browse Source
This commit is contained in:
Aaron Son
2019-07-17 13:07:42 -07:00
parent 52c8c1dea6
commit 1a604692e3
11 changed files with 47 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/go.sum
+13 -12
View File
@@ -2,10 +2,10 @@ cloud.google.com/go v0.0.0-20161216213822-686f0e89858e/go.mod h1:aQUYkXzVsufM+Dw
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.41.0 h1:NFvqUTDnSNYPX5oReekmB+D+90jrJIcVImxQ3qrBVgM=
cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
cloud.google.com/go v0.40.0 h1:FjSY7bOj+WzJe6TZRVtXI2b9kAYvtNg4lMbcH2+MUkk=
cloud.google.com/go v0.40.0/go.mod h1:Tk58MuI9rbLMKlAjeO/bDnteAx7tX2gJIXw4T5Jwlro=
cloud.google.com/go v0.41.0 h1:NFvqUTDnSNYPX5oReekmB+D+90jrJIcVImxQ3qrBVgM=
cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -46,7 +46,13 @@ github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible
github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible/go.mod h1:Cp18FeDCvsK+cD2QAGkqerGjrgSXLiJWnjHeY2mneBc=
github.com/aws/aws-sdk-go v0.0.0-20180223184012-ebef4262e06a/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
github.com/aws/aws-sdk-go v1.16.15/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.20.6 h1:kmy4Gvdlyez1fV4kw5RYxZzWKVyuHZHgPWeU/YvRsV4=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.20.7 h1:fRjZRYJg0wPCA8yaDdb3DeP4rVjjmEiuqYhYoqOaIJg=
github.com/aws/aws-sdk-go v1.20.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.20.8 h1:oXnNFH/8PosUvHXqZHYLUn6LCSzHZ/34H52vrB4YhpA=
github.com/aws/aws-sdk-go v1.20.8 h1:oXnNFH/8PosUvHXqZHYLUn6LCSzHZ/34H52vrB4YhpA=
github.com/aws/aws-sdk-go v1.20.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.20.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.20.17 h1:ZQ0cM9FpuufVDHlNViD8vD68IkEQL/VUOMwJPBqkaaM=
github.com/aws/aws-sdk-go v1.20.17/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
@@ -57,16 +63,10 @@ github.com/aws/aws-sdk-go v1.20.19/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi
github.com/aws/aws-sdk-go v1.20.20 h1:OAR/GtjMOhenkp1NNKr1N1FgIP3mQXHeGbRhvVIAQp0=
github.com/aws/aws-sdk-go v1.20.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20160229213445-3ac7bf7a47d1/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/aws/aws-sdk-go v1.20.6 h1:kmy4Gvdlyez1fV4kw5RYxZzWKVyuHZHgPWeU/YvRsV4=
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.20.7 h1:fRjZRYJg0wPCA8yaDdb3DeP4rVjjmEiuqYhYoqOaIJg=
github.com/aws/aws-sdk-go v1.20.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.20.8 h1:oXnNFH/8PosUvHXqZHYLUn6LCSzHZ/34H52vrB4YhpA=
github.com/aws/aws-sdk-go v1.20.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beorn7/perks v0.0.0-20160229213445-3ac7bf7a47d1/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY=
github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
@@ -242,6 +242,7 @@ github.com/liquidata-inc/go-mysql-server v0.4.1-0.20190710171053-b2883167103a h1
github.com/liquidata-inc/go-mysql-server v0.4.1-0.20190710171053-b2883167103a/go.mod h1:bXirMMsoAKR1OI9GTQJPGSyuwJlOseI9AFNAdpwRY2k=
github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0 h1:phMgajKClMUiIr+hF2LGt8KRuUa2Vd2GI1sNgHgSXoU=
github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0/go.mod h1:YC1rI9k5gx8D02ljlbxDfZe80s/iq8bGvaaQsvR+qxs=
github.com/liquidata-inc/ld v0.0.0-20190717175747-17efac059a62 h1:lcNZv5VZYpJy/w7acOWXlnFTmAR/HDLDuCD+YO/egYU=
github.com/liquidata-inc/noms v0.0.0-20190531204628-499e9652fee4/go.mod h1:hFhYPo5yEzNKzza0fdJugkoIhrubZ5yCJCxHLj7S9lI=
github.com/liquidata-inc/noms v0.9.3/go.mod h1:hFhYPo5yEzNKzza0fdJugkoIhrubZ5yCJCxHLj7S9lI=
github.com/liquidata-inc/vitess v0.0.0-20190625235908-66745781a796 h1:oDEoe6JmyZKBl+sYaqB0kwoTTKYzApnoZ2pCjvjt7us=
@@ -498,10 +499,10 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190626174449-989357319d63 h1:UsSJe9fhWNSz6emfIGPpH5DF23t7ALo2Pf3sC+/hsdg=
google.golang.org/genproto v0.0.0-20190626174449-989357319d63/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
google.golang.org/genproto v0.0.0-20190530194941-fb225487d101 h1:wuGevabY6r+ivPNagjUXGGxF+GqgMd+dBhjsxW4q9u4=
google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
google.golang.org/genproto v0.0.0-20190626174449-989357319d63 h1:UsSJe9fhWNSz6emfIGPpH5DF23t7ALo2Pf3sC+/hsdg=
google.golang.org/genproto v0.0.0-20190626174449-989357319d63/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
go/store/chunks/test_utils.go
+1
View File
@@ -6,6 +6,7 @@ package chunks
import (
"context"
"github.com/liquidata-inc/ld/dolt/go/store/d"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
"github.com/stretchr/testify/assert"
go/store/constants/version.go
+11
View File
@@ -5,6 +5,17 @@
// Package constants collects common constants used in Noms, such as the Noms data format version.
package constants
import "os"
var DefaultNomsBinFormat = "7.18"
func init() {
nbf := os.Getenv("DOLT_DEFAULT_BIN_FORMAT")
if nbf != "" {
DefaultNomsBinFormat = nbf
}
}
const NomsVersion = "7.18"
var NomsGitSHA = "<developer build>"
go/store/nbs/conjoiner.go
+1 -3
View File
@@ -10,8 +10,6 @@ import (
"sort"
"sync"
"time"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
)
type conjoiner interface {
@@ -60,7 +58,7 @@ func conjoin(ctx context.Context, upstream manifestContents, mm manifestUpdater,
specs = append(specs, keepers...)
newContents := manifestContents{
vers: constants.NomsVersion,
vers: upstream.vers,
root: upstream.root,
lock: generateLockHash(upstream.root, specs),
specs: specs,
go/store/nbs/dynamo_manifest.go
+1 -2
View File
@@ -15,7 +15,6 @@ import (
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/d"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
)
@@ -154,7 +153,7 @@ func (dm dynamoManifest) Update(ctx context.Context, lastLock addr, newContents
return manifestContents{}, errors.New("manifest not found")
}
if upstream.vers != constants.NomsVersion {
if upstream.vers != newContents.vers {
return manifestContents{}, errors.New("version mismatch")
}
go/store/nbs/file_manifest.go
+4 -4
View File
@@ -7,7 +7,6 @@ package nbs
import (
"context"
"errors"
"github.com/juju/fslock"
"io"
"io/ioutil"
"os"
@@ -15,7 +14,8 @@ import (
"strings"
"time"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/juju/fslock"
"github.com/liquidata-inc/ld/dolt/go/store/d"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
)
@@ -277,8 +277,8 @@ func (fm fileManifest) Update(ctx context.Context, lastLock addr, newContents ma
return manifestContents{}, ferr
}
if constants.NomsVersion != upstream.vers {
return manifestContents{}, errors.New("invalid noms version")
if newContents.vers != upstream.vers {
return manifestContents{}, errors.New("Update cannot change manifest version")
}
return upstream, nil
go/store/nbs/file_manifest_test.go
+2 -2
View File
@@ -158,7 +158,7 @@ func TestFileManifestUpdate(t *testing.T) {
assert.Equal(contents.specs, upstream.specs)
// Now, test the case where the optimistic lock fails, and someone else updated the root since last we checked.
contents2 := manifestContents{lock: computeAddr([]byte("locker 2")), root: hash.Of([]byte("new root 2"))}
contents2 := manifestContents{lock: computeAddr([]byte("locker 2")), root: hash.Of([]byte("new root 2")), vers: constants.NomsVersion}
upstream, err = fm.Update(context.Background(), addr{}, contents2, stats, nil)
assert.NoError(err)
assert.Equal(contents.lock, upstream.lock)
@@ -176,7 +176,7 @@ func TestFileManifestUpdate(t *testing.T) {
err = clobberManifest(fm.dir, strings.Join([]string{StorageVersion, constants.NomsVersion, jerkLock.String(), contents2.root.String(), tableName.String(), "1"}, ":"))
assert.NoError(err)
contents3 := manifestContents{lock: computeAddr([]byte("locker 3")), root: hash.Of([]byte("new root 3"))}
contents3 := manifestContents{lock: computeAddr([]byte("locker 3")), root: hash.Of([]byte("new root 3")), vers: constants.NomsVersion}
upstream, err = fm.Update(context.Background(), upstream.lock, contents3, stats, nil)
assert.NoError(err)
assert.Equal(jerkLock, upstream.lock)
go/store/nbs/store.go
+6 -5
View File
@@ -7,19 +7,20 @@ package nbs
import (
"context"
"fmt"
"github.com/pkg/errors"
"os"
"reflect"
"sort"
"sync"
"time"
"github.com/pkg/errors"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/blobstore"
"cloud.google.com/go/storage"
"github.com/dustin/go-humanize"
"github.com/liquidata-inc/ld/dolt/go/store/chunks"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
)
@@ -183,7 +184,7 @@ func (nbs *NomsBlockStore) UpdateManifest(ctx context.Context, updates map[hash.
if err != nil {
return manifestContents{}, err
} else if !ok {
contents = manifestContents{vers: constants.NomsVersion}
contents = manifestContents{vers: nbs.upstream.vers}
}
currSpecs := make(map[addr]bool)
@@ -281,7 +282,7 @@ func newNomsBlockStore(ctx context.Context, mm manifestManager, p tablePersister
p: p,
c: c,
tables: newTableSet(p),
upstream: manifestContents{vers: constants.NomsVersion},
upstream: manifestContents{vers: constants.DefaultNomsBinFormat},
mtSize: memTableSize,
stats: NewStats(),
}
@@ -761,7 +762,7 @@ func (nbs *NomsBlockStore) updateManifest(ctx context.Context, current, last has
}
newContents := manifestContents{
vers: constants.NomsVersion,
vers: nbs.upstream.vers,
root: current,
lock: generateLockHash(current, specs),
specs: specs,
go/store/types/float.go
-6
View File
@@ -7,9 +7,7 @@ package types
import (
"context"
"encoding/binary"
"math"
"github.com/liquidata-inc/ld/dolt/go/store/d"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
)
@@ -56,10 +54,6 @@ func (v Float) valueReadWriter() ValueReadWriter {
func (v Float) writeTo(w nomsWriter, nbf *NomsBinFormat) {
FloatKind.writeTo(w, nbf)
fl := float64(v)
if math.IsNaN(fl) || math.IsInf(fl, 0) {
d.Panic("%f is not a supported number", fl)
}
w.writeFloat(v, nbf)
}
go/store/types/float_util.go
+8 -1
View File
@@ -4,7 +4,10 @@
package types
import "math"
import (
"fmt"
"math"
)
func float64IsInt(f float64) bool {
return math.Trunc(f) == f
@@ -12,6 +15,10 @@ func float64IsInt(f float64) bool {
// convert float64 to int64 where f == i * 2^exp
func float64ToIntExp(f float64) (int64, int) {
if math.IsNaN(f) || math.IsInf(f, 0) {
panic(fmt.Errorf("%v is not a supported number", f))
}
if f == 0 {
return 0, 0
}
go/store/types/value_store.go
-4
View File
@@ -9,7 +9,6 @@ import (
"sync"
"github.com/liquidata-inc/ld/dolt/go/store/chunks"
"github.com/liquidata-inc/ld/dolt/go/store/constants"
"github.com/liquidata-inc/ld/dolt/go/store/d"
"github.com/liquidata-inc/ld/dolt/go/store/hash"
"github.com/liquidata-inc/ld/dolt/go/store/util/sizecache"
@@ -107,9 +106,6 @@ func newValueStoreWithCacheAndPending(cs chunks.ChunkStore, cacheSize, pendingMa
func (lvs *ValueStore) expectVersion() {
dataVersion := lvs.cs.Version()
if constants.NomsVersion != dataVersion {
d.Panic("SDK version %s incompatible with data of version %s", constants.NomsVersion, dataVersion)
}
lvs.nbf = getFormatForVersionString(dataVersion)
}