mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-12 11:29:01 -05:00
Code Issues Packages Projects Releases 164 Wiki Activity

Add new spec for nbs-aws (#2997)

Browse Source 
The new spec is a URI, akin to what we use for HTTP It allows the
specification of a DynamoDB table, an S3 bucket, a database ID, and a
dataset ID: aws://table-name:bucket-name/database::dataset

The bucket name is optional and, if not provided, Noms will use a
ChunkStore implementation backed only by DynamoDB.
This commit is contained in:
cmasone-attic
2017-01-02 08:24:45 -08:00
committed by GitHub
parent b5f3c8bc4f
commit ca31583a08
5 changed files with 51 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/spelling.md
+4 -2
View File
@@ -16,6 +16,8 @@ The `path` part of the name is interpreted differently depending on the protocol
- **ldb** specs describe a local [LevelDB](https://github.com/google/leveldb)-backed database. In this case, the path component should be a relative or absolute path on disk to a directory in which to store the LevelDB data. For example: `ldb:/tmp/noms-data`.
- In Go, `ldb:` can be ommitted (just `/tmp/noms-data` will work).
- **mem** specs describe an ephemeral memory-backed database. In this case, the path component is not used and must be empty.
- **nbs** specs describe a local [Noms Block Store (NBS)](https://github.com/attic-labs/noms/tree/master/go/nbs)-backed database. In this case, the path component should be a relative or absolute path on disk to a directory in which to store the data, e.g. `nbs:/tmp/noms-data`.
- **aws** specs describe a remote Noms Block Store backed directly by Amazon Web Services, specifically DynamoDB and S3. The format is a URI containing the names of the DynamoDB table to use, the S3 bucket to use, and the database to serve. For example: `aws://dynamo-table:s3-bucket/database`.
## Spelling Datasets
@@ -58,7 +60,7 @@ For example, if the `root` is a dataset, then one can use `.value` to get the ro
### Specifying Collection Values
Elements of a Noms list, map, or set can be retrieved using brackets `[...]`.
For example, if the dataset is a Noms map of number to struct then one could use `.value[42]` to get the Noms struct associated with the key 42. Similarly selecting the first element from a Noms list would be `.value[0]`. If the Noms map was keyed by string, then using `.value["0000024-02-999"]` would reference the Noms struct associated with key "0000024-02-999".
For example, if the dataset is a Noms map of number to struct then one could use `.value[42]` to get the Noms struct associated with the key 42. Similarly selecting the first element from a Noms list would be `.value[0]`. If the Noms map was keyed by string, then using `.value["0000024-02-999"]` would reference the Noms struct associated with key "0000024-02-999".
Noms lists also support indexing from the back, using `.value[-1]` to mean the last element of a last, `.value[-2]` for the 2nd last, and so on.
@@ -83,7 +85,7 @@ https://localhost:8000/monkey::#o38hugtf3l1e8rqtj89mijj1dq57eh4m
# “bonk” dataset at /foo/bar
/foo/bar::bonk
# from https://demo.noms.io/cli-tour, select the "sf-registered-business" dataset,
# from https://demo.noms.io/cli-tour, select the "sf-registered-business" dataset,
# the root value is a Noms map, select the value of the Noms map identified by string
# key "0000024-02-999", then from that resulting struct select the Ownership_Name field
https://demo.noms.io/cli-tour::sf-registered-business.value["0000024-02-999"].Ownership_Name
go/chunks/dynamo_store.go
+3 -5
View File
@@ -81,10 +81,8 @@ type DynamoStore struct {
}
// NewDynamoStore returns a new DynamoStore instance pointed at a DynamoDB table in the given region. All keys used to access items are prefixed with the given namespace.
// Uses credential from the AWS config parameter.
func NewDynamoStore(table, namespace string, config *aws.Config, showStats bool) *DynamoStore {
sess := session.New(config)
// Uses credentials from the AWS session parameter.
func NewDynamoStore(table, namespace string, sess *session.Session, showStats bool) *DynamoStore {
return newDynamoStoreFromDDBsvc(table, namespace, dynamodb.New(sess), showStats)
}
@@ -537,7 +535,7 @@ func (f DynamoStoreFlags) CreateStore(ns string) ChunkStore {
if *f.awsKey != "" {
config = config.WithCredentials(credentials.NewStaticCredentials(*f.awsKey, *f.awsSecret, ""))
}
return NewDynamoStore(*f.dynamoTable, ns, config, *f.dynamoStats)
return NewDynamoStore(*f.dynamoTable, ns, session.Must(session.NewSession(config)), *f.dynamoStats)
}
return nil
}
go/nbs/benchmarks/main.go
+2 -2
View File
@@ -93,7 +93,7 @@ func main() {
reset = func() { os.RemoveAll(dir); os.MkdirAll(dir, 0777) }
} else if *toAWS != "" {
sess := session.New(aws.NewConfig().WithRegion("us-west-2"))
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
open = func() blockStore {
return nbs.NewAWSStore(dynamoTable, *toAWS, s3Bucket, sess, bufSize)
}
@@ -118,7 +118,7 @@ func main() {
if *useNBS != "" {
open = func() blockStore { return nbs.NewLocalStore(*useNBS, bufSize) }
} else if *useAWS != "" {
sess := session.New(aws.NewConfig().WithRegion("us-west-2"))
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
open = func() blockStore {
return nbs.NewAWSStore(dynamoTable, *useAWS, s3Bucket, sess, bufSize)
}
go/spec/spec.go
+23 -10
View File
@@ -17,6 +17,8 @@ import (
"github.com/attic-labs/noms/go/datas"
"github.com/attic-labs/noms/go/nbs"
"github.com/attic-labs/noms/go/types"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
)
const (
@@ -154,7 +156,7 @@ func (sp Spec) GetDatabase() datas.Database {
// time. If there is no ChunkStore, for example remote databases, returns nil.
func (sp Spec) NewChunkStore() chunks.ChunkStore {
switch sp.Protocol {
case "http", "https":
case "http", "https", "aws":
return nil
case "nbs":
return nbs.NewLocalStore(sp.DatabaseName, 1<<28)
@@ -191,11 +193,12 @@ func (sp Spec) GetValue() (val types.Value) {
// "http://example.com/path". If the Protocol is not "http" or "http", returns
// an empty string.
func (sp Spec) Href() string {
proto := sp.Protocol
if proto == "http" || proto == "https" {
switch proto := sp.Protocol; proto {
case "http", "https", "aws":
return proto + ":" + sp.DatabaseName
default:
return ""
}
return ""
}
// Pin returns a Spec in which the dataset component, if any, has been replaced
@@ -247,6 +250,14 @@ func (sp Spec) createDatabase() datas.Database {
switch sp.Protocol {
case "http", "https":
return datas.NewRemoteDatabase(sp.Href(), sp.Options.Authorization)
case "aws":
u, _ := url.Parse(sp.Href())
parts := strings.SplitN(u.Host, ":", 2) // [table] [, bucket]?
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
if len(parts) == 1 {
return datas.NewDatabase(chunks.NewDynamoStore(parts[0], u.Path, sess, false))
}
return datas.NewDatabase(nbs.NewAWSStore(parts[0], u.Path, parts[1], sess, 1<<28))
case "ldb":
return datas.NewDatabase(getLdbStore(sp.DatabaseName))
case "nbs":
@@ -281,13 +292,15 @@ func parseDatabaseSpec(spec string) (protocol, name string, err error) {
case "ldb", "nbs":
protocol, name = parts[0], parts[1]
case "http", "https":
var u *url.URL
u, err = url.Parse(spec)
if err == nil && u.Host == "" {
case "http", "https", "aws":
u, perr := url.Parse(spec)
if perr != nil {
err = perr
} else if u.Host == "" {
err = fmt.Errorf("%s has empty host", spec)
}
if err == nil {
} else if parts[0] == "aws" && u.Path == "" {
err = fmt.Errorf("%s does not specify a database ID", spec)
} else {
protocol, name = parts[0], parts[1]
}
go/spec/spec_test.go
+19
View File
@@ -167,9 +167,18 @@ func TestHref(t *testing.T) {
assert.Equal("https://my.example.com/foo/bar/baz", sp.Href())
sp, _ = ForDataset("https://my.example.com/foo/bar/baz::myds")
assert.Equal("https://my.example.com/foo/bar/baz", sp.Href())
sp, _ = ForDataset("https://my.example.com:8080/foo/bar/baz::myds")
assert.Equal("https://my.example.com:8080/foo/bar/baz", sp.Href())
sp, _ = ForPath("https://my.example.com/foo/bar/baz::myds.my.path")
assert.Equal("https://my.example.com/foo/bar/baz", sp.Href())
sp, _ = ForDatabase("aws://table/foo/bar/baz")
assert.Equal("aws://table/foo/bar/baz", sp.Href())
sp, _ = ForDataset("aws://table:bucket/foo/bar/baz::myds")
assert.Equal("aws://table:bucket/foo/bar/baz", sp.Href())
sp, _ = ForPath("aws://table:bucket/foo/bar/baz::myds.my.path")
assert.Equal("aws://table:bucket/foo/bar/baz", sp.Href())
sp, err := ForPath("mem::myds.my.path")
assert.NoError(err)
assert.Equal("", sp.Href())
@@ -191,6 +200,9 @@ func TestForDatabase(t *testing.T) {
"random:",
"random:random",
"/file/ba:d",
"aws://t:b",
"aws://t",
"aws://t:",
}
for _, spec := range badSpecs {
@@ -220,6 +232,8 @@ func TestForDatabase(t *testing.T) {
{"http://0:0:0:0:0:ffff:c01e:fc9a", "http", "//0:0:0:0:0:ffff:c01e:fc9a"},
{"http://::ffff:c01e:fc9a", "http", "//::ffff:c01e:fc9a"},
{"http://::ffff::1e::9a", "http", "//::ffff::1e::9a"},
{"aws://table:bucket/db", "aws", "//table:bucket/db"},
{"aws://table/db", "aws", "//table/db"},
}
for _, tc := range testCases {
@@ -252,6 +266,7 @@ func TestForDataset(t *testing.T) {
"http://localhost:8000/one",
"ldb:",
"ldb:hello",
"aws://t:b/db",
}
for _, spec := range badSpecs {
@@ -292,6 +307,8 @@ func TestForDataset(t *testing.T) {
{"http://0:0:0:0:0:ffff:c01e:fc9a::foo", "http", "//0:0:0:0:0:ffff:c01e:fc9a", "foo"},
{"http://::ffff:c01e:fc9a::foo", "http", "//::ffff:c01e:fc9a", "foo"},
{"http://::ffff::1e::9a::foo", "http", "//::ffff::1e::9a", "foo"},
{"aws://table:bucket/db::ds", "aws", "//table:bucket/db", "ds"},
{"aws://table/db::ds", "aws", "//table/db", "ds"},
}
for _, tc := range testCases {
@@ -341,6 +358,8 @@ func TestForPath(t *testing.T) {
{"http://0:0:0:0:0:ffff:c01e:fc9a::foo[42].bar", "http", "//0:0:0:0:0:ffff:c01e:fc9a", "foo[42].bar"},
{"http://::ffff:c01e:fc9a::foo.foo", "http", "//::ffff:c01e:fc9a", "foo.foo"},
{"http://::ffff::1e::9a::hello[\"world\"]", "http", "//::ffff::1e::9a", "hello[\"world\"]"},
{"aws://table:bucket/db::foo.foo", "aws", "//table:bucket/db", "foo.foo"},
{"aws://table/db::foo.foo", "aws", "//table/db", "foo.foo"},
}
for _, tc := range testCases {