diff --git a/doc/spelling.md b/doc/spelling.md index 595df7f0d4..3552ce96ed 100644 --- a/doc/spelling.md +++ b/doc/spelling.md @@ -16,6 +16,8 @@ The `path` part of the name is interpreted differently depending on the protocol - **ldb** specs describe a local [LevelDB](https://github.com/google/leveldb)-backed database. In this case, the path component should be a relative or absolute path on disk to a directory in which to store the LevelDB data. For example: `ldb:/tmp/noms-data`. - In Go, `ldb:` can be ommitted (just `/tmp/noms-data` will work). - **mem** specs describe an ephemeral memory-backed database. In this case, the path component is not used and must be empty. +- **nbs** specs describe a local [Noms Block Store (NBS)](https://github.com/attic-labs/noms/tree/master/go/nbs)-backed database. In this case, the path component should be a relative or absolute path on disk to a directory in which to store the data, e.g. `nbs:/tmp/noms-data`. +- **aws** specs describe a remote Noms Block Store backed directly by Amazon Web Services, specifically DynamoDB and S3. The format is a URI containing the names of the DynamoDB table to use, the S3 bucket to use, and the database to serve. For example: `aws://dynamo-table:s3-bucket/database`. ## Spelling Datasets @@ -58,7 +60,7 @@ For example, if the `root` is a dataset, then one can use `.value` to get the ro ### Specifying Collection Values Elements of a Noms list, map, or set can be retrieved using brackets `[...]`. -For example, if the dataset is a Noms map of number to struct then one could use `.value[42]` to get the Noms struct associated with the key 42. Similarly selecting the first element from a Noms list would be `.value[0]`. If the Noms map was keyed by string, then using `.value["0000024-02-999"]` would reference the Noms struct associated with key "0000024-02-999". +For example, if the dataset is a Noms map of number to struct then one could use `.value[42]` to get the Noms struct associated with the key 42. Similarly selecting the first element from a Noms list would be `.value[0]`. If the Noms map was keyed by string, then using `.value["0000024-02-999"]` would reference the Noms struct associated with key "0000024-02-999". Noms lists also support indexing from the back, using `.value[-1]` to mean the last element of a last, `.value[-2]` for the 2nd last, and so on. @@ -83,7 +85,7 @@ https://localhost:8000/monkey::#o38hugtf3l1e8rqtj89mijj1dq57eh4m # “bonk” dataset at /foo/bar /foo/bar::bonk -# from https://demo.noms.io/cli-tour, select the "sf-registered-business" dataset, +# from https://demo.noms.io/cli-tour, select the "sf-registered-business" dataset, # the root value is a Noms map, select the value of the Noms map identified by string # key "0000024-02-999", then from that resulting struct select the Ownership_Name field https://demo.noms.io/cli-tour::sf-registered-business.value["0000024-02-999"].Ownership_Name diff --git a/go/chunks/dynamo_store.go b/go/chunks/dynamo_store.go index 6f7db9e4b4..c619774627 100644 --- a/go/chunks/dynamo_store.go +++ b/go/chunks/dynamo_store.go @@ -81,10 +81,8 @@ type DynamoStore struct { } // NewDynamoStore returns a new DynamoStore instance pointed at a DynamoDB table in the given region. All keys used to access items are prefixed with the given namespace. -// Uses credential from the AWS config parameter. -func NewDynamoStore(table, namespace string, config *aws.Config, showStats bool) *DynamoStore { - sess := session.New(config) - +// Uses credentials from the AWS session parameter. +func NewDynamoStore(table, namespace string, sess *session.Session, showStats bool) *DynamoStore { return newDynamoStoreFromDDBsvc(table, namespace, dynamodb.New(sess), showStats) } @@ -537,7 +535,7 @@ func (f DynamoStoreFlags) CreateStore(ns string) ChunkStore { if *f.awsKey != "" { config = config.WithCredentials(credentials.NewStaticCredentials(*f.awsKey, *f.awsSecret, "")) } - return NewDynamoStore(*f.dynamoTable, ns, config, *f.dynamoStats) + return NewDynamoStore(*f.dynamoTable, ns, session.Must(session.NewSession(config)), *f.dynamoStats) } return nil } diff --git a/go/nbs/benchmarks/main.go b/go/nbs/benchmarks/main.go index b9a33f543d..68120f3768 100644 --- a/go/nbs/benchmarks/main.go +++ b/go/nbs/benchmarks/main.go @@ -93,7 +93,7 @@ func main() { reset = func() { os.RemoveAll(dir); os.MkdirAll(dir, 0777) } } else if *toAWS != "" { - sess := session.New(aws.NewConfig().WithRegion("us-west-2")) + sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2"))) open = func() blockStore { return nbs.NewAWSStore(dynamoTable, *toAWS, s3Bucket, sess, bufSize) } @@ -118,7 +118,7 @@ func main() { if *useNBS != "" { open = func() blockStore { return nbs.NewLocalStore(*useNBS, bufSize) } } else if *useAWS != "" { - sess := session.New(aws.NewConfig().WithRegion("us-west-2")) + sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2"))) open = func() blockStore { return nbs.NewAWSStore(dynamoTable, *useAWS, s3Bucket, sess, bufSize) } diff --git a/go/spec/spec.go b/go/spec/spec.go index 8b46defd9c..a882779f72 100644 --- a/go/spec/spec.go +++ b/go/spec/spec.go @@ -17,6 +17,8 @@ import ( "github.com/attic-labs/noms/go/datas" "github.com/attic-labs/noms/go/nbs" "github.com/attic-labs/noms/go/types" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" ) const ( @@ -154,7 +156,7 @@ func (sp Spec) GetDatabase() datas.Database { // time. If there is no ChunkStore, for example remote databases, returns nil. func (sp Spec) NewChunkStore() chunks.ChunkStore { switch sp.Protocol { - case "http", "https": + case "http", "https", "aws": return nil case "nbs": return nbs.NewLocalStore(sp.DatabaseName, 1<<28) @@ -191,11 +193,12 @@ func (sp Spec) GetValue() (val types.Value) { // "http://example.com/path". If the Protocol is not "http" or "http", returns // an empty string. func (sp Spec) Href() string { - proto := sp.Protocol - if proto == "http" || proto == "https" { + switch proto := sp.Protocol; proto { + case "http", "https", "aws": return proto + ":" + sp.DatabaseName + default: + return "" } - return "" } // Pin returns a Spec in which the dataset component, if any, has been replaced @@ -247,6 +250,14 @@ func (sp Spec) createDatabase() datas.Database { switch sp.Protocol { case "http", "https": return datas.NewRemoteDatabase(sp.Href(), sp.Options.Authorization) + case "aws": + u, _ := url.Parse(sp.Href()) + parts := strings.SplitN(u.Host, ":", 2) // [table] [, bucket]? + sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2"))) + if len(parts) == 1 { + return datas.NewDatabase(chunks.NewDynamoStore(parts[0], u.Path, sess, false)) + } + return datas.NewDatabase(nbs.NewAWSStore(parts[0], u.Path, parts[1], sess, 1<<28)) case "ldb": return datas.NewDatabase(getLdbStore(sp.DatabaseName)) case "nbs": @@ -281,13 +292,15 @@ func parseDatabaseSpec(spec string) (protocol, name string, err error) { case "ldb", "nbs": protocol, name = parts[0], parts[1] - case "http", "https": - var u *url.URL - u, err = url.Parse(spec) - if err == nil && u.Host == "" { + case "http", "https", "aws": + u, perr := url.Parse(spec) + if perr != nil { + err = perr + } else if u.Host == "" { err = fmt.Errorf("%s has empty host", spec) - } - if err == nil { + } else if parts[0] == "aws" && u.Path == "" { + err = fmt.Errorf("%s does not specify a database ID", spec) + } else { protocol, name = parts[0], parts[1] } diff --git a/go/spec/spec_test.go b/go/spec/spec_test.go index 42d5046db3..c58d058a16 100644 --- a/go/spec/spec_test.go +++ b/go/spec/spec_test.go @@ -167,9 +167,18 @@ func TestHref(t *testing.T) { assert.Equal("https://my.example.com/foo/bar/baz", sp.Href()) sp, _ = ForDataset("https://my.example.com/foo/bar/baz::myds") assert.Equal("https://my.example.com/foo/bar/baz", sp.Href()) + sp, _ = ForDataset("https://my.example.com:8080/foo/bar/baz::myds") + assert.Equal("https://my.example.com:8080/foo/bar/baz", sp.Href()) sp, _ = ForPath("https://my.example.com/foo/bar/baz::myds.my.path") assert.Equal("https://my.example.com/foo/bar/baz", sp.Href()) + sp, _ = ForDatabase("aws://table/foo/bar/baz") + assert.Equal("aws://table/foo/bar/baz", sp.Href()) + sp, _ = ForDataset("aws://table:bucket/foo/bar/baz::myds") + assert.Equal("aws://table:bucket/foo/bar/baz", sp.Href()) + sp, _ = ForPath("aws://table:bucket/foo/bar/baz::myds.my.path") + assert.Equal("aws://table:bucket/foo/bar/baz", sp.Href()) + sp, err := ForPath("mem::myds.my.path") assert.NoError(err) assert.Equal("", sp.Href()) @@ -191,6 +200,9 @@ func TestForDatabase(t *testing.T) { "random:", "random:random", "/file/ba:d", + "aws://t:b", + "aws://t", + "aws://t:", } for _, spec := range badSpecs { @@ -220,6 +232,8 @@ func TestForDatabase(t *testing.T) { {"http://0:0:0:0:0:ffff:c01e:fc9a", "http", "//0:0:0:0:0:ffff:c01e:fc9a"}, {"http://::ffff:c01e:fc9a", "http", "//::ffff:c01e:fc9a"}, {"http://::ffff::1e::9a", "http", "//::ffff::1e::9a"}, + {"aws://table:bucket/db", "aws", "//table:bucket/db"}, + {"aws://table/db", "aws", "//table/db"}, } for _, tc := range testCases { @@ -252,6 +266,7 @@ func TestForDataset(t *testing.T) { "http://localhost:8000/one", "ldb:", "ldb:hello", + "aws://t:b/db", } for _, spec := range badSpecs { @@ -292,6 +307,8 @@ func TestForDataset(t *testing.T) { {"http://0:0:0:0:0:ffff:c01e:fc9a::foo", "http", "//0:0:0:0:0:ffff:c01e:fc9a", "foo"}, {"http://::ffff:c01e:fc9a::foo", "http", "//::ffff:c01e:fc9a", "foo"}, {"http://::ffff::1e::9a::foo", "http", "//::ffff::1e::9a", "foo"}, + {"aws://table:bucket/db::ds", "aws", "//table:bucket/db", "ds"}, + {"aws://table/db::ds", "aws", "//table/db", "ds"}, } for _, tc := range testCases { @@ -341,6 +358,8 @@ func TestForPath(t *testing.T) { {"http://0:0:0:0:0:ffff:c01e:fc9a::foo[42].bar", "http", "//0:0:0:0:0:ffff:c01e:fc9a", "foo[42].bar"}, {"http://::ffff:c01e:fc9a::foo.foo", "http", "//::ffff:c01e:fc9a", "foo.foo"}, {"http://::ffff::1e::9a::hello[\"world\"]", "http", "//::ffff::1e::9a", "hello[\"world\"]"}, + {"aws://table:bucket/db::foo.foo", "aws", "//table:bucket/db", "foo.foo"}, + {"aws://table/db::foo.foo", "aws", "//table/db", "foo.foo"}, } for _, tc := range testCases {