mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-06 11:20:30 -05:00
Code Issues Packages Projects Releases 160 Wiki Activity

Share s3, dynamodb clients (#3212)

Browse Source 
These objects manage their own pools of HTTP connections and
other resources, so it's generally best to share them
process-wide if you can.

Fixes #3027
This commit is contained in:
cmasone-attic
2017-02-22 13:41:23 -08:00
committed by GitHub
parent 5b63cf6873
commit b1e918d1d4
6 changed files with 26 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/chunks/dynamo_store.go
+4 -7
View File
@@ -81,12 +81,8 @@ type DynamoStore struct {
}
// NewDynamoStore returns a new DynamoStore instance pointed at a DynamoDB table in the given region. All keys used to access items are prefixed with the given namespace.
// Uses credentials from the AWS session parameter.
func NewDynamoStore(table, namespace string, sess *session.Session, showStats bool) *DynamoStore {
return newDynamoStoreFromDDBsvc(table, namespace, dynamodb.New(sess), showStats)
}
func newDynamoStoreFromDDBsvc(table, namespace string, ddb ddbsvc, showStats bool) *DynamoStore {
// Uses the given ddbsvc object to access DynamoDB.
func NewDynamoStore(table, namespace string, ddb ddbsvc, showStats bool) *DynamoStore {
store := &DynamoStore{
table: table,
namespace: []byte(namespace),
@@ -538,7 +534,8 @@ func (f DynamoStoreFlags) CreateStore(ns string) ChunkStore {
if *f.awsKey != "" {
config = config.WithCredentials(credentials.NewStaticCredentials(*f.awsKey, *f.awsSecret, ""))
}
return NewDynamoStore(*f.dynamoTable, ns, session.Must(session.NewSession(config)), *f.dynamoStats)
sess := session.Must(session.NewSession(config))
return NewDynamoStore(*f.dynamoTable, ns, dynamodb.New(sess), *f.dynamoStats)
}
return nil
}
go/chunks/dynamo_store_test.go
+2 -2
View File
@@ -22,7 +22,7 @@ type DynamoStoreTestSuite struct {
func (suite *DynamoStoreTestSuite) SetupTest() {
suite.ddb = createFakeDDB(suite.Assert())
suite.Store = newDynamoStoreFromDDBsvc("table", "namespace", suite.ddb, false)
suite.Store = NewDynamoStore("table", "namespace", suite.ddb, false)
suite.putCountFn = func() int {
return suite.ddb.numPuts
}
@@ -34,7 +34,7 @@ func (suite *DynamoStoreTestSuite) TearDownTest() {
func TestGetRetrying(t *testing.T) {
assert := assert.New(t)
store := newDynamoStoreFromDDBsvc("table", "namespace", createLowCapFakeDDB(assert), false)
store := NewDynamoStore("table", "namespace", createLowCapFakeDDB(assert), false)
c1 := NewChunk([]byte("abc"))
go/nbs/benchmarks/main.go
+3 -2
View File
@@ -20,6 +20,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/dustin/go-humanize"
flag "github.com/juju/gnuflag"
)
@@ -96,7 +97,7 @@ func main() {
} else if *toAWS != "" {
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
open = func() types.BatchStore {
return nbs.NewAWSStore(dynamoTable, *toAWS, s3Bucket, sess, bufSize)
return nbs.NewAWSStore(dynamoTable, *toAWS, s3Bucket, s3.New(sess), dynamodb.New(sess), bufSize)
}
reset = func() {
ddb := dynamodb.New(sess)
@@ -121,7 +122,7 @@ func main() {
} else if *useAWS != "" {
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
open = func() types.BatchStore {
return nbs.NewAWSStore(dynamoTable, *useAWS, s3Bucket, sess, bufSize)
return nbs.NewAWSStore(dynamoTable, *useAWS, s3Bucket, s3.New(sess), dynamodb.New(sess), bufSize)
}
}
writeDB = func() {}
go/nbs/frag/main.go
+3 -1
View File
@@ -18,6 +18,8 @@ import (
"github.com/attic-labs/noms/go/util/profile"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/dustin/go-humanize"
flag "github.com/juju/gnuflag"
)
@@ -51,7 +53,7 @@ func main() {
*dbName = *dir
} else if *table != "" && *bucket != "" && *dbName != "" {
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
store = nbs.NewAWSStore(*table, *dbName, *bucket, sess, memTableSize)
store = nbs.NewAWSStore(*table, *dbName, *bucket, s3.New(sess), dynamodb.New(sess), memTableSize)
} else {
log.Fatalf("Must set either --dir or ALL of --table, --bucket and --db\n")
}
go/nbs/store.go
+9 -8
View File
@@ -64,7 +64,8 @@ type NomsBlockStore struct {
}
type AWSStoreFactory struct {
sess *session.Session
s3 s3svc
ddb ddbsvc
table, bucket string
indexCache *indexCache
readRl chan struct{}
@@ -75,11 +76,11 @@ func NewAWSStoreFactory(sess *session.Session, table, bucket string, indexCacheS
if indexCacheSize > 0 {
indexCache = newIndexCache(indexCacheSize)
}
return &AWSStoreFactory{sess, table, bucket, indexCache, make(chan struct{}, defaultAWSReadLimit)}
return &AWSStoreFactory{s3.New(sess), dynamodb.New(sess), table, bucket, indexCache, make(chan struct{}, defaultAWSReadLimit)}
}
func (asf *AWSStoreFactory) CreateStore(ns string) chunks.ChunkStore {
return newAWSStore(asf.table, ns, asf.bucket, asf.sess, defaultMemTableSize, asf.indexCache, asf.readRl)
return newAWSStore(asf.table, ns, asf.bucket, asf.s3, asf.ddb, defaultMemTableSize, asf.indexCache, asf.readRl)
}
func (asf *AWSStoreFactory) Shutter() {
@@ -109,14 +110,14 @@ func (lsf *LocalStoreFactory) CreateStore(ns string) chunks.ChunkStore {
func (lsf *LocalStoreFactory) Shutter() {
}
func NewAWSStore(table, ns, bucket string, sess *session.Session, memTableSize uint64) *NomsBlockStore {
func NewAWSStore(table, ns, bucket string, s3 s3svc, ddb ddbsvc, memTableSize uint64) *NomsBlockStore {
indexCacheOnce.Do(makeGlobalIndexCache)
return newAWSStore(table, ns, bucket, sess, memTableSize, globalIndexCache, make(chan struct{}, 32))
return newAWSStore(table, ns, bucket, s3, ddb, memTableSize, globalIndexCache, make(chan struct{}, 32))
}
func newAWSStore(table, ns, bucket string, sess *session.Session, memTableSize uint64, indexCache *indexCache, readRl chan struct{}) *NomsBlockStore {
mm := newDynamoManifest(table, ns, dynamodb.New(sess))
ts := newS3TableSet(s3.New(sess), bucket, indexCache, readRl)
func newAWSStore(table, ns, bucket string, s3 s3svc, ddb ddbsvc, memTableSize uint64, indexCache *indexCache, readRl chan struct{}) *NomsBlockStore {
mm := newDynamoManifest(table, ns, ddb)
ts := newS3TableSet(s3, bucket, indexCache, readRl)
return newNomsBlockStore(mm, ts, memTableSize, defaultMaxTables)
}
go/spec/spec.go
+5 -2
View File
@@ -19,6 +19,8 @@ import (
"github.com/attic-labs/noms/go/types"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/s3"
)
const Separator = "::"
@@ -179,10 +181,11 @@ func parseAWSSpec(awsURL string) chunks.ChunkStore {
u, _ := url.Parse(awsURL)
parts := strings.SplitN(u.Host, ":", 2) // [table] [, bucket]?
sess := session.Must(session.NewSession(aws.NewConfig().WithRegion("us-west-2")))
ddb := dynamodb.New(sess)
if len(parts) == 1 {
return chunks.NewDynamoStore(parts[0], u.Path, sess, false)
return chunks.NewDynamoStore(parts[0], u.Path, ddb, false)
}
return nbs.NewAWSStore(parts[0], u.Path, parts[1], sess, 1<<28)
return nbs.NewAWSStore(parts[0], u.Path, parts[1], s3.New(sess), ddb, 1<<28)
}
// GetDataset returns the current Dataset instance for this Spec's Database.