Dataspec improvements

chunks/leveldb_store.go

@@ -214,6 +214,10 @@ func NewLevelDBStoreFactory(dir string, maxHandles int, dumpStats bool) Factory
 	return &LevelDBStoreFactory{dir, maxHandles, dumpStats, newBackingStore(dir, maxHandles, dumpStats)}
 }
 
+func NewLevelDBStoreFactoryUseFlags(dir string) Factory {
+	return NewLevelDBStoreFactory(dir, ldbFlags.maxFileHandles, ldbFlags.dumpStats)
+}
+
 type LevelDBStoreFactory struct {
 	dir            string
 	maxFileHandles int

clients/go/flags/dataspec.go

@@ -2,8 +2,8 @@ package flags
 
 import (
 	"fmt"
+	"net/url"
 	"regexp"
-	"strings"
 
 	"github.com/attic-labs/noms/chunks"
 	"github.com/attic-labs/noms/datas"
@@ -13,13 +13,14 @@ import (
 )
 
 var (
-	storeRegex = regexp.MustCompile("^(.+?)(:.+)?$")
+	storeRegex = regexp.MustCompile("^([^:]+):?(.+)?$")
 	pathRegex  = regexp.MustCompile("^(.+):(.+)$")
 )
 
 type DatabaseSpec struct {
-	Protocol string
-	Path     string
+	Protocol    string
+	Path        string
+	accessToken string
 }
 
 type DatasetSpec struct {
@@ -37,19 +38,30 @@ type PathSpec interface {
 }
 
 func ParseDatabaseSpec(spec string) (DatabaseSpec, error) {
-	res := storeRegex.FindStringSubmatch(spec)
-	if len(res) != 3 {
+	parts := storeRegex.FindStringSubmatch(spec)
+	if len(parts) != 3 {
 		return DatabaseSpec{}, fmt.Errorf("Invalid database spec: %s", spec)
 	}
-	protocol := res[1]
+	protocol := parts[1]
+	path := parts[2]
 	switch protocol {
-	case "http", "https", "ldb":
-		if len(res[2]) == 0 {
+	case "http", "https":
+		if len(parts[2]) == 0 {
 			return DatabaseSpec{}, fmt.Errorf("Invalid database spec: %s", spec)
 		}
-		return DatabaseSpec{Protocol: protocol, Path: strings.TrimRight(res[2][1:], "/")}, nil
+		u, err := url.Parse(spec)
+		if err != nil {
+			return DatabaseSpec{}, fmt.Errorf("Invalid path for %s protocol, spec: %s\n", protocol, spec)
+		}
+		token := u.Query().Get("access_token")
+		return DatabaseSpec{Protocol: protocol, Path: path, accessToken: token}, nil
+	case "ldb":
+		if len(parts[2]) == 0 {
+			return DatabaseSpec{}, fmt.Errorf("Invalid database spec: %s", spec)
+		}
+		return DatabaseSpec{Protocol: protocol, Path: path}, nil
 	case "mem":
-		if len(res[2]) > 0 {
+		if len(parts[2]) > 0 {
 			return DatabaseSpec{}, fmt.Errorf("Invalid database spec: %s", spec)
 		}
 		return DatabaseSpec{Protocol: protocol, Path: ""}, nil
@@ -58,15 +70,15 @@ func ParseDatabaseSpec(spec string) (DatabaseSpec, error) {
 }
 
 func ParseDatasetSpec(spec string) (DatasetSpec, error) {
-	res := pathRegex.FindStringSubmatch(spec)
-	if len(res) != 3 {
+	parts := pathRegex.FindStringSubmatch(spec)
+	if len(parts) != 3 {
 		return DatasetSpec{}, fmt.Errorf("Invalid dataset spec: %s", spec)
 	}
-	storeSpec, err := ParseDatabaseSpec(res[1])
+	storeSpec, err := ParseDatabaseSpec(parts[1])
 	if err != nil {
 		return DatasetSpec{}, err
 	}
-	return DatasetSpec{StoreSpec: storeSpec, DatasetName: res[2]}, nil
+	return DatasetSpec{StoreSpec: storeSpec, DatasetName: parts[2]}, nil
 }
 
 func ParseRefSpec(spec string) (RefSpec, error) {
@@ -95,10 +107,14 @@ func ParsePathSpec(spec string) (PathSpec, error) {
 
 }
 
+func (s DatabaseSpec) String() string {
+	return s.Protocol + ":" + s.Path
+}
+
 func (spec DatabaseSpec) Database() (ds datas.Database, err error) {
 	switch spec.Protocol {
-	case "http":
-		ds = datas.NewRemoteDatabase(spec.Protocol+":"+spec.Path, "")
+	case "http", "https":
+		ds = datas.NewRemoteDatabase(spec.String(), "Bearer "+spec.accessToken)
 	case "ldb":
 		ds = datas.NewDatabase(chunks.NewLevelDBStoreUseFlags(spec.Path, ""))
 	case "mem":

clients/go/flags/dataspec_test.go

@@ -235,7 +235,7 @@ func TestReadRef(t *testing.T) {
 func TestDatabaseSpecs(t *testing.T) {
 	assert := assert.New(t)
 
-	badSpecs := []string{"mem:", "mem:stuff", "http:", "https:", "random:", "random:random"}
+	badSpecs := []string{"mem:stuff", "http:", "https:", "random:", "random:random"}
 	for _, spec := range badSpecs {
 		_, err := ParseDatabaseSpec(spec)
 		assert.Error(err)
@@ -247,7 +247,7 @@ func TestDatabaseSpecs(t *testing.T) {
 
 	storeSpec, err = ParseDatabaseSpec("http://localhost:8000/")
 	assert.NoError(err)
-	assert.Equal(DatabaseSpec{Protocol: "http", Path: "//localhost:8000"}, storeSpec)
+	assert.Equal(DatabaseSpec{Protocol: "http", Path: "//localhost:8000/"}, storeSpec)
 
 	storeSpec, err = ParseDatabaseSpec("http://localhost:8000/fff")
 	assert.NoError(err)
@@ -264,6 +264,18 @@ func TestDatabaseSpecs(t *testing.T) {
 	storeSpec, err = ParseDatabaseSpec("mem")
 	assert.NoError(err)
 	assert.Equal(DatabaseSpec{Protocol: "mem"}, storeSpec)
+
+	storeSpec, err = ParseDatabaseSpec("mem:")
+	assert.NoError(err)
+	assert.Equal(DatabaseSpec{Protocol: "mem"}, storeSpec)
+
+	storeSpec, err = ParseDatabaseSpec("http://server.com/john/doe?access_token=jane")
+	assert.NoError(err)
+	assert.Equal(DatabaseSpec{Protocol: "http", Path: "//server.com/john/doe?access_token=jane", accessToken: "jane"}, storeSpec)
+
+	storeSpec, err = ParseDatabaseSpec("http://server.com/john/doe/?arg=2&qp1=true&access_token=jane")
+	assert.NoError(err)
+	assert.Equal(DatabaseSpec{Protocol: "http", Path: "//server.com/john/doe/?arg=2&qp1=true&access_token=jane", accessToken: "jane"}, storeSpec)
 }
 
 func TestDatasetSpecs(t *testing.T) {
@@ -281,7 +293,7 @@ func TestDatasetSpecs(t *testing.T) {
 
 	setSpec, err = ParseDatasetSpec("http://localhost:8000/john/doe/:dsname")
 	assert.NoError(err)
-	assert.Equal(DatasetSpec{StoreSpec: DatabaseSpec{Protocol: "http", Path: "//localhost:8000/john/doe"}, DatasetName: "dsname"}, setSpec)
+	assert.Equal(DatasetSpec{StoreSpec: DatabaseSpec{Protocol: "http", Path: "//localhost:8000/john/doe/"}, DatasetName: "dsname"}, setSpec)
 
 	setSpec, err = ParseDatasetSpec("https://local.attic.io/john/doe:dsname")
 	assert.NoError(err)
@@ -331,7 +343,7 @@ func TestPathSpec(t *testing.T) {
 	pathSpec, err = ParsePathSpec("http://localhost:8000/john/doe/:dsname")
 	assert.NoError(err)
 	setSpec := pathSpec.(*DatasetSpec)
-	assert.Equal(DatasetSpec{StoreSpec: DatabaseSpec{Protocol: "http", Path: "//localhost:8000/john/doe"}, DatasetName: "dsname"}, *setSpec)
+	assert.Equal(DatasetSpec{StoreSpec: DatabaseSpec{Protocol: "http", Path: "//localhost:8000/john/doe/"}, DatasetName: "dsname"}, *setSpec)
 
 	_, err = ParsePathSpec("http://local.attic.io")
 	assert.Error(err)