Merge pull request #1670 from willhite/json

Make json-import map object to nomsStruct rather than nomsMaps.

clients/go/json-import/json_importer.go

@@ -51,7 +51,7 @@ func main() {
 		log.Fatalln("Error decoding JSON: ", err)
 	}
 
-	_, err = ds.Commit(util.NomsValueFromDecodedJSON(jsonObject))
+	_, err = ds.Commit(util.NomsValueFromDecodedJSON(jsonObject, true))
 	d.Exp.NoError(err)
 	ds.Database().Close()
 }

clients/go/test_util/util_test.go

@@ -17,23 +17,23 @@ type LibTestSuite struct {
 }
 
 func (suite *LibTestSuite) TestPrimitiveTypes() {
-	suite.EqualValues(types.NewString("expected"), util.NomsValueFromDecodedJSON("expected"))
-	suite.EqualValues(types.Bool(false), util.NomsValueFromDecodedJSON(false))
-	suite.EqualValues(types.Number(1.7), util.NomsValueFromDecodedJSON(1.7))
-	suite.False(util.NomsValueFromDecodedJSON(1.7).Equals(types.Bool(true)))
+	suite.EqualValues(types.NewString("expected"), util.NomsValueFromDecodedJSON("expected", false))
+	suite.EqualValues(types.Bool(false), util.NomsValueFromDecodedJSON(false, false))
+	suite.EqualValues(types.Number(1.7), util.NomsValueFromDecodedJSON(1.7, false))
+	suite.False(util.NomsValueFromDecodedJSON(1.7, false).Equals(types.Bool(true)))
 }
 
 func (suite *LibTestSuite) TestCompositeTypes() {
 	// [false true]
 	suite.EqualValues(
 		types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
-		util.NomsValueFromDecodedJSON([]interface{}{false, true}))
+		util.NomsValueFromDecodedJSON([]interface{}{false, true}, false))
 
 	// [[false true]]
 	suite.EqualValues(
 		types.NewList().Append(
 			types.NewList().Append(types.Bool(false)).Append(types.Bool(true))),
-		util.NomsValueFromDecodedJSON([]interface{}{[]interface{}{false, true}}))
+		util.NomsValueFromDecodedJSON([]interface{}{[]interface{}{false, true}}, false))
 
 	// {"string": "string",
 	//  "list": [false true],
@@ -52,11 +52,32 @@ func (suite *LibTestSuite) TestCompositeTypes() {
 		"string": "string",
 		"list":   []interface{}{false, true},
 		"map":    map[string]interface{}{"nested": "string"},
-	})
+	}, false)
 
 	suite.True(m.Equals(o))
 }
 
-func (suite *LibTestSuite) TestPanicOnUnsupportedType() {
-	suite.Panics(func() { util.NomsValueFromDecodedJSON(map[int]string{1: "one"}) }, "Should panic on map[int]string!")
+func (suite *LibTestSuite) TestCompositeTypeWithStruct() {
+	// {"string": "string",
+	//  "list": [false true],
+	//  "struct": {"nested": "string"}
+	// }
+	tstruct := types.NewStruct("", map[string]types.Value{
+		"string": types.NewString("string"),
+		"list":   types.NewList().Append(types.Bool(false)).Append(types.Bool(true)),
+		"struct": types.NewStruct("", map[string]types.Value{
+			"nested": types.NewString("string"),
+		}),
+	})
+	o := util.NomsValueFromDecodedJSON(map[string]interface{}{
+		"string": "string",
+		"list":   []interface{}{false, true},
+		"struct": map[string]interface{}{"nested": "string"},
+	}, true)
+
+	suite.True(tstruct.Equals(o))
+}
+
+func (suite *LibTestSuite) TestPanicOnUnsupportedType() {
+	suite.Panics(func() { util.NomsValueFromDecodedJSON(map[int]string{1: "one"}, false) }, "Should panic on map[int]string!")
 }

clients/go/util/util.go

@@ -21,7 +21,7 @@ import (
 // Composites:
 //  - []interface{}
 //  - map[string]interface{}
-func NomsValueFromDecodedJSON(o interface{}) types.Value {
+func NomsValueFromDecodedJSON(o interface{}, useStruct bool) types.Value {
 	switch o := o.(type) {
 	case string:
 		return types.NewString(o)
@@ -34,23 +34,35 @@ func NomsValueFromDecodedJSON(o interface{}) types.Value {
 	case []interface{}:
 		items := make([]types.Value, 0, len(o))
 		for _, v := range o {
-			nv := NomsValueFromDecodedJSON(v)
+			nv := NomsValueFromDecodedJSON(v, useStruct)
 			if nv != nil {
 				items = append(items, nv)
 			}
 		}
 		return types.NewList(items...)
 	case map[string]interface{}:
-		kv := make([]types.Value, 0, len(o)*2)
-
-		for k, v := range o {
-			nv := NomsValueFromDecodedJSON(v)
-			if nv != nil {
-				kv = append(kv, types.NewString(k), nv)
+		var v types.Value
+		if useStruct {
+			fields := make(map[string]types.Value, len(o))
+			for k, v := range o {
+				nv := NomsValueFromDecodedJSON(v, useStruct)
+				if nv != nil {
+					k := types.EscapeStructField(k)
+					fields[k] = nv
+				}
 			}
+			v = types.NewStruct("", fields)
+		} else {
+			kv := make([]types.Value, 0, len(o)*2)
+			for k, v := range o {
+				nv := NomsValueFromDecodedJSON(v, useStruct)
+				if nv != nil {
+					kv = append(kv, types.NewString(k), nv)
+				}
+			}
+			v = types.NewMap(kv...)
 		}
-
-		return types.NewMap(kv...)
+		return v
 
 	default:
 		d.Chk.Fail("Nomsification failed.", "I don't understand %+v, which is of type %s!\n", o, reflect.TypeOf(o).String())

clients/go/xml-import/xml_importer.go

@@ -96,7 +96,7 @@ func main() {
 				object := xmlObject.Old()
 				file.Close()
 
-				nomsObj := util.NomsValueFromDecodedJSON(object)
+				nomsObj := util.NomsValueFromDecodedJSON(object, false)
 				d.Chk.IsType(expectedType, nomsObj)
 
 				var r types.Ref

types/struct.go

@@ -1,6 +1,10 @@
 package types
 
 import (
+	"bytes"
+	"fmt"
+	"regexp"
+
 	"github.com/attic-labs/noms/d"
 	"github.com/attic-labs/noms/hash"
 )
@@ -167,3 +171,41 @@ func StructDiff(s1, s2 Struct) (changed []string) {
 
 	return
 }
+
+var escapeChar = "Q"
+var headPattern = regexp.MustCompile("[a-zA-PR-Z]")
+var tailPattern = regexp.MustCompile("[a-zA-PR-Z1-9_]")
+var completePattern = regexp.MustCompile("^" + headPattern.String() + tailPattern.String() + "*$")
+
+// Escapes names for use as noms structs. Disallowed characters are encoded as
+// 'Q<hex-encoded-utf8-bytes>'. Note that Q itself is also escaped since it is
+// the escape character.
+func EscapeStructField(input string) string {
+	if completePattern.MatchString(input) {
+		return input
+	}
+
+	encode := func(s1 string, p *regexp.Regexp) string {
+		if p.MatchString(s1) && s1 != escapeChar {
+			return s1
+		}
+
+		var hs = fmt.Sprintf("%X", s1)
+		var buf bytes.Buffer
+		buf.WriteString(escapeChar)
+		if len(hs) == 1 {
+			buf.WriteString("0")
+		}
+		buf.WriteString(hs)
+		return buf.String()
+	}
+
+	output := ""
+	pattern := headPattern
+	for _, ch := range input {
+		output += encode(string([]rune{ch}), pattern)
+		pattern = tailPattern
+	}
+
+	return output
+}

types/struct_test.go

@@ -133,3 +133,23 @@ func TestStructDiff(t *testing.T) {
 			"c": NewSet(Number(0), Number(1), NewString("bar")),
 		}))
 }
+
+func TestEscStructField(t *testing.T) {
+	assert := assert.New(t)
+	cases := []string{
+		"a", "a",
+		"AaZz19_", "AaZz19_",
+		"Q", "Q51",
+		"AQ1", "AQ511",
+		"$", "Q24",
+		"_content", "Q5Fcontent",
+		"Few ¢ents Short", "FewQ20QC2A2entsQ20Short",
+		"💩", "QF09F92A9",
+		"https://picasaweb.google.com/data", "httpsQ3AQ2FQ2FpicasawebQ2EgoogleQ2EcomQ2Fdata",
+	}
+
+	for i := 0; i < len(cases); i += 2 {
+		orig, expected := cases[i], cases[i+1]
+		assert.Equal(expected, EscapeStructField(orig))
+	}
+}