mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-01-08 00:39:48 -06:00
Code Issues Packages Projects Releases 116 Wiki Activity

Add support for parsing values (#3688)

Browse Source 
This allows parsing all Noms values from the string representation
used by human readable encoding:

```
v, err := nomdl.Parse(vrw, `map {"abc": 42}`)
```

Fixes #1466
This commit is contained in:
Erik Arvidsson
2017-09-13 15:02:01 -07:00
committed by GitHub
parent d3be53d164
commit 5ff6432c7b
6 changed files with 620 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
go/datas/commit.go
View File

@@ -22,8 +22,8 @@ const (
var commitTemplate = types.MakeStructTemplate(commitName, []string{MetaField, ParentsField, ValueField})
var valueCommitType = nomdl.MustParseType(`struct Commit {
meta: struct {},
var valueCommitType = nomdl.MustParseType(`Struct Commit {
meta: Struct {},
parents: Set<Ref<Cycle<Commit>>>,
value: Value,
}`)

18
go/datas/commit_test.go
View File

@@ -38,8 +38,8 @@ func TestNewCommit(t *testing.T) {
// Committing another Number
commit2 := NewCommit(types.Number(2), types.NewSet(db, types.NewRef(commit)), types.EmptyStruct)
at2 := types.TypeOf(commit2)
et2 := nomdl.MustParseType(`struct Commit {
meta: struct {},
et2 := nomdl.MustParseType(`Struct Commit {
meta: Struct {},
parents: Set<Ref<Cycle<Commit>>>,
value: Number,
}`)
@@ -48,8 +48,8 @@ func TestNewCommit(t *testing.T) {
// Now commit a String
commit3 := NewCommit(types.String("Hi"), types.NewSet(db, types.NewRef(commit2)), types.EmptyStruct)
at3 := types.TypeOf(commit3)
et3 := nomdl.MustParseType(`struct Commit {
meta: struct {},
et3 := nomdl.MustParseType(`Struct Commit {
meta: Struct {},
parents: Set<Ref<Cycle<Commit>>>,
value: Number | String,
}`)
@@ -57,15 +57,15 @@ func TestNewCommit(t *testing.T) {
// Now commit a String with MetaInfo
meta := types.NewStruct("Meta", types.StructData{"date": types.String("some date"), "number": types.Number(9)})
metaType := nomdl.MustParseType(`struct Meta {
metaType := nomdl.MustParseType(`Struct Meta {
date: String,
number: Number,
}`)
assertTypeEquals(metaType, types.TypeOf(meta))
commit4 := NewCommit(types.String("Hi"), types.NewSet(db, types.NewRef(commit2)), meta)
at4 := types.TypeOf(commit4)
et4 := nomdl.MustParseType(`struct Commit {
meta: struct {} | struct Meta {
et4 := nomdl.MustParseType(`Struct Commit {
meta: Struct {} | Struct Meta {
date: String,
number: Number,
},
@@ -77,8 +77,8 @@ func TestNewCommit(t *testing.T) {
// Merge-commit with different parent types
commit5 := NewCommit(types.String("Hi"), types.NewSet(db, types.NewRef(commit2), types.NewRef(commit3)), types.EmptyStruct)
at5 := types.TypeOf(commit5)
et5 := nomdl.MustParseType(`struct Commit {
meta: struct {},
et5 := nomdl.MustParseType(`Struct Commit {
meta: Struct {},
parents: Set<Ref<Cycle<Commit>>>,
value: Number | String,
}`)

4
go/marshal/encode_type_test.go
View File

@@ -702,8 +702,8 @@ func TestMarshalTypeOutface(t *testing.T) {
defer vs.Close()
typ := MustMarshalType(vs, OutPhoto{})
expectedType := nomdl.MustParseType(`struct OutPhoto {
faces: Set<struct Face {
expectedType := nomdl.MustParseType(`Struct OutPhoto {
faces: Set<Struct Face {
blob: Ref<Value>,
}>,
someOtherFacesSet: Set<Cycle<Face>>,

356
go/nomdl/parser.go
View File

@@ -5,7 +5,10 @@
package nomdl
import (
"bytes"
"fmt"
"io"
"strconv"
"strings"
"text/scanner"
@@ -16,6 +19,7 @@ import (
// Parser provides ways to parse Noms types.
type Parser struct {
lex *lexer
vrw types.ValueReadWriter
}
// ParserOptions allows passing options into New.
@@ -25,18 +29,19 @@ type ParserOptions struct {
}
// New creates a new Parser.
func New(r io.Reader, options ParserOptions) *Parser {
func New(vrw types.ValueReadWriter, r io.Reader, options ParserOptions) *Parser {
s := scanner.Scanner{}
s.Filename = options.Filename
s.Mode = scanner.ScanIdents | scanner.ScanComments | scanner.SkipComments
s.Init(r)
s.Filename = options.Filename
s.Mode = scanner.ScanIdents | scanner.ScanComments | scanner.SkipComments | scanner.ScanFloats | scanner.ScanStrings // | scanner.ScanRawStrings
s.Error = func(s *scanner.Scanner, msg string) {}
lex := lexer{scanner: &s}
return &Parser{&lex}
return &Parser{&lex, vrw}
}
// ParseType parses a string describing a Noms type.
func ParseType(code string) (typ *types.Type, err error) {
p := New(strings.NewReader(code), ParserOptions{})
p := New(nil, strings.NewReader(code), ParserOptions{})
err = catchSyntaxError(func() {
typ = p.parseType()
p.ensureAtEnd()
@@ -52,6 +57,24 @@ func MustParseType(code string) *types.Type {
return typ
}
// Parse parses a string describing a Noms value.
func Parse(vrw types.ValueReadWriter, code string) (v types.Value, err error) {
p := New(vrw, strings.NewReader(code), ParserOptions{})
err = catchSyntaxError(func() {
v = p.parseValue()
p.ensureAtEnd()
})
return
}
// MustParse parses a string describing a Noms value and panics if there
// is an error.
func MustParse(vrw types.ValueReadWriter, code string) types.Value {
v, err := Parse(vrw, code)
d.PanicIfError(err)
return v
}
func (p *Parser) ensureAtEnd() {
p.lex.eat(scanner.EOF)
}
@@ -89,24 +112,29 @@ func (p *Parser) ensureAtEnd() {
// `Set` `<` Type? `>`
//
// StructType :
// `struct` StructName? `{` StructFields? `}`
// `Struct` StructName? `{` StructTypeFields? `}`
//
// StructFields :
// StructField
// StructField `,` StructFields?
// StructTypeFields :
// StructTypeField
// StructTypeField `,` StructTypeFields?
//
// StructName :
// Ident
//
// StructField :
// StructTypeField :
// StructFieldName `?`? `:` Type
//
// StructFieldName :
// Ident
func (p *Parser) parseType() *types.Type {
t := p.parseTypeWithoutUnion()
tok := p.lex.peek()
tok := p.lex.eat(scanner.Ident)
return p.parseTypeWithToken(tok, p.lex.tokenText())
}
func (p *Parser) parseTypeWithToken(tok rune, tokenText string) *types.Type {
t := p.parseSingleTypeWithToken(tok, tokenText)
tok = p.lex.peek()
if tok != '|' {
return t
}
@@ -119,45 +147,47 @@ func (p *Parser) parseType() *types.Type {
} else {
break
}
unionTypes = append(unionTypes, p.parseTypeWithoutUnion())
unionTypes = append(unionTypes, p.parseSingleType())
}
return types.MakeUnionType(unionTypes...)
}
func (p *Parser) parseTypeWithoutUnion() *types.Type {
tok := p.lex.next()
switch tok {
case scanner.Ident:
switch p.lex.tokenText() {
case "Bool":
return types.BoolType
case "Blob":
return types.BlobType
case "Number":
return types.NumberType
case "String":
return types.StringType
case "Type":
return types.TypeType
case "Value":
return types.ValueType
case "struct":
return p.parseStructType()
case "Map":
return p.parseMapType()
case "List":
elemType := p.parseSingleElemType(true)
return types.MakeListType(elemType)
case "Set":
elemType := p.parseSingleElemType(true)
return types.MakeSetType(elemType)
case "Ref":
elemType := p.parseSingleElemType(false)
return types.MakeRefType(elemType)
case "Cycle":
return p.parseCycleType()
}
func (p *Parser) parseSingleType() *types.Type {
tok := p.lex.eat(scanner.Ident)
return p.parseSingleTypeWithToken(tok, p.lex.tokenText())
}
func (p *Parser) parseSingleTypeWithToken(tok rune, tokenText string) *types.Type {
switch tokenText {
case "Bool":
return types.BoolType
case "Blob":
return types.BlobType
case "Number":
return types.NumberType
case "String":
return types.StringType
case "Type":
return types.TypeType
case "Value":
return types.ValueType
case "Struct":
return p.parseStructType()
case "Map":
return p.parseMapType()
case "List":
elemType := p.parseSingleElemType(true)
return types.MakeListType(elemType)
case "Set":
elemType := p.parseSingleElemType(true)
return types.MakeSetType(elemType)
case "Ref":
elemType := p.parseSingleElemType(false)
return types.MakeRefType(elemType)
case "Cycle":
return p.parseCycleType()
}
p.lex.unexpectedToken(tok)
return nil
}
@@ -229,3 +259,237 @@ func (p *Parser) parseMapType() *types.Type {
}
return types.MakeMapType(keyType, valueType)
}
// Value :
// Type
// Bool
// Number
// String
// List
// Set
// Map
// Struct
//
// Bool :
// `true`
// `false`
//
// Number :
// ...
//
// String :
// ...
//
// List :
// `[` Values? `]`
//
// Values :
// Value
// Value `,` Values?
//
// Set :
// `set` `{` Values? `}`
//
// Map :
// `map` `{` MapEntries? `}`
//
// MapEntries :
// MapEntry
// MapEntry `,` MapEntries?
//
// MapEntry :
// Value `:` Value
//
// Struct :
// `struct` StructName? `{` StructFields? `}`
//
// StructFields :
// StructField
// StructField `,` StructFields?
//
// StructField :
// StructFieldName `:` Value
func (p *Parser) parseValue() types.Value {
tok := p.lex.next()
switch tok {
case scanner.Ident:
switch tokenText := p.lex.tokenText(); tokenText {
case "true":
return types.Bool(true)
case "false":
return types.Bool(false)
case "set":
return p.parseSet()
case "map":
return p.parseMap()
case "struct":
return p.parseStruct()
case "blob":
return p.parseBlob()
default:
return p.parseTypeWithToken(tok, tokenText)
}
case scanner.Float, scanner.Int:
f := p.parseFloat()
return types.Number(f)
case '-':
if !p.lex.eatIf(scanner.Float) {
p.lex.eat(scanner.Int)
}
n := p.parseFloat()
return types.Number(-float64(n))
case '+':
if !p.lex.eatIf(scanner.Float) {
p.lex.eat(scanner.Int)
}
return p.parseFloat()
case '[':
return p.parseList()
case scanner.String:
s := p.lex.tokenText()
s2, err := strconv.Unquote(s)
if err != nil {
raiseSyntaxError(fmt.Sprintf("Invalid string %s", s), p.lex.pos())
}
return types.String(s2)
}
p.lex.unexpectedToken(tok)
panic("unreachable")
}
func (p *Parser) parseFloat() types.Number {
s := p.lex.tokenText()
f, _ := strconv.ParseFloat(s, 64)
return types.Number(f)
}
func (p *Parser) parseList() types.List {
// already swallowed '['
le := types.NewList(p.vrw).Edit()
for p.lex.peek() != ']' {
v := p.parseValue()
le.Append(v)
if p.lex.eatIf(',') {
continue
}
break
}
p.lex.eat(']')
return le.List()
}
func (p *Parser) parseSet() types.Set {
// already swallowed 'set'
p.lex.eat('{')
se := types.NewSet(p.vrw).Edit()
for p.lex.peek() != '}' {
v := p.parseValue()
se.Insert(v)
if p.lex.eatIf(',') {
continue
}
break
}
p.lex.eat('}')
return se.Set()
}
func (p *Parser) parseMap() types.Map {
// already swallowed 'map'
p.lex.eat('{')
me := types.NewMap(p.vrw).Edit()
for p.lex.peek() != '}' {
key := p.parseValue()
p.lex.eat(':')
value := p.parseValue()
me.Set(key, value)
if p.lex.eatIf(',') {
continue
}
break
}
p.lex.eat('}')
return me.Map()
}
func (p *Parser) blobString(s string) []byte {
raise := func() {
raiseSyntaxError(fmt.Sprintf("Invalid blob \"%s\"", s), p.lex.pos())
}
if len(s)%2 != 0 {
raise()
}
var buff bytes.Buffer
for i := 0; i < len(s); i += 2 {
n, err := strconv.ParseUint(s[i:i+2], 16, 8)
if err != nil {
raise()
}
buff.WriteByte(uint8(n))
}
return buff.Bytes()
}
func (p *Parser) parseBlob() types.Blob {
// already swallowed 'blob'
p.lex.eat('{')
var buff bytes.Buffer
for p.lex.peek() != '}' {
tok := p.lex.next()
switch tok {
case scanner.Ident, scanner.Int:
s := p.lex.tokenText()
buff.Write(p.blobString(s))
default:
p.lex.unexpectedToken(tok)
}
}
p.lex.eat('}')
return types.NewBlob(p.vrw, bytes.NewReader(buff.Bytes()))
}
func (p *Parser) parseStruct() types.Struct {
// already swallowed 'struct'
tok := p.lex.next()
name := ""
if tok == scanner.Ident {
name = p.lex.tokenText()
p.lex.eat('{')
} else {
p.lex.check('{', tok)
}
data := types.StructData{}
for p.lex.peek() != '}' {
p.lex.eat(scanner.Ident)
fieldName := p.lex.tokenText()
p.lex.eat(':')
v := p.parseValue()
data[fieldName] = v
if p.lex.eatIf(',') {
continue
}
break
}
p.lex.eat('}')
return types.NewStruct(name, data)
}

337
go/nomdl/parser_test.go
View File

@@ -5,13 +5,20 @@
package nomdl
import (
"bytes"
"strings"
"testing"
"github.com/attic-labs/noms/go/chunks"
"github.com/attic-labs/noms/go/types"
"github.com/stretchr/testify/assert"
)
func newTestValueStore() *types.ValueStore {
st := &chunks.TestStorage{}
return types.NewValueStore(st.NewView())
}
func assertParseType(t *testing.T, code string, expected *types.Type) {
t.Run(code, func(t *testing.T) {
actual, err := ParseType(code)
@@ -20,14 +27,24 @@ func assertParseType(t *testing.T, code string, expected *types.Type) {
})
}
func assertParse(t *testing.T, vrw types.ValueReadWriter, code string, expected types.Value) {
t.Run(code, func(t *testing.T) {
actual, err := Parse(vrw, code)
if !assert.NoError(t, err) {
return
}
assert.True(t, expected.Equals(actual), "Expected: %s, Actual: %s", types.EncodedValue(expected), types.EncodedValue(actual))
})
}
func assertParseError(t *testing.T, code, msg string) {
t.Run(code, func(t *testing.T) {
p := New(strings.NewReader(code), ParserOptions{
vrw := newTestValueStore()
p := New(vrw, strings.NewReader(code), ParserOptions{
Filename: "example",
})
err := catchSyntaxError(func() {
typ := p.parseType()
assert.Nil(t, typ)
p.parseValue()
})
if assert.Error(t, err) {
assert.Equal(t, msg, err.Error())
@@ -63,21 +80,21 @@ func TestCompoundTypes(t *testing.T) {
assertParseType(t, "List<Bool>", types.MakeListType(types.BoolType))
assertParseError(t, "List<Bool, Number>", `Unexpected token ",", expected ">", example:1:11`)
assertParseError(t, "List<Bool", `Unexpected token EOF, expected ">", example:1:10`)
assertParseError(t, "List<", `Unexpected token EOF, example:1:6`)
assertParseError(t, "List<", `Unexpected token EOF, expected Ident, example:1:6`)
assertParseError(t, "List", `Unexpected token EOF, expected "<", example:1:5`)
assertParseType(t, "Set<>", types.MakeSetType(types.MakeUnionType()))
assertParseType(t, "Set<Bool>", types.MakeSetType(types.BoolType))
assertParseError(t, "Set<Bool, Number>", `Unexpected token ",", expected ">", example:1:10`)
assertParseError(t, "Set<Bool", `Unexpected token EOF, expected ">", example:1:9`)
assertParseError(t, "Set<", `Unexpected token EOF, example:1:5`)
assertParseError(t, "Set<", `Unexpected token EOF, expected Ident, example:1:5`)
assertParseError(t, "Set", `Unexpected token EOF, expected "<", example:1:4`)
assertParseError(t, "Ref<>", `Unexpected token ">", example:1:6`)
assertParseError(t, "Ref<>", `Unexpected token ">", expected Ident, example:1:6`)
assertParseType(t, "Ref<Bool>", types.MakeRefType(types.BoolType))
assertParseError(t, "Ref<Number, Bool>", `Unexpected token ",", expected ">", example:1:12`)
assertParseError(t, "Ref<Number", `Unexpected token EOF, expected ">", example:1:11`)
assertParseError(t, "Ref<", `Unexpected token EOF, example:1:5`)
assertParseError(t, "Ref<", `Unexpected token EOF, expected Ident, example:1:5`)
assertParseError(t, "Ref", `Unexpected token EOF, expected "<", example:1:4`)
// Cannot use Equals on unresolved cycles.
@@ -92,33 +109,33 @@ func TestCompoundTypes(t *testing.T) {
assertParseType(t, "Map<>", types.MakeMapType(types.MakeUnionType(), types.MakeUnionType()))
assertParseType(t, "Map<Bool, String>", types.MakeMapType(types.BoolType, types.StringType))
assertParseError(t, "Map<Bool,>", `Unexpected token ">", example:1:11`)
assertParseError(t, "Map<,Bool>", `Unexpected token ",", example:1:6`)
assertParseError(t, "Map<,>", `Unexpected token ",", example:1:6`)
assertParseError(t, "Map<Bool,>", `Unexpected token ">", expected Ident, example:1:11`)
assertParseError(t, "Map<,Bool>", `Unexpected token ",", expected Ident, example:1:6`)
assertParseError(t, "Map<,>", `Unexpected token ",", expected Ident, example:1:6`)
assertParseError(t, "Map<Bool,Bool", `Unexpected token EOF, expected ">", example:1:14`)
assertParseError(t, "Map<Bool,", `Unexpected token EOF, example:1:10`)
assertParseError(t, "Map<Bool,", `Unexpected token EOF, expected Ident, example:1:10`)
assertParseError(t, "Map<Bool", `Unexpected token EOF, expected ",", example:1:9`)
assertParseError(t, "Map<", `Unexpected token EOF, example:1:5`)
assertParseError(t, "Map<", `Unexpected token EOF, expected Ident, example:1:5`)
assertParseError(t, "Map", `Unexpected token EOF, expected "<", example:1:4`)
}
func TestStructTypes(t *testing.T) {
assertParseType(t, "struct {}", types.MakeStructTypeFromFields("", types.FieldMap{}))
assertParseType(t, "struct S {}", types.MakeStructTypeFromFields("S", types.FieldMap{}))
assertParseType(t, "Struct {}", types.MakeStructTypeFromFields("", types.FieldMap{}))
assertParseType(t, "Struct S {}", types.MakeStructTypeFromFields("S", types.FieldMap{}))
assertParseType(t, `struct S {
assertParseType(t, `Struct S {
x: Number
}`, types.MakeStructTypeFromFields("S", types.FieldMap{
"x": types.NumberType,
}))
assertParseType(t, `struct S {
assertParseType(t, `Struct S {
x: Number,
}`, types.MakeStructTypeFromFields("S", types.FieldMap{
"x": types.NumberType,
}))
assertParseType(t, `struct S {
assertParseType(t, `Struct S {
x: Number,
y: String
}`, types.MakeStructTypeFromFields("S", types.FieldMap{
@@ -126,7 +143,7 @@ func TestStructTypes(t *testing.T) {
"y": types.StringType,
}))
assertParseType(t, `struct S {
assertParseType(t, `Struct S {
x: Number,
y: String,
}`, types.MakeStructTypeFromFields("S", types.FieldMap{
@@ -134,9 +151,9 @@ func TestStructTypes(t *testing.T) {
"y": types.StringType,
}))
assertParseType(t, `struct S {
assertParseType(t, `Struct S {
x: Number,
y: struct {
y: Struct {
z: String,
},
}`, types.MakeStructTypeFromFields("S", types.FieldMap{
@@ -146,36 +163,36 @@ func TestStructTypes(t *testing.T) {
}),
}))
assertParseType(t, `struct S {
assertParseType(t, `Struct S {
x?: Number,
y: String,
}`, types.MakeStructType("S",
types.StructField{"x", types.NumberType, true},
types.StructField{"y", types.StringType, false},
types.StructField{Name: "x", Type: types.NumberType, Optional: true},
types.StructField{Name: "y", Type: types.StringType, Optional: false},
))
assertParseError(t, `struct S {
assertParseError(t, `Struct S {
x: Number
y: String
}`, `Unexpected token Ident, expected "}", example:3:11`)
assertParseError(t, `struct S {,}`, `Unexpected token ",", expected Ident, example:1:12`)
assertParseError(t, `struct S {`, `Unexpected token EOF, expected Ident, example:1:11`)
assertParseError(t, `struct S { x }`, `Unexpected token "}", expected ":", example:1:15`)
assertParseError(t, `struct S { x`, `Unexpected token EOF, expected ":", example:1:13`)
assertParseError(t, `struct S { x: }`, `Unexpected token "}", example:1:16`)
assertParseError(t, `struct S { x: `, `Unexpected token EOF, example:1:15`)
assertParseError(t, `struct S { x?: `, `Unexpected token EOF, example:1:16`)
assertParseError(t, `struct S { x? `, `Unexpected token EOF, expected ":", example:1:15`)
assertParseError(t, `struct S { x? Bool`, `Unexpected token Ident, expected ":", example:1:19`)
assertParseError(t, `struct S { x: Bool`, `Unexpected token EOF, expected "}", example:1:19`)
assertParseError(t, `struct S { x: Bool,`, `Unexpected token EOF, expected Ident, example:1:20`)
assertParseError(t, `struct S { x: Bool,,`, `Unexpected token ",", expected Ident, example:1:21`)
assertParseError(t, `Struct S {,}`, `Unexpected token ",", expected Ident, example:1:12`)
assertParseError(t, `Struct S {`, `Unexpected token EOF, expected Ident, example:1:11`)
assertParseError(t, `Struct S { x }`, `Unexpected token "}", expected ":", example:1:15`)
assertParseError(t, `Struct S { x`, `Unexpected token EOF, expected ":", example:1:13`)
assertParseError(t, `Struct S { x: }`, `Unexpected token "}", expected Ident, example:1:16`)
assertParseError(t, `Struct S { x: `, `Unexpected token EOF, expected Ident, example:1:15`)
assertParseError(t, `Struct S { x?: `, `Unexpected token EOF, expected Ident, example:1:16`)
assertParseError(t, `Struct S { x? `, `Unexpected token EOF, expected ":", example:1:15`)
assertParseError(t, `Struct S { x? Bool`, `Unexpected token Ident, expected ":", example:1:19`)
assertParseError(t, `Struct S { x: Bool`, `Unexpected token EOF, expected "}", example:1:19`)
assertParseError(t, `Struct S { x: Bool,`, `Unexpected token EOF, expected Ident, example:1:20`)
assertParseError(t, `Struct S { x: Bool,,`, `Unexpected token ",", expected Ident, example:1:21`)
assertParseError(t, `struct S {`, `Unexpected token EOF, expected Ident, example:1:11`)
assertParseError(t, `struct S `, `Unexpected token EOF, expected "{", example:1:10`)
assertParseError(t, `struct {`, `Unexpected token EOF, expected Ident, example:1:9`)
assertParseError(t, `struct`, `Unexpected token EOF, expected "{", example:1:7`)
assertParseError(t, `Struct S {`, `Unexpected token EOF, expected Ident, example:1:11`)
assertParseError(t, `Struct S `, `Unexpected token EOF, expected "{", example:1:10`)
assertParseError(t, `Struct {`, `Unexpected token EOF, expected Ident, example:1:9`)
assertParseError(t, `Struct`, `Unexpected token EOF, expected "{", example:1:7`)
}
func TestUnionTypes(t *testing.T) {
@@ -188,12 +205,12 @@ func TestUnionTypes(t *testing.T) {
types.MakeUnionType(types.BoolType, types.NumberType),
),
)
assertParseType(t, `struct S {
assertParseType(t, `Struct S {
x: Number | Bool
}`, types.MakeStructTypeFromFields("S", types.FieldMap{
"x": types.MakeUnionType(types.BoolType, types.NumberType),
}))
assertParseType(t, `struct S {
assertParseType(t, `Struct S {
x: Number | Bool,
y: String
}`, types.MakeStructTypeFromFields("S", types.FieldMap{
@@ -201,9 +218,239 @@ func TestUnionTypes(t *testing.T) {
"y": types.StringType,
}))
assertParseError(t, "Bool |", "Unexpected token EOF, example:1:7")
assertParseError(t, "Bool | Number |", "Unexpected token EOF, example:1:16")
assertParseError(t, "Bool | | ", `Unexpected token "|", example:1:9`)
assertParseError(t, "Bool |", "Unexpected token EOF, expected Ident, example:1:7")
assertParseError(t, "Bool | Number |", "Unexpected token EOF, expected Ident, example:1:16")
assertParseError(t, "Bool | | ", `Unexpected token "|", expected Ident, example:1:9`)
assertParseError(t, "", `Unexpected token EOF, example:1:1`)
}
func TestValuePrimitives(t *testing.T) {
vs := newTestValueStore()
assertParse(t, vs, "Number", types.NumberType)
assertParse(t, vs, "Number | String", types.MakeUnionType(types.NumberType, types.StringType))
assertParse(t, vs, "true", types.Bool(true))
assertParse(t, vs, "false", types.Bool(false))
assertParse(t, vs, "0", types.Number(0))
assertParse(t, vs, "1", types.Number(1))
assertParse(t, vs, "1.1", types.Number(1.1))
assertParse(t, vs, "1.1e1", types.Number(1.1e1))
assertParse(t, vs, "1e1", types.Number(1e1))
assertParse(t, vs, "1e-1", types.Number(1e-1))
assertParse(t, vs, "1e+1", types.Number(1e+1))
assertParse(t, vs, "+0", types.Number(0))
assertParse(t, vs, "+1", types.Number(1))
assertParse(t, vs, "+1.1", types.Number(1.1))
assertParse(t, vs, "+1.1e1", types.Number(1.1e1))
assertParse(t, vs, "+1e1", types.Number(1e1))
assertParse(t, vs, "+1e-1", types.Number(1e-1))
assertParse(t, vs, "+1e+1", types.Number(1e+1))
assertParse(t, vs, "-0", types.Number(-0))
assertParse(t, vs, "-1", types.Number(-1))
assertParse(t, vs, "-1.1", types.Number(-1.1))
assertParse(t, vs, "-1.1e1", types.Number(-1.1e1))
assertParse(t, vs, "-1e1", types.Number(-1e1))
assertParse(t, vs, "-1e-1", types.Number(-1e-1))
assertParse(t, vs, "-1e+1", types.Number(-1e+1))
assertParse(t, vs, `"a"`, types.String("a"))
assertParse(t, vs, `""`, types.String(""))
assertParse(t, vs, `"\""`, types.String("\""))
assertParseError(t, `"\"`, "Invalid string \"\\\", example:1:4")
assertParseError(t, `"abc`, "Invalid string \"abc, example:1:5")
assertParseError(t, `"`, "Invalid string \", example:1:2")
assertParseError(t, `"
"`, "Invalid string \"\n, example:2:1")
assertParseError(t, "`", "Unexpected token \"`\", example:1:2")
}
func TestValueList(t *testing.T) {
vs := newTestValueStore()
assertParse(t, vs, "[]", types.NewList(vs))
assertParse(t, vs, "[42]", types.NewList(vs, types.Number(42)))
assertParse(t, vs, "[42,]", types.NewList(vs, types.Number(42)))
assertParseError(t, "[", "Unexpected token EOF, example:1:2")
assertParseError(t, "[,", "Unexpected token \",\", example:1:3")
assertParseError(t, "[42", "Unexpected token EOF, expected \"]\", example:1:4")
assertParseError(t, "[42,", "Unexpected token EOF, example:1:5")
assertParseError(t, "[,]", "Unexpected token \",\", example:1:3")
assertParse(t, vs, `[42,
Bool,
]`, types.NewList(vs, types.Number(42), types.BoolType))
assertParse(t, vs, `[42,
Bool
]`, types.NewList(vs, types.Number(42), types.BoolType))
}
func TestValueSet(t *testing.T) {
vs := newTestValueStore()
assertParse(t, vs, "set {}", types.NewSet(vs))
assertParse(t, vs, "set {42}", types.NewSet(vs, types.Number(42)))
assertParse(t, vs, "set {42,}", types.NewSet(vs, types.Number(42)))
assertParseError(t, "set", "Unexpected token EOF, expected \"{\", example:1:4")
assertParseError(t, "set {", "Unexpected token EOF, example:1:6")
assertParseError(t, "set {,", "Unexpected token \",\", example:1:7")
assertParseError(t, "set {42", "Unexpected token EOF, expected \"}\", example:1:8")
assertParseError(t, "set {42,", "Unexpected token EOF, example:1:9")
assertParseError(t, "set {,}", "Unexpected token \",\", example:1:7")
assertParse(t, vs, `set {42,
Bool,
}`, types.NewSet(vs, types.Number(42), types.BoolType))
assertParse(t, vs, `set {42,
Bool
}`, types.NewSet(vs, types.Number(42), types.BoolType))
}
func TestValueMap(t *testing.T) {
vs := newTestValueStore()
assertParse(t, vs, "map {}", types.NewMap(vs))
assertParse(t, vs, "map {42: true}", types.NewMap(vs, types.Number(42), types.Bool(true)))
assertParse(t, vs, "map {42: true,}", types.NewMap(vs, types.Number(42), types.Bool(true)))
assertParseError(t, "map", "Unexpected token EOF, expected \"{\", example:1:4")
assertParseError(t, "map {", "Unexpected token EOF, example:1:6")
assertParseError(t, "map {,", "Unexpected token \",\", example:1:7")
assertParseError(t, "map {42", "Unexpected token EOF, expected \":\", example:1:8")
assertParseError(t, "map {42,", "Unexpected token \",\", expected \":\", example:1:9")
assertParseError(t, "map {42:", "Unexpected token EOF, example:1:9")
assertParseError(t, "map {42: true", "Unexpected token EOF, expected \"}\", example:1:14")
assertParseError(t, "map {,}", "Unexpected token \",\", example:1:7")
assertParse(t, vs, `map {42:
Bool,
}`, types.NewMap(vs, types.Number(42), types.BoolType))
assertParse(t, vs, `map {42:
Bool
}`, types.NewMap(vs, types.Number(42), types.BoolType))
}
func TestValueType(t *testing.T) {
vs := newTestValueStore()
assertParse(t, vs, "Bool", types.BoolType)
assertParse(t, vs, "Number", types.NumberType)
assertParse(t, vs, "String", types.StringType)
}
func TestValueStruct(t *testing.T) {
vs := newTestValueStore()
assertParse(t, vs, "struct {}", types.NewStruct("", nil))
assertParseError(t, "struct", "Unexpected token EOF, expected \"{\", example:1:7")
assertParseError(t, "struct {", "Unexpected token EOF, expected Ident, example:1:9")
assertParse(t, vs, "struct name {}", types.NewStruct("name", nil))
assertParseError(t, "struct name", "Unexpected token EOF, expected \"{\", example:1:12")
assertParseError(t, "struct name {", "Unexpected token EOF, expected Ident, example:1:14")
assertParse(t, vs, "struct name {a: 42}", types.NewStruct("name", types.StructData{"a": types.Number(42)}))
assertParse(t, vs, "struct name {a: 42,}", types.NewStruct("name", types.StructData{"a": types.Number(42)}))
assertParseError(t, "struct name {a", "Unexpected token EOF, expected \":\", example:1:15")
assertParseError(t, "struct name {a: ", "Unexpected token EOF, example:1:17")
assertParseError(t, "struct name {a,", "Unexpected token \",\", expected \":\", example:1:16")
assertParseError(t, "struct name {a}", "Unexpected token \"}\", expected \":\", example:1:16")
assertParseError(t, "struct name {a: 42", "Unexpected token EOF, expected \"}\", example:1:19")
assertParseError(t, "struct name {a: 42,", "Unexpected token EOF, expected Ident, example:1:20")
assertParseError(t, "struct name {a:}", "Unexpected token \"}\", example:1:17")
assertParse(t, vs, "struct name {b: 42, a: true}", types.NewStruct("name", types.StructData{"b": types.Number(42), "a": types.Bool(true)}))
assertParse(t, vs, `struct name {
b: 42,
a: true,
}`, types.NewStruct("name", types.StructData{"b": types.Number(42), "a": types.Bool(true)}))
assertParse(t, vs, "struct name {a: Struct {}}", types.NewStruct("name", types.StructData{"a": types.MakeStructType("")}))
}
func TestValueBlob(t *testing.T) {
vs := newTestValueStore()
test := func(code string, bs ...byte) {
assertParse(t, vs, code, types.NewBlob(vs, bytes.NewBuffer(bs)))
}
test("blob {}")
test("blob {// comment\n}")
test("blob {10}", 0x10)
test("blob {10/* comment */}", 0x10)
test("blob {0000ff}", 0, 0, 0xff)
test("blob {00 00 ff}", 0, 0, 0xff)
test("blob { 00\n00\nff }", 0, 0, 0xff)
test("blob { ffffffff ffffffff ffffffff ffffffff}",
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
)
test("blob { ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff}",
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
)
assertParseError(t, "blob", "Unexpected token EOF, expected \"{\", example:1:5")
assertParseError(t, "blob {", "Unexpected token EOF, example:1:7")
assertParseError(t, "blob { 00", "Unexpected token EOF, example:1:10")
assertParseError(t, "blob {hh}", "Invalid blob \"hh\", example:1:9")
assertParseError(t, "blob {0}", "Invalid blob \"0\", example:1:8")
assertParseError(t, "blob {00 0}", "Invalid blob \"0\", example:1:11")
assertParseError(t, "blob {ff 0 0}", "Invalid blob \"0\", example:1:11")
}
func TestRoundTrips(t *testing.T) {
vs := newTestValueStore()
test := func(v types.Value) {
code := types.EncodedValue(v)
assertParse(t, vs, code, v)
}
test(types.Number(0))
test(types.Number(42))
test(types.Number(-0))
test(types.Number(-42))
test(types.Number(0.05))
test(types.Number(-0.05))
test(types.Number(1e50))
test(types.Number(-1e50))
test(types.Bool(true))
test(types.Bool(false))
test(types.String(""))
test(types.String("a"))
test(types.String("\""))
test(types.String("'"))
test(types.String("`"))
test(types.NewEmptyBlob(vs))
test(types.NewBlob(vs, bytes.NewBufferString("abc")))
test(types.NewList(vs))
test(types.NewList(vs, types.Number(42), types.Bool(true), types.String("abc")))
test(types.NewSet(vs))
test(types.NewSet(vs, types.Number(42), types.Bool(true), types.String("abc")))
test(types.NewMap(vs))
test(types.NewMap(vs, types.Number(42), types.Bool(true), types.String("abc"), types.NewMap(vs)))
test(types.NewStruct("", nil))
test(types.NewStruct("Number", nil))
test(types.NewStruct("Number", types.StructData{
"Number": types.NumberType,
}))
test(types.MakeStructType("S", types.StructField{
Name: "cycle", Type: types.MakeCycleType("S"), Optional: true,
}))
}

10
samples/go/nomsfs/nomsfs.go
View File

@@ -103,8 +103,8 @@ type mount func(fs pathfs.FileSystem)
var fsType, inodeType, attrType, directoryType, fileType, symlinkType *types.Type
func init() {
inodeType = nomdl.MustParseType(`struct Inode {
attr: struct Attr {
inodeType = nomdl.MustParseType(`Struct Inode {
attr: Struct Attr {
ctime: Number,
gid: Number,
mode: Number,
@@ -112,11 +112,11 @@ func init() {
uid: Number,
xattr: Map<String, Blob>,
},
contents: struct Symlink {
contents: Struct Symlink {
targetPath: String,
} | struct File {
} | Struct File {
data: Ref<Blob>,
} | struct Directory {
} | Struct Directory {
entries: Map<String, Cycle<Inode>>,
},
}`)