mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-01-31 03:18:43 -06:00
Code Issues Packages Projects Releases 124 Wiki Activity

Merge pull request #629 from rafael-atticlabs/noMarshall

Browse Source 
Remove Marshall
This commit is contained in:
Rafael Weinstein
2015-11-16 13:58:59 -08:00
parent 2dc0d4d29a 217501ab55
commit 1cbdf794e4
9 changed files with 0 additions and 2933 deletions
Download Patch File Download Diff File Expand all files Collapse all files

300
marshal/field.go
View File

@@ -1,300 +0,0 @@
// Extracted and modified from golang's encoding/json/encode.go at 80e6d638bf309181eadcb3fecbe99d2d8518e364.
package marshal
import (
"reflect"
"sort"
"strings"
"sync"
"unicode"
)
func isValidTag(s string) bool {
if s == "" {
return false
}
for _, c := range s {
switch {
case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c):
// Backslash and quote chars are reserved, but
// otherwise any punctuation chars are allowed
// in a tag name.
default:
if !unicode.IsLetter(c) && !unicode.IsDigit(c) {
return false
}
}
}
return true
}
func fieldByIndex(v reflect.Value, index []int) reflect.Value {
for _, i := range index {
if v.Kind() == reflect.Ptr {
if v.IsNil() {
return reflect.Value{}
}
v = v.Elem()
}
v = v.Field(i)
}
return v
}
func typeByIndex(t reflect.Type, index []int) reflect.Type {
for _, i := range index {
if t.Kind() == reflect.Ptr {
t = t.Elem()
}
t = t.Field(i).Type
}
return t
}
// A field represents a single field found in a struct.
type field struct {
name string
nameBytes []byte // []byte(name)
equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent
tag bool
index []int
typ reflect.Type
omitEmpty bool
}
func fillField(f field) field {
f.nameBytes = []byte(f.name)
f.equalFold = foldFunc(f.nameBytes)
return f
}
// byName sorts field by name, breaking ties with depth,
// then breaking ties with "name came from json tag", then
// breaking ties with index sequence.
type byName []field
func (x byName) Len() int { return len(x) }
func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func (x byName) Less(i, j int) bool {
if x[i].name != x[j].name {
return x[i].name < x[j].name
}
if len(x[i].index) != len(x[j].index) {
return len(x[i].index) < len(x[j].index)
}
if x[i].tag != x[j].tag {
return x[i].tag
}
return byIndex(x).Less(i, j)
}
// byIndex sorts field by index sequence.
type byIndex []field
func (x byIndex) Len() int { return len(x) }
func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func (x byIndex) Less(i, j int) bool {
for k, xik := range x[i].index {
if k >= len(x[j].index) {
return false
}
if xik != x[j].index[k] {
return xik < x[j].index[k]
}
}
return len(x[i].index) < len(x[j].index)
}
// typeFields returns a list of fields that Noms should recognize for the given type.
// The algorithm is breadth-first search over the set of structs to include - the top struct
// and then any reachable anonymous structs.
func typeFields(t reflect.Type) []field {
// Anonymous fields to explore at the current level and the next.
current := []field{}
next := []field{{typ: t}}
// Count of queued names for current level and the next.
count := map[reflect.Type]int{}
nextCount := map[reflect.Type]int{}
// Types already visited at an earlier level.
visited := map[reflect.Type]bool{}
// Fields found.
var fields []field
for len(next) > 0 {
current, next = next, current[:0]
count, nextCount = nextCount, map[reflect.Type]int{}
for _, f := range current {
if visited[f.typ] {
continue
}
visited[f.typ] = true
// Scan f.typ for fields to include.
for i := 0; i < f.typ.NumField(); i++ {
sf := f.typ.Field(i)
if sf.PkgPath != "" { // unexported
continue
}
tag := sf.Tag.Get("noms")
if tag == "-" {
continue
}
name, opts := parseTag(tag)
if !isValidTag(name) {
name = ""
}
index := make([]int, len(f.index)+1)
copy(index, f.index)
index[len(f.index)] = i
ft := sf.Type
if ft.Name() == "" && ft.Kind() == reflect.Ptr {
// Follow pointer.
ft = ft.Elem()
}
// Record found field and index sequence.
if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct {
tagged := name != ""
if name == "" {
name = sf.Name
}
fields = append(fields, fillField(field{
name: name,
tag: tagged,
index: index,
typ: ft,
omitEmpty: opts.Contains("omitempty"),
}))
if count[f.typ] > 1 {
// If there were multiple instances, add a second,
// so that the annihilation code will see a duplicate.
// It only cares about the distinction between 1 or 2,
// so don't bother generating any more copies.
fields = append(fields, fields[len(fields)-1])
}
continue
}
// Record new anonymous struct to explore in next round.
nextCount[ft]++
if nextCount[ft] == 1 {
next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft}))
}
}
}
}
sort.Sort(byName(fields))
// Delete all fields that are hidden by the Go rules for embedded fields,
// except that fields with Noms tags are promoted.
// The fields are sorted in primary order of name, secondary order
// of field index length. Loop over names; for each name, delete
// hidden fields by choosing the one dominant field that survives.
out := fields[:0]
for advance, i := 0, 0; i < len(fields); i += advance {
// One iteration per name.
// Find the sequence of fields with the name of this first field.
fi := fields[i]
name := fi.name
for advance = 1; i+advance < len(fields); advance++ {
fj := fields[i+advance]
if fj.name != name {
break
}
}
if advance == 1 { // Only one field with this name
out = append(out, fi)
continue
}
dominant, ok := dominantField(fields[i : i+advance])
if ok {
out = append(out, dominant)
}
}
fields = out
sort.Sort(byIndex(fields))
return fields
}
// dominantField looks through the fields, all of which are known to
// have the same name, to find the single field that dominates the
// others using Go's embedding rules, modified by the presence of
// Noms tags. If there are multiple top-level fields, the boolean
// will be false: This condition is an error in Go and we skip all
// the fields.
func dominantField(fields []field) (field, bool) {
// The fields are sorted in increasing index-length order. The winner
// must therefore be one with the shortest index length. Drop all
// longer entries, which is easy: just truncate the slice.
length := len(fields[0].index)
tagged := -1 // Index of first tagged field.
for i, f := range fields {
if len(f.index) > length {
fields = fields[:i]
break
}
if f.tag {
if tagged >= 0 {
// Multiple tagged fields at the same level: conflict.
// Return no field.
return field{}, false
}
tagged = i
}
}
if tagged >= 0 {
return fields[tagged], true
}
// All remaining fields have the same length. If there's more than one,
// we have a conflict (two fields named "X" at the same level) and we
// return no field.
if len(fields) > 1 {
return field{}, false
}
return fields[0], true
}
var fieldCache struct {
sync.RWMutex
m map[reflect.Type][]field
}
// cachedTypeFields is like typeFields but uses a cache to avoid repeated work.
func cachedTypeFields(t reflect.Type) []field {
fieldCache.RLock()
f := fieldCache.m[t]
fieldCache.RUnlock()
if f != nil {
return f
}
// Compute fields without lock.
// Might duplicate effort but won't hold other computations back.
f = typeFields(t)
if f == nil {
f = []field{}
}
fieldCache.Lock()
if fieldCache.m == nil {
fieldCache.m = map[reflect.Type][]field{}
}
fieldCache.m[t] = f
fieldCache.Unlock()
return f
}

145
marshal/fold.go
View File

@@ -1,145 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Copied from the encoding/json package at 80e6d638bf309181eadcb3fecbe99d2d8518e364.
package marshal
import (
"bytes"
"unicode/utf8"
)
const (
caseMask = ^byte(0x20) // Mask to ignore case in ASCII.
kelvin = '\u212a'
smallLongEss = '\u017f'
)
// foldFunc returns one of four different case folding equivalence
// functions, from most general (and slow) to fastest:
//
// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8
// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S')
// 3) asciiEqualFold, no special, but includes non-letters (including _)
// 4) simpleLetterEqualFold, no specials, no non-letters.
//
// The letters S and K are special because they map to 3 runes, not just 2:
// * S maps to s and to U+017F 'ſ' Latin small letter long s
// * k maps to K and to U+212A 'K' Kelvin sign
// See https://play.golang.org/p/tTxjOc0OGo
//
// The returned function is specialized for matching against s and
// should only be given s. It's not curried for performance reasons.
func foldFunc(s []byte) func(s, t []byte) bool {
nonLetter := false
special := false // special letter
for _, b := range s {
if b >= utf8.RuneSelf {
return bytes.EqualFold
}
upper := b & caseMask
if upper < 'A' || upper > 'Z' {
nonLetter = true
} else if upper == 'K' || upper == 'S' {
// See above for why these letters are special.
special = true
}
}
if special {
return equalFoldRight
}
if nonLetter {
return asciiEqualFold
}
return simpleLetterEqualFold
}
// equalFoldRight is a specialization of bytes.EqualFold when s is
// known to be all ASCII (including punctuation), but contains an 's',
// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t.
// See comments on foldFunc.
func equalFoldRight(s, t []byte) bool {
for _, sb := range s {
if len(t) == 0 {
return false
}
tb := t[0]
if tb < utf8.RuneSelf {
if sb != tb {
sbUpper := sb & caseMask
if 'A' <= sbUpper && sbUpper <= 'Z' {
if sbUpper != tb&caseMask {
return false
}
} else {
return false
}
}
t = t[1:]
continue
}
// sb is ASCII and t is not. t must be either kelvin
// sign or long s; sb must be s, S, k, or K.
tr, size := utf8.DecodeRune(t)
switch sb {
case 's', 'S':
if tr != smallLongEss {
return false
}
case 'k', 'K':
if tr != kelvin {
return false
}
default:
return false
}
t = t[size:]
}
if len(t) > 0 {
return false
}
return true
}
// asciiEqualFold is a specialization of bytes.EqualFold for use when
// s is all ASCII (but may contain non-letters) and contains no
// special-folding letters.
// See comments on foldFunc.
func asciiEqualFold(s, t []byte) bool {
if len(s) != len(t) {
return false
}
for i, sb := range s {
tb := t[i]
if sb == tb {
continue
}
if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') {
if sb&caseMask != tb&caseMask {
return false
}
} else {
return false
}
}
return true
}
// simpleLetterEqualFold is a specialization of bytes.EqualFold for
// use when s is all ASCII letters (no underscores, etc) and also
// doesn't contain 'k', 'K', 's', or 'S'.
// See comments on foldFunc.
func simpleLetterEqualFold(s, t []byte) bool {
if len(s) != len(t) {
return false
}
for i, b := range s {
if b&caseMask != t[i]&caseMask {
return false
}
}
return true
}

118
marshal/fold_test.go
View File

@@ -1,118 +0,0 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Copied from the encoding/json package at 80e6d638bf309181eadcb3fecbe99d2d8518e364.
package marshal
import (
"bytes"
"strings"
"testing"
"unicode/utf8"
)
var foldTests = []struct {
fn func(s, t []byte) bool
s, t string
want bool
}{
{equalFoldRight, "", "", true},
{equalFoldRight, "a", "a", true},
{equalFoldRight, "", "a", false},
{equalFoldRight, "a", "", false},
{equalFoldRight, "a", "A", true},
{equalFoldRight, "AB", "ab", true},
{equalFoldRight, "AB", "ac", false},
{equalFoldRight, "sbkKc", "ſbKKc", true},
{equalFoldRight, "SbKkc", "ſbKKc", true},
{equalFoldRight, "SbKkc", "ſbKK", false},
{equalFoldRight, "e", "é", false},
{equalFoldRight, "s", "S", true},
{simpleLetterEqualFold, "", "", true},
{simpleLetterEqualFold, "abc", "abc", true},
{simpleLetterEqualFold, "abc", "ABC", true},
{simpleLetterEqualFold, "abc", "ABCD", false},
{simpleLetterEqualFold, "abc", "xxx", false},
{asciiEqualFold, "a_B", "A_b", true},
{asciiEqualFold, "aa@", "aa`", false}, // verify 0x40 and 0x60 aren't case-equivalent
}
func TestFold(t *testing.T) {
for i, tt := range foldTests {
if got := tt.fn([]byte(tt.s), []byte(tt.t)); got != tt.want {
t.Errorf("%d. %q, %q = %v; want %v", i, tt.s, tt.t, got, tt.want)
}
truth := strings.EqualFold(tt.s, tt.t)
if truth != tt.want {
t.Errorf("strings.EqualFold doesn't agree with case %d", i)
}
}
}
func TestFoldAgainstUnicode(t *testing.T) {
const bufSize = 5
buf1 := make([]byte, 0, bufSize)
buf2 := make([]byte, 0, bufSize)
var runes []rune
for i := 0x20; i <= 0x7f; i++ {
runes = append(runes, rune(i))
}
runes = append(runes, kelvin, smallLongEss)
funcs := []struct {
name string
fold func(s, t []byte) bool
letter bool // must be ASCII letter
simple bool // must be simple ASCII letter (not 'S' or 'K')
}{
{
name: "equalFoldRight",
fold: equalFoldRight,
},
{
name: "asciiEqualFold",
fold: asciiEqualFold,
simple: true,
},
{
name: "simpleLetterEqualFold",
fold: simpleLetterEqualFold,
simple: true,
letter: true,
},
}
for _, ff := range funcs {
for _, r := range runes {
if r >= utf8.RuneSelf {
continue
}
if ff.letter && !isASCIILetter(byte(r)) {
continue
}
if ff.simple && (r == 's' || r == 'S' || r == 'k' || r == 'K') {
continue
}
for _, r2 := range runes {
buf1 := append(buf1[:0], 'x')
buf2 := append(buf2[:0], 'x')
buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)]
buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)]
buf1 = append(buf1, 'x')
buf2 = append(buf2, 'x')
want := bytes.EqualFold(buf1, buf2)
if got := ff.fold(buf1, buf2); got != want {
t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want)
}
}
}
}
}
func isASCIILetter(b byte) bool {
return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z')
}

443
marshal/marshal.go
View File

@@ -1,443 +0,0 @@
// Modified from golang's encoding/json/encode.go at 80e6d638bf309181eadcb3fecbe99d2d8518e364.
// Package marshal implements encoding and decoding of Noms values into native Go types.
// The mapping between Noms objects and Go values is described
// in the documentation for the Marshal and Unmarshal functions.
package marshal
import (
"bytes"
"io"
"math"
"reflect"
"strconv"
"sync"
"github.com/attic-labs/noms/d"
"github.com/attic-labs/noms/ref"
"github.com/attic-labs/noms/types"
)
// Marshal returns the Noms types.Value of v.
//
// Marshal uses the following type-dependent default encodings:
//
// Boolean values encode as Noms booleans.
//
// Floating point and integer values encode as the equivalent Noms primitive;
// int and uint values with no bit-width are encoded as 32 bit regardless of
// platform defaults, unless they overflow. In that case, they're ignored.
//
// String values encode as Noms strings.
//
// Array and slice values encode as Noms Lists, except that
// []byte encodes as a Noms Blob. A nil slice encodes as an empty List.
//
// Struct values encode as Noms Maps. Each exported struct field
// becomes a member of the object unless
// - the field's tag is "-", or
// - the field is empty and its tag specifies the "omitempty" option.
// The empty values are the standard zero values.
// The Map's default keys are the struct field name as a string,
// but can be specified in the struct field's tag value. The "noms" key in
// the struct field's tag value is the key name, followed by an optional comma
// and options. Examples:
//
// // Field is ignored by this package.
// Field int `noms:"-"`
//
// // Field appears in Noms as key "myName".
// Field int `noms:"myName"`
//
// // Field appears in Noms as key "myName" and
// // the field is omitted from the object if its value is empty,
// // as defined above.
// Field int `noms:"myName,omitempty"`
//
// // Field appears in Noms as key "Field" (the default), but
// // the field is skipped if empty.
// // Note the leading comma.
// Field int `noms:",omitempty"`
//
// The key name will be used if it's a non-empty string consisting of
// only Unicode letters, digits, dollar signs, percent signs, hyphens,
// underscores and slashes.
//
// Anonymous struct fields (i.e. embedded structs) are marshaled as if their inner exported fields
// were fields in the outer struct, mostly subject to the usual Go visibility rules.
// These are amended slightly, as follows:
// 1) An anonymous struct field with a name given in its 'noms' tag is treated as
// having that name, rather than being anonymous.
// 2) An anonymous struct field of interface type is treated the same as having
// that type as its name, rather than being anonymous.
//
// The Go visibility rules for struct fields are amended for us when
// deciding which field to marshal or unmarshal. If there are
// multiple fields at the same level, and that level is the least
// nested (and would therefore be the nesting level selected by the
// usual Go rules), the following extra rules apply:
//
// 1) Of those fields, if any are Noms-tagged, only tagged fields are considered,
// even if there are multiple untagged fields that would otherwise conflict.
// 2) If there is exactly one field (tagged or not according to the first rule), that is selected.
// 3) Otherwise there are multiple fields, and all are ignored; no error occurs.
//
// Map values encode as Noms Maps.
//
// Pointer values encode as the value pointed to.
// A nil pointer is an error.
//
// Interface values encode as the value contained in the interface.
// A nil interface is an error.
//
// Channel, complex, and function values cannot be encoded in Noms.
// Attempting to encode such a value causes Marshal to return
// an UnsupportedTypeError.
//
// Marshal does not currently handle cyclic data structures, though Noms could
// handle them. Passing cyclic structures to Marshal will result in
// an infinite recursion.
//
func Marshal(v interface{}) types.Value {
return reflectValue(reflect.ValueOf(v))
}
func unsupportedTypeMsg(t reflect.Type) string {
return "noms: unsupported type: " + t.String()
}
func unsupportedValueMsg(s string) string {
return "noms: unsupported value: " + s
}
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
}
return false
}
func reflectValue(v reflect.Value) types.Value {
return valueEncoder(v)(v)
}
type encoderFunc func(v reflect.Value) types.Value
var encoderCache struct {
sync.RWMutex
m map[reflect.Type]encoderFunc
}
func valueEncoder(v reflect.Value) encoderFunc {
if !v.IsValid() {
return invalidValueEncoder
}
return typeEncoder(v.Type())
}
func typeEncoder(t reflect.Type) encoderFunc {
encoderCache.RLock()
f := encoderCache.m[t]
encoderCache.RUnlock()
if f != nil {
return f
}
// To deal with recursive types, populate the map with an
// indirect func before we build it. This type waits on the
// real func (f) to be ready and then calls it. This indirect
// func is only used for recursive types.
encoderCache.Lock()
if encoderCache.m == nil {
encoderCache.m = make(map[reflect.Type]encoderFunc)
}
var wg sync.WaitGroup
wg.Add(1)
encoderCache.m[t] = func(v reflect.Value) types.Value {
wg.Wait()
return f(v)
}
encoderCache.Unlock()
// Compute fields without lock.
// Might duplicate effort but won't hold other computations back.
f = newTypeEncoder(t)
wg.Done()
encoderCache.Lock()
encoderCache.m[t] = f
encoderCache.Unlock()
return f
}
var (
bytesBufferType = reflect.TypeOf(&bytes.Buffer{})
readerType = reflect.TypeOf((*io.Reader)(nil)).Elem()
)
// newTypeEncoder constructs an encoderFunc for a type.
func newTypeEncoder(t reflect.Type) encoderFunc {
if t.Implements(readerType) {
return readerEncoder
}
switch t.Kind() {
case reflect.Bool:
return boolEncoder
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return intEncoder
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return uintEncoder
case reflect.Float32:
return float32Encoder
case reflect.Float64:
return float64Encoder
case reflect.String:
return stringEncoder
case reflect.Interface:
return interfaceEncoder
case reflect.Struct:
return newStructEncoder(t)
case reflect.Map:
return newMapEncoder(t)
case reflect.Slice:
return newSliceEncoder(t)
case reflect.Array:
return newArrayEncoder(t)
case reflect.Ptr:
return newPtrEncoder(t)
default:
return unsupportedTypeEncoder
}
}
func invalidValueEncoder(v reflect.Value) types.Value {
return types.NewRef(ref.Ref{}) // Eh?
}
func boolEncoder(v reflect.Value) types.Value {
return types.Bool(v.Bool())
}
func intEncoder(v reflect.Value) types.Value {
switch v.Kind() {
case reflect.Int8:
return types.Int8(v.Int())
case reflect.Int16:
return types.Int16(v.Int())
case reflect.Int32:
return types.Int32(v.Int())
case reflect.Int:
n := v.Int()
d.Exp.False(reflect.ValueOf(types.Int32(0)).OverflowInt(n), " Unsized integers must be 32 bit values; %d is too large.")
return types.Int32(n)
case reflect.Int64:
return types.Int64(v.Int())
default:
d.Exp.Fail("Not an integer")
panic("unreachable")
}
}
func uintEncoder(v reflect.Value) types.Value {
n := v.Uint()
switch v.Kind() {
case reflect.Uint8:
return types.UInt8(n)
case reflect.Uint16:
return types.UInt16(n)
case reflect.Uint32:
return types.UInt32(n)
case reflect.Uint:
n := v.Uint()
d.Exp.False(reflect.ValueOf(types.UInt32(0)).OverflowUint(n), "Unsized integers must be 32 bit values; %d is too large.", n)
return types.UInt32(n)
case reflect.Uint64:
return types.UInt64(n)
default:
d.Exp.Fail("Not an unsigned integer", "%d", n)
panic("unreachable")
}
}
type floatEncoder int // number of bits
func (bits floatEncoder) encode(v reflect.Value) types.Value {
f := v.Float()
d.Exp.False(math.IsInf(f, 0), "Noms can't encode infinity", strconv.FormatFloat(f, 'g', -1, int(bits)))
d.Exp.False(math.IsNaN(f), "Noms can't encode NaN", strconv.FormatFloat(f, 'g', -1, int(bits)))
if bits == 64 {
return types.Float64(f)
}
return types.Float32(f)
}
var (
float32Encoder = (floatEncoder(32)).encode
float64Encoder = (floatEncoder(64)).encode
)
func stringEncoder(v reflect.Value) types.Value {
return types.NewString(v.String())
}
func interfaceEncoder(v reflect.Value) types.Value {
d.Exp.False(v.IsNil(), "Noms can't encode nil interface.")
return reflectValue(v.Elem())
}
func unsupportedTypeEncoder(v reflect.Value) types.Value {
d.Exp.Fail(unsupportedTypeMsg(v.Type()))
panic("unreachable")
}
type structEncoder struct {
fields []field
fieldEncs []encoderFunc
}
func isNilPtrOrNilInterface(v reflect.Value) bool {
switch v.Kind() {
case reflect.Ptr, reflect.Interface:
return v.IsNil()
default:
return false
}
}
func readerEncoder(v reflect.Value) types.Value {
d.Chk.True(v.Type().Implements(readerType))
blob := types.NewMemoryBlob(v.Interface().(io.Reader))
return blob
}
// Noms has no notion of a general-purpose nil value. Thus, if struct encoding encounters a field that holds a nil pointer or interface, it skips it even if that field doesn't have the omitempty option set. Nil maps and slices are encoded as an empty Noms map, set, list or blob as appropriate.
func (se *structEncoder) encode(v reflect.Value) types.Value {
if v.Type() == refRefType {
r := ref.Ref{}
reflect.ValueOf(&r).Elem().Set(v)
return types.NewRef(r)
}
nom := types.NewMap()
for i, f := range se.fields {
fv := fieldByIndex(v, f.index)
if !fv.IsValid() || f.omitEmpty && isEmptyValue(fv) || isNilPtrOrNilInterface(fv) {
continue
}
nom = nom.Set(types.NewString(f.name), se.fieldEncs[i](fv))
}
return nom
}
func newStructEncoder(t reflect.Type) encoderFunc {
fields := cachedTypeFields(t)
se := &structEncoder{
fields: fields,
fieldEncs: make([]encoderFunc, len(fields)),
}
for i, f := range fields {
se.fieldEncs[i] = typeEncoder(typeByIndex(t, f.index))
}
return se.encode
}
type setEncoder struct {
elemEnc encoderFunc
}
// Noms has no notion of a general-purpose nil value. Thus, if set encoding encounters a value that holds a nil pointer or interface, it skips the value. Nil maps and slices are encoded as an empty Noms map, set, list or blob as appropriate.
func (se *setEncoder) encode(v reflect.Value) types.Value {
tmp := make([]types.Value, 0, v.Len())
for _, k := range v.MapKeys() {
if isNilPtrOrNilInterface(k) || !v.MapIndex(k).Bool() {
continue
}
tmp = append(tmp, se.elemEnc(k))
}
return types.NewSet(tmp...)
}
type mapEncoder struct {
keyEnc encoderFunc
elemEnc encoderFunc
}
// Noms has no notion of a general-purpose nil value. Thus, if map encoding encounters a key or value that holds a nil pointer or interface, it skips the whole key/value pair. Nil maps and slices are encoded as an empty Noms map, set, list or blob as appropriate.
func (me *mapEncoder) encode(v reflect.Value) types.Value {
nom := types.NewMap()
for _, k := range v.MapKeys() {
valueAtK := v.MapIndex(k)
if isNilPtrOrNilInterface(k) || isNilPtrOrNilInterface(valueAtK) {
continue
}
nom = nom.Set(me.keyEnc(k), me.elemEnc(valueAtK))
}
return nom
}
func newMapEncoder(t reflect.Type) encoderFunc {
// Noms sets are unmarshaled to map[interface{}]bool, so we marshal anything that maps to bool as a set.
if t.Elem().Kind() == reflect.Bool {
se := &setEncoder{typeEncoder(t.Key())}
return se.encode
}
me := &mapEncoder{typeEncoder(t.Key()), typeEncoder(t.Elem())}
return me.encode
}
func encodeByteSlice(v reflect.Value) types.Value {
if v.IsNil() {
return types.NewMemoryBlob(&bytes.Buffer{})
}
return types.NewMemoryBlob(bytes.NewReader(v.Bytes()))
}
func newSliceEncoder(t reflect.Type) encoderFunc {
return newArrayEncoder(t)
}
type arrayEncoder struct {
elemEnc encoderFunc
}
// Noms has no notion of a general-purpose nil value. Thus, if array/slice encoding encounters a value that holds a nil pointer or interface, it skips the value. Nil maps and slices are encoded as an empty Noms map, set, list or blob as appropriate.
func (ae *arrayEncoder) encode(v reflect.Value) types.Value {
tmp := make([]types.Value, 0, v.Len())
for i := 0; i < v.Len(); i++ {
valueAtI := v.Index(i)
if isNilPtrOrNilInterface(valueAtI) {
continue
}
tmp = append(tmp, ae.elemEnc(valueAtI))
}
return types.NewList(tmp...)
}
func newArrayEncoder(t reflect.Type) encoderFunc {
enc := &arrayEncoder{typeEncoder(t.Elem())}
return enc.encode
}
type ptrEncoder struct {
elemEnc encoderFunc
}
func (pe *ptrEncoder) encode(v reflect.Value) types.Value {
d.Exp.False(v.IsNil(), "Noms can't encode nil ptr.")
return pe.elemEnc(v.Elem())
}
func newPtrEncoder(t reflect.Type) encoderFunc {
enc := &ptrEncoder{typeEncoder(t.Elem())}
return enc.encode
}

600
marshal/marshal_test.go
View File

@@ -1,600 +0,0 @@
// Modified from golang's encoding/json/encode_test.go at 80e6d638bf309181eadcb3fecbe99d2d8518e364.
package marshal
import (
"bytes"
"image"
"io"
"io/ioutil"
"testing"
"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert"
"github.com/attic-labs/noms/types"
)
type Optionals struct {
Sr string `noms:"sr"`
So string `noms:"so,omitempty"`
Sw string `noms:"-"`
Ir int `noms:"omitempty"` // actually named omitempty, not an option
Io int `noms:"io,omitempty"`
Slr []string `noms:"slr,random"`
Slo []string `noms:"slo,omitempty"`
Mr map[string]interface{} `noms:"mr"`
Mo map[string]interface{} `noms:",omitempty"`
Fr float64 `noms:"fr"`
Fo float64 `noms:"fo,omitempty"`
Br bool `noms:"br"`
Bo bool `noms:"bo,omitempty"`
Ur uint `noms:"ur"`
Uo uint `noms:"uo,omitempty"`
Str struct{} `noms:"str"`
Sto struct{} `noms:"sto,omitempty"`
}
var optionalsExpected = types.NewMap(
types.NewString("sr"), types.NewString(""),
types.NewString("omitempty"), types.Int32(0),
types.NewString("slr"), types.NewList(),
types.NewString("mr"), types.NewMap(),
types.NewString("fr"), types.Float64(0),
types.NewString("br"), types.Bool(false),
types.NewString("ur"), types.UInt32(0),
types.NewString("str"), types.NewMap(),
types.NewString("sto"), types.NewMap(),
)
func TestOmitEmpty(t *testing.T) {
assert := assert.New(t)
var o Optionals
o.Sw = "something"
o.Mr = map[string]interface{}{}
o.Mo = map[string]interface{}{}
nom := Marshal(&o)
if nom, ok := nom.(types.Map); !ok {
assert.Fail("%+v should be a Map", nom)
} else {
assert.True(optionalsExpected.Equals(nom))
}
}
type IntType int
type MyStruct struct {
IntType
}
func TestAnonymousNonstruct(t *testing.T) {
var i IntType = 11
a := MyStruct{i}
nom := Marshal(a)
if nom, ok := nom.(types.Map); !ok {
assert.Fail(t, "nom should be a Map, not %T", nom)
} else {
assert.EqualValues(t, i, nom.Get(types.NewString("IntType")))
}
}
var marshaledEmbeds = types.NewMap(
types.NewString("Level0"), types.Int32(1),
types.NewString("Level1b"), types.Int32(2),
types.NewString("Level1c"), types.Int32(3),
types.NewString("Level1a"), types.Int32(5),
types.NewString("LEVEL1B"), types.Int32(6),
types.NewString("e"), types.NewMap(
types.NewString("Level1a"), types.Int32(8),
types.NewString("Level1b"), types.Int32(9),
types.NewString("Level1c"), types.Int32(10),
types.NewString("Level1d"), types.Int32(11),
types.NewString("x"), types.Int32(12)),
types.NewString("Loop1"), types.Int32(13),
types.NewString("Loop2"), types.Int32(14),
types.NewString("X"), types.Int32(15),
types.NewString("Y"), types.Int32(16),
types.NewString("Z"), types.Int32(17))
func TestMarshalEmbeds(t *testing.T) {
top := &Top{
Level0: 1,
Embed0: Embed0{
Level1b: 2,
Level1c: 3,
},
Embed0a: &Embed0a{
Level1a: 5,
Level1b: 6,
},
Embed0b: &Embed0b{
Level1a: 8,
Level1b: 9,
Level1c: 10,
Level1d: 11,
Level1e: 12,
},
Loop: Loop{
Loop1: 13,
Loop2: 14,
},
Embed0p: Embed0p{
Point: image.Point{X: 15, Y: 16},
},
Embed0q: Embed0q{
Point: Point{Z: 17},
},
}
b := Marshal(top)
assert.EqualValues(t, marshaledEmbeds, b)
}
type BugA struct {
S string
}
type BugB struct {
BugA
S string
}
type BugC struct {
S string
}
// Legal Go: We never use the repeated embedded field (S).
type BugX struct {
A int
BugA
BugB
}
// Issue 5245.
func TestEmbeddedBug(t *testing.T) {
assert := assert.New(t)
v := BugB{
BugA{"A"},
"B",
}
nom := Marshal(v)
nom = nom.(types.Map)
expected := types.NewMap(types.NewString("S"), types.NewString("B"))
assert.EqualValues(expected, nom)
// Now check that the duplicate field, S, does not appear.
x := BugX{
A: 23,
}
nom = Marshal(x)
nom = nom.(types.Map)
expected = types.NewMap(types.NewString("A"), types.Int32(23))
assert.EqualValues(expected, nom)
}
type BugD struct { // Same as BugA after tagging.
XXX string `noms:"S"`
}
// BugD's tagged S field should dominate BugA's.
type BugY struct {
BugA
BugD
}
// Test that a field with a tag dominates untagged fields.
func TestTaggedFieldDominates(t *testing.T) {
assert := assert.New(t)
v := BugY{
BugA{"BugA"},
BugD{"BugD"},
}
nom := Marshal(v)
nom = nom.(types.Map)
expected := types.NewMap(types.NewString("S"), types.NewString("BugD"))
assert.EqualValues(expected, nom)
}
// There are no tags here, so S should not appear.
type BugZ struct {
BugA
BugC
BugY // Contains a tagged S field through BugD; should not dominate.
}
func TestDuplicatedFieldDisappears(t *testing.T) {
assert := assert.New(t)
v := BugZ{
BugA{"BugA"},
BugC{"BugC"},
BugY{
BugA{"nested BugA"},
BugD{"nested BugD"},
},
}
nom := Marshal(v)
nom = nom.(types.Map)
expected := types.NewMap()
assert.EqualValues(expected, nom)
}
func TestMarshalSetP(t *testing.T) {
assert := assert.New(t)
setP := map[*Small]bool{
&Small{Tag: "tag"}: true,
nil: true,
}
expected := types.NewSet(types.NewMap(types.NewString("Tag"), types.NewString("tag")))
nom := Marshal(setP)
// Check against canned marshalled representation.
if nom, ok := nom.(types.Set); !ok {
assert.Fail("Marshal should return set.", "nom is %v", nom)
return
} else if assert.NotNil(nom) {
assert.True(expected.Equals(nom), "%v != %v", expected, nom)
}
}
func TestMarshalBuffer(t *testing.T) {
assert := assert.New(t)
expected := []byte("abc")
nom := Marshal(bytes.NewBufferString("abc"))
validateBlob(assert, expected, nom)
}
func TestMarshalReader(t *testing.T) {
assert := assert.New(t)
expected := []byte("abc")
var in io.Reader = bytes.NewBuffer(expected)
nom := Marshal(in)
validateBlob(assert, expected, nom)
in = bytes.NewBuffer(expected)
nom = Marshal(&in)
validateBlob(assert, expected, nom)
}
func validateBlob(assert *assert.Assertions, expected []byte, nom types.Value) {
if nom, ok := nom.(types.Blob); !ok || nom == nil {
assert.Fail("Marshal should return blob.", "nom is %v", nom)
} else {
nomBytes, err := ioutil.ReadAll(nom.Reader())
assert.NoError(err)
assert.EqualValues(expected, nomBytes)
}
}
func TestMarshal(t *testing.T) {
assert := assert.New(t)
nom := Marshal(allValue)
// Check against canned marshalled representation.
if nom, ok := nom.(types.Map); !ok {
assert.Fail("Marshal should return map.", "nom is %v", nom)
return
}
nomMap := nom.(types.Map)
assert.NotNil(nomMap)
assert.Equal(allNomsValue.Len(), nomMap.Len(), "%d != %d", allNomsValue.Len(), nomMap.Len())
nomMap.Iter(func(k, v types.Value) (stop bool) {
expected := allNomsValue.Get(k)
assert.True(expected.Equals(v), "%s: %v != %v", k.(types.String).String(), expected, v)
return
})
nom = Marshal(pallValue)
if nom, ok := nom.(types.Map); !ok {
assert.Fail("Marshal should return map.", "nom is %v", nom)
return
}
nomMap = nom.(types.Map)
assert.NotNil(nomMap)
assert.Equal(pallNomsValue.Len(), nomMap.Len(), "%d != %d", pallNomsValue.Len(), nomMap.Len())
nomMap.Iter(func(k, v types.Value) (stop bool) {
expected := pallNomsValue.Get(k)
assert.True(expected.Equals(v), "%s: %v != %v", k.(types.String).String(), expected, v)
return
})
}
// A struct with fields for all the things we can marshal and unmarshal relatively symmetrically.
type All struct {
Bool bool
Int int
Int8 int8
Int16 int16
Int32 int32
Int64 int64
Uint uint
Uint8 uint8
Uint16 uint16
Uint32 uint32
Uint64 uint64
Float32 float32
Float64 float64
Foo string `noms:"bar"`
Foo2 string `noms:"bar2,dummyopt"`
PBool *bool
PInt *int
PInt8 *int8
PInt16 *int16
PInt32 *int32
PInt64 *int64
PUint *uint
PUint8 *uint8
PUint16 *uint16
PUint32 *uint32
PUint64 *uint64
PFloat32 *float32
PFloat64 *float64
String string
PString *string
Map map[string]Small
MapP map[string]*Small
PMap *map[string]Small
PMapP *map[string]*Small
EmptyMap map[string]Small
NilMap map[string]Small
Slice []Small
SliceP []*Small
PSlice *[]Small
PSliceP *[]*Small
EmptySlice []Small
NilSlice []Small
StringSlice []string
ByteSlice []byte
Small Small
PSmall *Small
PPSmall **Small
Interface interface{}
PInterface *interface{}
Set map[Small]bool
// SetP map[*Small]bool must be tested separately. Two maps that use pointers for keys will never compare equal unless literally the same pointers are used as keys in each. So, even if the maps had as keys pointers to structs that were equal, the maps would not be equal. This breaks the test harness.
PSet *map[Small]bool
PSetP *map[*Small]bool
EmptySet map[Small]bool
NilSet map[Small]bool
unexported int
}
type Small struct {
Tag string
}
// Sets values for everything except the fields that are pointers.
var allValue = All{
Bool: true,
Int: 2,
Int8: 3,
Int16: 4,
Int32: 5,
Int64: 6,
Uint: 7,
Uint8: 8,
Uint16: 9,
Uint32: 10,
Uint64: 11,
Float32: 14.1,
Float64: 15.1,
Foo: "foo",
Foo2: "foo2",
String: "16",
Map: map[string]Small{
"17": {Tag: "tag17"},
"18": {Tag: "tag18"},
},
MapP: map[string]*Small{
"19": {Tag: "tag19"},
"20": nil,
},
EmptyMap: map[string]Small{},
NilMap: nil,
Slice: []Small{{Tag: "tag20"}, {Tag: "tag21"}},
SliceP: []*Small{{Tag: "tag22"}, nil, {Tag: "tag23"}},
EmptySlice: []Small{},
NilSlice: nil,
StringSlice: []string{"str24", "str25", "str26"},
ByteSlice: []byte{27, 28, 29},
Small: Small{Tag: "tag30"},
PSmall: &Small{Tag: "tag31"},
Interface: 5.2,
Set: map[Small]bool{
Small{Tag: "tag32"}: false,
Small{Tag: "tag33"}: true,
},
EmptySet: map[Small]bool{},
NilSet: nil,
}
// Sets values for ONLY the fields that are pointers.
var pallValue = All{
PBool: &allValue.Bool,
PInt: &allValue.Int,
PInt8: &allValue.Int8,
PInt16: &allValue.Int16,
PInt32: &allValue.Int32,
PInt64: &allValue.Int64,
PUint: &allValue.Uint,
PUint8: &allValue.Uint8,
PUint16: &allValue.Uint16,
PUint32: &allValue.Uint32,
PUint64: &allValue.Uint64,
PFloat32: &allValue.Float32,
PFloat64: &allValue.Float64,
PString: &allValue.String,
PMap: &allValue.Map,
PMapP: &allValue.MapP,
PSlice: &allValue.Slice,
PSliceP: &allValue.SliceP,
PPSmall: &allValue.PSmall,
PInterface: &allValue.Interface,
PSet: &allValue.Set,
}
// Used in creating canned marshaled values below.
func makeNewBlob(b []byte) types.Blob {
return types.NewMemoryBlob(bytes.NewBuffer(b))
}
// Canned marshaled version of allValue
var allNomsValue = types.NewMap(
types.NewString("Bool"), types.Bool(true),
types.NewString("Int"), types.Int32(2),
types.NewString("Int8"), types.Int8(3),
types.NewString("Int16"), types.Int16(4),
types.NewString("Int32"), types.Int32(5),
types.NewString("Int64"), types.Int64(6),
types.NewString("Uint"), types.UInt32(7),
types.NewString("Uint8"), types.UInt8(8),
types.NewString("Uint16"), types.UInt16(9),
types.NewString("Uint32"), types.UInt32(10),
types.NewString("Uint64"), types.UInt64(11),
types.NewString("Float32"), types.Float32(14.1),
types.NewString("Float64"), types.Float64(15.1),
types.NewString("bar"), types.NewString("foo"),
types.NewString("bar2"), types.NewString("foo2"),
types.NewString("String"), types.NewString("16"),
types.NewString("Map"), types.NewMap(
types.NewString("17"), types.NewMap(
types.NewString("Tag"), types.NewString("tag17")),
types.NewString("18"), types.NewMap(
types.NewString("Tag"), types.NewString("tag18"))),
types.NewString("MapP"), types.NewMap(
types.NewString("19"), types.NewMap(
types.NewString("Tag"), types.NewString("tag19"))),
types.NewString("EmptyMap"), types.NewMap(),
types.NewString("NilMap"), types.NewMap(),
types.NewString("Slice"), types.NewList(
types.NewMap(types.NewString("Tag"), types.NewString("tag20")),
types.NewMap(types.NewString("Tag"), types.NewString("tag21"))),
types.NewString("SliceP"), types.NewList(
types.NewMap(types.NewString("Tag"), types.NewString("tag22")),
types.NewMap(types.NewString("Tag"), types.NewString("tag23"))),
types.NewString("EmptySlice"), types.NewList(),
types.NewString("NilSlice"), types.NewList(),
types.NewString("StringSlice"), types.NewList(
types.NewString("str24"), types.NewString("str25"), types.NewString("str26")),
types.NewString("ByteSlice"), types.NewList(
types.UInt8(27), types.UInt8(28), types.UInt8(29)),
types.NewString("Small"), types.NewMap(types.NewString("Tag"), types.NewString("tag30")),
types.NewString("PSmall"), types.NewMap(types.NewString("Tag"), types.NewString("tag31")),
types.NewString("Interface"), types.Float64(5.2),
types.NewString("Set"), types.NewSet(types.NewMap(types.NewString("Tag"), types.NewString("tag33"))),
types.NewString("EmptySet"), types.NewSet(),
types.NewString("NilSet"), types.NewSet())
// Canned marshaled version of pallValue.
var pallNomsValue = types.NewMap(
types.NewString("Bool"), types.Bool(false),
types.NewString("Int"), types.Int32(0),
types.NewString("Int8"), types.Int8(0),
types.NewString("Int16"), types.Int16(0),
types.NewString("Int32"), types.Int32(0),
types.NewString("Int64"), types.Int64(0),
types.NewString("Uint"), types.UInt32(0),
types.NewString("Uint8"), types.UInt8(0),
types.NewString("Uint16"), types.UInt16(0),
types.NewString("Uint32"), types.UInt32(0),
types.NewString("Uint64"), types.UInt64(0),
types.NewString("Float32"), types.Float32(0),
types.NewString("Float64"), types.Float64(0),
types.NewString("bar"), types.NewString(""),
types.NewString("bar2"), types.NewString(""),
types.NewString("PBool"), types.Bool(true),
types.NewString("PInt"), types.Int32(2),
types.NewString("PInt8"), types.Int8(3),
types.NewString("PInt16"), types.Int16(4),
types.NewString("PInt32"), types.Int32(5),
types.NewString("PInt64"), types.Int64(6),
types.NewString("PUint"), types.UInt32(7),
types.NewString("PUint8"), types.UInt8(8),
types.NewString("PUint16"), types.UInt16(9),
types.NewString("PUint32"), types.UInt32(10),
types.NewString("PUint64"), types.UInt64(11),
types.NewString("PFloat32"), types.Float32(14.1),
types.NewString("PFloat64"), types.Float64(15.1),
types.NewString("String"), types.NewString(""),
types.NewString("PString"), types.NewString("16"),
types.NewString("Map"), types.NewMap(),
types.NewString("MapP"), types.NewMap(),
types.NewString("PMap"), types.NewMap(
types.NewString("17"), types.NewMap(
types.NewString("Tag"), types.NewString("tag17")),
types.NewString("18"), types.NewMap(
types.NewString("Tag"), types.NewString("tag18"))),
types.NewString("PMapP"), types.NewMap(
types.NewString("19"), types.NewMap(
types.NewString("Tag"), types.NewString("tag19"))),
types.NewString("EmptyMap"), types.NewMap(),
types.NewString("NilMap"), types.NewMap(),
types.NewString("Slice"), types.NewList(),
types.NewString("SliceP"), types.NewList(),
types.NewString("PSlice"), types.NewList(
types.NewMap(types.NewString("Tag"), types.NewString("tag20")),
types.NewMap(types.NewString("Tag"), types.NewString("tag21"))),
types.NewString("PSliceP"), types.NewList(
types.NewMap(types.NewString("Tag"), types.NewString("tag22")),
types.NewMap(types.NewString("Tag"), types.NewString("tag23"))),
types.NewString("EmptySlice"), types.NewList(),
types.NewString("NilSlice"), types.NewList(),
types.NewString("StringSlice"), types.NewList(),
types.NewString("ByteSlice"), types.NewList(),
types.NewString("Small"), types.NewMap(types.NewString("Tag"), types.NewString("")),
// PSmall and Interface are not marhsaled, as they're a nil ptr and nil interface, respectively.
types.NewString("PPSmall"), types.NewMap(types.NewString("Tag"), types.NewString("tag31")),
types.NewString("PInterface"), types.Float64(5.2),
types.NewString("Set"), types.NewSet(),
types.NewString("PSet"), types.NewSet(types.NewMap(types.NewString("Tag"), types.NewString("tag33"))),
types.NewString("EmptySet"), types.NewSet(),
types.NewString("NilSet"), types.NewSet())

46
marshal/tags.go
View File

@@ -1,46 +0,0 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Copied from the encoding/json package at 80e6d638bf309181eadcb3fecbe99d2d8518e364.
package marshal
import (
"strings"
)
// tagOptions is the string following a comma in a struct field's "json"
// tag, or the empty string. It does not include the leading comma.
type tagOptions string
// parseTag splits a struct field's json tag into its name and
// comma-separated options.
func parseTag(tag string) (string, tagOptions) {
if idx := strings.Index(tag, ","); idx != -1 {
return tag[:idx], tagOptions(tag[idx+1:])
}
return tag, tagOptions("")
}
// Contains reports whether a comma-separated list of options
// contains a particular substr flag. substr must be surrounded by a
// string boundary or commas.
func (o tagOptions) Contains(optionName string) bool {
if len(o) == 0 {
return false
}
s := string(o)
for s != "" {
var next string
i := strings.Index(s, ",")
if i >= 0 {
s, next = s[:i], s[i+1:]
}
if s == optionName {
return true
}
s = next
}
return false
}

30
marshal/tags_test.go
View File

@@ -1,30 +0,0 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Copied from the encoding/json package at 80e6d638bf309181eadcb3fecbe99d2d8518e364.
package marshal
import (
"testing"
)
func TestTagParsing(t *testing.T) {
name, opts := parseTag("field,foobar,foo")
if name != "field" {
t.Fatalf("name = %q, want field", name)
}
for _, tt := range []struct {
opt string
want bool
}{
{"foobar", true},
{"foo", true},
{"bar", false},
} {
if opts.Contains(tt.opt) != tt.want {
t.Errorf("Contains(%q) = %v", tt.opt, !tt.want)
}
}
}

548
marshal/unmarshal.go
View File

@@ -1,548 +0,0 @@
// Modified from golang's encoding/json/decode.go at 80e6d638bf309181eadcb3fecbe99d2d8518e364.
package marshal
import (
"bytes"
"fmt"
"io"
"math"
"reflect"
"github.com/attic-labs/noms/d"
"github.com/attic-labs/noms/ref"
"github.com/attic-labs/noms/types"
)
// Unmarshal unmarshals nom into v
//
// Unmarshal traverses the value trees of nom and v in tandem,
// unmarshaling the data from nom into v as it goes. New space is
// allocated for reference types encountered in v as needed. The semantics
// are essentially like those provided by encoding/json; Unmarshal will do
// its best to map data from nom onto the value in v, skipping fields in
// nom that don't have an analog in v. The primary difference is that
// there's no analog to the json.Unmarshaler interface.
//
// Any value can be "unmarshaled" into a ref.Ref, though the target will be populated only with the value's ref.
// Primitive values can be unmarshaled into Go primitives.
// Lists can be unmarshaled into slices, with space being allocated dynamically.
// Lists can be unmarshaled into arrays if there is room for all the data.
// Maps can be unmarshaled into Go maps.
// Sets can be unmarshaled into Go maps of the form map[ElementType]bool
// Blobs can be unmarshaled into anything that implements io.Writer by using io.Copy.
// Note that your Writer will not be cleared or destroyed before data is written to it.
// Blobs can be unmarshaled into slices, with space being allocated dynamically.
// Blobs can be unmarshaled into arrays if there is room for all the data.
//
// Unline json.Unmarshal, this code treats overflows and significant field
// type mismatches as fatal. For example, a types.Int32 will be unmarshalled
// into an int64, because that's safe, but a types.Float64 won't be allowed
// to overflow a float32 in the target. Similarly, Unmarshal will error on a
// piece of data in nom that maps to a target of the wrong type in v -- e.g.
// both nom and v have a field named Foo, but it's a types.String in the
// former and an int in the latter.
func Unmarshal(nom types.Value, v interface{}) {
rv := reflect.ValueOf(v)
d.Exp.False(rv.Kind() != reflect.Ptr || rv.IsNil(), invalidUnmarshalMsg(reflect.TypeOf(v)))
unmarshalValue(nom, rv)
}
var (
refRefType = reflect.TypeOf(ref.Ref{})
writerType = reflect.TypeOf((*io.Writer)(nil)).Elem()
)
func invalidTypeMsg(v string, t reflect.Type) string {
return "noms: cannot unmarshal noms " + v + " into Go value of type " + t.String()
}
// The argument to Unmarshal must be a non-nil pointer
func invalidUnmarshalMsg(t reflect.Type) string {
if t == nil {
return "noms: Unmarshal(nil)"
}
if t.Kind() != reflect.Ptr {
return "noms: Unmarshal(non-pointer " + t.String() + ")"
}
return "noms: Unmarshal(nil " + t.String() + ")"
}
// unmarshalValue unpacks an arbitrary types.Value into v.
func unmarshalValue(nom types.Value, v reflect.Value) {
if !v.IsValid() {
return
}
switch nom := nom.(type) {
case types.Blob:
unmarshalBlob(nom, v)
case types.List:
unmarshalList(nom, v)
case types.Map:
unmarshalMap(nom, v)
case primitive:
unmarshalPrimitive(nom, v)
case types.Ref:
unmarshalRef(nom, v)
case types.Set:
unmarshalSet(nom, v)
case types.String:
unmarshalString(nom, v)
default:
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), v.Type()))
}
}
// indirect walks down v, allocating pointers as needed,
// until it gets to a non-pointer.
func indirect(v reflect.Value) reflect.Value {
// If v is a named type and is addressable,
// start with its address, so that if the type has pointer methods,
// we find them.
if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
v = v.Addr()
}
for {
if nv, ok := loadValueFromInterfaceIfAddressable(v); ok {
v = nv
continue
}
if v.Kind() != reflect.Ptr {
break
}
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
v = v.Elem()
}
return v
}
// indirect walks down v, allocating pointers as needed,
// until it gets to a non-pointer.
func findImplementor(v reflect.Value, i reflect.Type) (reflect.Value, bool) {
d.Chk.Equal(reflect.Interface, i.Kind())
// If v is a named type and is addressable,
// start with its address, so that if the type has pointer methods,
// we find them.
if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
v = v.Addr()
}
for {
if v.Type().Implements(writerType) {
return v, true
}
if nv, ok := loadValueFromInterfaceIfAddressable(v); ok {
v = nv
continue
}
if v.Kind() != reflect.Ptr {
break
}
d.Chk.False(v.IsNil())
v = v.Elem()
}
return v, false
}
// Load value from interface, but only if the result will be usefully addressable.
func loadValueFromInterfaceIfAddressable(v reflect.Value) (reflect.Value, bool) {
if v.Kind() == reflect.Interface && !v.IsNil() {
e := v.Elem()
if e.Kind() == reflect.Ptr && !e.IsNil() {
return e, true
}
}
return v, false
}
// TODO: unmarshal into *io.Reader? BUG 160
func unmarshalBlob(nom types.Blob, v reflect.Value) {
origType := v.Type() // For error reporting.
finalV := indirect(v) // To populate any nil pointers.
if v, ok := findImplementor(v, writerType); ok {
n, err := io.Copy(v.Interface().(io.Writer), nom.Reader())
d.Exp.NoError(err)
d.Exp.EqualValues(nom.Len(), n, "Blob too large")
return
}
v = finalV
// Decoding into nil interface? Stuff a reader in there.
if v.Kind() == reflect.Interface && v.NumMethod() == 0 {
v.Set(reflect.ValueOf(nom.Reader()))
return
} else if v.Kind() == reflect.Struct && v.Type() == refRefType {
v.Set(reflect.ValueOf(nom.Ref()))
return
}
// The reflection stuff all uses int, so I'll need this.
nomLen := truncateUint64(nom.Len())
// Check type of target.
switch v.Kind() {
default:
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), origType))
return
case reflect.Array:
break
case reflect.Slice:
// If nom is too big, just make v as big as possible.
if nomLen == 0 || nomLen > v.Cap() {
v.Set(reflect.MakeSlice(v.Type(), nomLen, nomLen))
} else {
v.SetLen(nomLen)
}
}
read, err := io.ReadFull(nom.Reader(), v.Bytes())
d.Exp.NoError(err)
d.Exp.Equal(nomLen, read, "blob too large")
return
}
func unmarshalList(nom types.List, v reflect.Value) {
origType := v.Type()
v = indirect(v)
// Decoding into nil interface? Switch to non-reflect code.
if v.Kind() == reflect.Interface && v.NumMethod() == 0 {
v.Set(reflect.ValueOf(listInterface(nom)))
return
} else if v.Kind() == reflect.Struct && v.Type() == refRefType {
v.Set(reflect.ValueOf(nom.Ref()))
return
}
// Check type of target.
switch v.Kind() {
default:
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), origType))
return
case reflect.Slice:
// The reflection stuff all uses int, so if nom is too big, just make v as big as possible.
nomLen := truncateUint64(nom.Len())
if nomLen == 0 || nomLen > v.Cap() {
v.Set(reflect.MakeSlice(v.Type(), nomLen, nomLen))
} else {
v.SetLen(nomLen)
}
case reflect.Array:
break
}
i := 0
for ; uint64(i) < nom.Len(); i++ {
d.Exp.True(i < v.Len(), "list is too large for target array of size %d", v.Len())
// Decode into element.
unmarshalValue(nom.Get(uint64(i)), v.Index(i))
}
if i < v.Len() {
if v.Kind() == reflect.Array {
// Array. Zero the rest.
z := reflect.Zero(v.Type().Elem())
for ; i < v.Len(); i++ {
v.Index(i).Set(z)
}
} else {
v.SetLen(i)
}
}
}
func unmarshalMap(nom types.Map, v reflect.Value) {
origType := v.Type()
v = indirect(v)
// Decoding into nil interface? Switch to non-reflect code.
if v.Kind() == reflect.Interface && v.NumMethod() == 0 {
v.Set(reflect.ValueOf(mapInterface(nom)))
return
} else if v.Kind() == reflect.Struct && v.Type() == refRefType {
v.Set(reflect.ValueOf(nom.Ref()))
return
}
switch v.Kind() {
default:
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), origType))
return
case reflect.Struct:
unmarshalStruct(nom, v)
return
case reflect.Map:
if v.IsNil() {
v.Set(reflect.MakeMap(v.Type()))
}
}
keyType := v.Type().Key()
elemType := v.Type().Elem()
mapKey := reflect.New(keyType).Elem()
mapElem := reflect.New(elemType).Elem()
nom.Iter(func(key, value types.Value) (stop bool) {
mapKey.Set(reflect.Zero(keyType))
unmarshalValue(key, mapKey)
mapElem.Set(reflect.Zero(elemType))
unmarshalValue(value, mapElem)
v.SetMapIndex(mapKey, mapElem)
return
})
}
// TODO: Should be exported from types package?
type primitive interface {
Ref() ref.Ref
ToPrimitive() interface{}
}
func unmarshalPrimitive(nom primitive, v reflect.Value) {
origType := v.Type()
v = indirect(v)
nomValue := reflect.ValueOf(nom.ToPrimitive())
if v.Kind() == reflect.Interface && v.NumMethod() == 0 {
// You can set the nil interface to a value of any type.
v.Set(nomValue)
return
} else if v.Kind() == reflect.Struct && v.Type() == refRefType {
v.Set(reflect.ValueOf(nom.Ref()))
return
}
switch v.Kind() {
default:
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), origType))
case reflect.Bool:
v.SetBool(nomValue.Bool())
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
n := nomValue.Int()
d.Exp.False(v.OverflowInt(n), invalidTypeMsg(fmt.Sprintf("number %d", n), origType))
v.SetInt(n)
case reflect.Float32, reflect.Float64:
n := nomValue.Float()
d.Exp.False(v.OverflowFloat(n), invalidTypeMsg(fmt.Sprintf("number %f", n), origType))
v.SetFloat(n)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
n := nomValue.Uint()
d.Exp.False(v.OverflowUint(n), invalidTypeMsg(fmt.Sprintf("number %d", n), origType))
v.SetUint(n)
}
}
func unmarshalRef(nom types.Ref, v reflect.Value) {
origType := v.Type()
v = indirect(v)
// Decoding into nil interface? Stuff a string in there.
if v.Kind() == reflect.Interface && v.NumMethod() == 0 {
v.Set(reflect.ValueOf(nom.TargetRef().String()))
return
} else if v.Kind() == reflect.Struct && v.Type() == refRefType {
v.Set(reflect.ValueOf(nom.TargetRef()))
return
}
// Check type of target.
switch v.Kind() {
default:
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), origType))
return
case reflect.String:
v.SetString(nom.TargetRef().String())
return
case reflect.Slice:
if v.Type().Elem().Kind() == reflect.Uint8 {
// A byte-slice
digestLen := len(nom.TargetRef().Digest())
v.Set(reflect.MakeSlice(v.Type(), digestLen, digestLen))
reflect.Copy(v, reflect.ValueOf(nom.TargetRef().Digest()))
return
}
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), origType))
return
}
}
func unmarshalSet(nom types.Set, v reflect.Value) {
origType := v.Type()
v = indirect(v)
// Decoding into nil interface? Switch to non-reflect code.
if v.Kind() == reflect.Interface && v.NumMethod() == 0 {
v.Set(reflect.ValueOf(setInterface(nom)))
return
} else if v.Kind() == reflect.Struct && v.Type() == refRefType {
v.Set(reflect.ValueOf(nom.Ref()))
return
}
// Check type of target.
switch v.Kind() {
default:
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), origType))
return
case reflect.Map:
// map must have bool values.
t := v.Type()
if t.Elem().Kind() != reflect.Bool {
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), origType))
return
}
if v.IsNil() {
v.Set(reflect.MakeMap(t))
}
break
}
// Iterate through nom, unmarshaling into new elements of the same type as v's keys.
newElem := reflect.New(v.Type().Key()).Elem() // New returns a pointer, hence the Elem().
trueValue := reflect.ValueOf(true)
nom.Iter(func(elem types.Value) (stop bool) {
unmarshalValue(elem, newElem)
v.SetMapIndex(newElem, trueValue)
return
})
return
}
func unmarshalString(nom types.String, v reflect.Value) {
origType := v.Type()
v = indirect(v)
if v.Kind() == reflect.Interface && v.NumMethod() == 0 {
// You can set the nil interface to a value of any type.
v.Set(reflect.ValueOf(nom.String()))
return
} else if v.Kind() == reflect.Struct && v.Type() == refRefType {
v.Set(reflect.ValueOf(nom.Ref()))
return
}
switch v.Kind() {
default:
d.Exp.Fail(invalidTypeMsg(reflect.TypeOf(nom).Name(), origType))
case reflect.String:
v.SetString(nom.String())
}
return
}
func unmarshalStruct(nom types.Map, v reflect.Value) {
v = indirect(v)
d.Chk.Equal(reflect.Struct, v.Kind())
if v.Type() == refRefType {
v.Set(reflect.ValueOf(nom.Ref()))
return
}
nom.Iter(func(key, value types.Value) (stop bool) {
if key, ok := key.(types.String); ok {
// Look at the fields defined for v and see if any match the key.
var f *field
kb := []byte(key.String())
fields := cachedTypeFields(v.Type())
for i := range fields {
ff := &fields[i]
if bytes.Equal(ff.nameBytes, kb) {
f = ff
break
}
if f == nil && ff.equalFold(ff.nameBytes, kb) {
f = ff
}
}
if f != nil {
// If a field is found, walk down any nested struct definitions and allocate space for any pointers along the way to ensure that the field actually has storage allocated.
subv := v
for _, i := range f.index {
if subv.Kind() == reflect.Ptr {
if subv.IsNil() {
subv.Set(reflect.New(subv.Type().Elem()))
}
subv = subv.Elem()
}
subv = subv.Field(i)
}
// subv is left pointing to the field we want to unmarshal into.
unmarshalValue(value, subv)
}
}
return
})
return
}
func truncateUint64(u uint64) (out int) {
// TODO: check at runtime to see if ints are 32 or 64 bits and use the right constant.
out = math.MaxInt32
if u < uint64(out) {
out = int(u)
}
return
}
// The xxxInterface routines build up a value to be stored
// in an empty interface. They are not strictly necessary,
// but they avoid the weight of reflection in this common case.
// valueInterface is like value but returns interface{}
func valueInterface(nom types.Value) interface{} {
switch nom := nom.(type) {
case types.Blob:
d.Chk.Fail("Blobs should be handled by returing blob.Reader() directly.")
panic("unreachable")
case types.List:
return listInterface(nom)
case types.Map:
return mapInterface(nom)
case types.Set:
return setInterface(nom)
case types.String:
return nom.String()
case primitive:
return nom.ToPrimitive()
default:
panic("unreachable")
}
}
// listInterface is like unmarshalList but returns []interface{}.
func listInterface(nom types.List) (v []interface{}) {
v = make([]interface{}, 0)
for i := uint64(0); i < nom.Len(); i++ {
v = append(v, valueInterface(nom.Get(i)))
}
return
}
// setInterface is like unmarshalSet but returns map[interface{}]bool.
func setInterface(nom types.Set) (v map[interface{}]bool) {
v = make(map[interface{}]bool)
nom.Iter(func(elem types.Value) (stop bool) {
v[valueInterface(elem)] = true
return
})
return
}
// mapInterface is like unmarshalMap but returns map[string]interface{}.
func mapInterface(nom types.Map) (m map[interface{}]interface{}) {
m = make(map[interface{}]interface{})
nom.Iter(func(key, value types.Value) (stop bool) {
m[valueInterface(key)] = valueInterface(value)
return
})
return
}

703
marshal/unmarshal_test.go
View File

@@ -1,703 +0,0 @@
// Modified from golang's encoding/json/decode_test.go at 80e6d638bf309181eadcb3fecbe99d2d8518e364.
package marshal
import (
"bytes"
"fmt"
"image"
"io"
"io/ioutil"
"net"
"reflect"
"strings"
"testing"
"github.com/attic-labs/noms/Godeps/_workspace/src/github.com/stretchr/testify/assert"
"github.com/attic-labs/noms/d"
"github.com/attic-labs/noms/ref"
"github.com/attic-labs/noms/types"
)
type T struct {
X string
Y int
Z int `noms:"-"`
}
type U struct {
Alphabet string `noms:"alpha"`
}
type tx struct {
x int
}
// Test data structures for anonymous fields.
type Point struct {
Z int
}
type Top struct {
Level0 int
Embed0
*Embed0a
*Embed0b `noms:"e,omitempty"` // treated as named
Embed0c `noms:"-"` // ignored
Loop
Embed0p // has Point with X, Y, used
Embed0q // has Point with Z, used
}
type Embed0 struct {
Level1a int // overridden by Embed0a's Level1a with noms tag
Level1b int // used because Embed0a's Level1b is renamed
Level1c int // used because Embed0a's Level1c is ignored
Level1d int // annihilated by Embed0a's Level1d
Level1e int `noms:"x"` // annihilated by Embed0a.Level1f
}
type Embed0a struct {
Level1a int `noms:"Level1a,omitempty"`
Level1b int `noms:"LEVEL1B,omitempty"`
Level1c int `noms:"-"`
Level1d int // annihilated by Embed0's Level1d
Level1f int `noms:"x"` // annihilated by Embed0's Level1e
}
type Embed0b Embed0
type Embed0c Embed0
type Embed0p struct {
image.Point
}
type Embed0q struct {
Point
}
type Loop struct {
Loop1 int `noms:",omitempty"`
Loop2 int `noms:",omitempty"`
*Loop
}
// From reflect test:
// The X in S6 and S7 annihilate, but they also block the X in S8.S9.
type S5 struct {
S6
S7
S8
}
type S6 struct {
X int
}
type S7 S6
type S8 struct {
S9
}
type S9 struct {
X int
Y int
}
// From reflect test:
// The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9.
type S10 struct {
S11
S12
S13
}
type S11 struct {
S6
}
type S12 struct {
S6
}
type S13 struct {
S8
}
type unmarshalTest struct {
in types.Value
ptr interface{}
out interface{}
err string
}
type Ambig struct {
// Given "hello", the first match should win.
First int `noms:"HELLO"`
Second int `noms:"Hello"`
}
type XYZ struct {
X interface{}
Y interface{}
Z interface{}
}
var unmarshalTests = []unmarshalTest{
// basic types
{in: types.Bool(true), ptr: new(bool), out: true},
{in: types.Int32(1), ptr: new(int32), out: 1},
{in: types.Int8(1), ptr: new(int8), out: int8(1)},
{in: types.Float64(1.2), ptr: new(float64), out: 1.2},
{in: types.Int16(-5), ptr: new(int16), out: int16(-5)},
{in: types.Float64(2), ptr: new(interface{}), out: float64(2.0)},
{in: types.NewString("a\u1234"), ptr: new(string), out: "a\u1234"},
{in: types.NewString("http://"), ptr: new(string), out: "http://"},
{in: types.NewMap(
types.NewString("X"), types.NewList(types.Int16(1)),
types.NewString("Y"), types.Int32(4)),
ptr: new(T), out: T{Y: 4}, err: invalidTypeMsg("List", reflect.TypeOf(""))},
{in: strIntMap(si{"x", 1}), ptr: new(tx), out: tx{}},
// Z has a "-" tag.
{in: strIntMap(si{"Y", 1}, si{"Z", 2}), ptr: new(T), out: T{Y: 1}},
// map tests
{in: strStrMap("alpha", "abc", "alphabet", "xyz"), ptr: new(U), out: U{Alphabet: "abc"}},
{in: strStrMap("alpha", "abc"), ptr: new(U), out: U{Alphabet: "abc"}},
{in: strStrMap("alphabet", "xyz"), ptr: new(U), out: U{}},
// array tests
{in: list(1, 2, 3), ptr: new([3]int), out: [3]int{1, 2, 3}},
{in: list(1, 2, 3), ptr: new([1]int), out: [1]int{1}, err: "list is too large"},
{in: list(1, 2, 3), ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}},
// blob tests
{in: blob(6, 7, 8), ptr: &[]byte{}, out: []byte{6, 7, 8}},
// ref tests
{
in: types.NewRef(ref.Parse("sha1-ffffffffffffffffffffffffffffffffffffffff")),
ptr: new(string),
out: "sha1-" + strings.Repeat("f", 40),
},
{
in: types.NewRef(ref.Parse("sha1-ffffffffffffffffffffffffffffffffffffffff")),
ptr: &[]byte{},
out: byteSlice(0xff, len(ref.Sha1Digest{})),
},
// empty array to interface test
{in: types.NewList(), ptr: new([]interface{}), out: []interface{}{}},
{in: types.NewMap(types.NewString("T"), types.NewList()), ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}},
// composite tests. allNomsValue is in marshal_test.go
{in: allNomsValue, ptr: new(All), out: allValueUnmarshal},
{in: allNomsValue, ptr: new(*All), out: &allValueUnmarshal},
// Overwriting of data.
// This is what encoding/json does.
{in: types.NewList(types.Int32(2)), ptr: sliceAddr([]int32{1}), out: []int32{2}},
{in: strIntMap(si{"key", 2}), ptr: mapAddr(map[string]int32{"old": 0, "key": 1}), out: map[string]int32{"key": 2}},
// embedded structs
// TODO: The ordering of the fields is based on the map iteration order. BUG 396
// {
// in: marshaledEmbedsPlus,
// ptr: new(Top),
// out: Top{
// Level0: 1,
// Embed0: Embed0{
// Level1b: 2,
// Level1c: 3,
// },
// Embed0a: &Embed0a{
// Level1a: 5,
// Level1b: 6,
// },
// Embed0b: &Embed0b{
// Level1a: 8,
// Level1b: 9,
// Level1c: 10,
// Level1d: 11,
// Level1e: 12,
// },
// Loop: Loop{
// Loop1: 13,
// Loop2: 14,
// },
// Embed0p: Embed0p{
// Point: image.Point{X: 15, Y: 16},
// },
// Embed0q: Embed0q{
// Point: Point{Z: 17},
// },
// },
// },
// TODO: The ordering of the fields is based on the map iteration order. BUG 396
// {
// in: types.NewMap(types.NewString("hello"), types.Int32(1)),
// ptr: new(Ambig),
// out: Ambig{First: 1},
// },
{
in: types.NewMap(types.NewString("X"), types.Int32(1), types.NewString("Y"), types.Int32(2)),
ptr: new(S5),
out: S5{S8: S8{S9: S9{Y: 2}}},
},
{
in: types.NewMap(types.NewString("X"), types.Int32(1), types.NewString("Y"), types.Int32(2)),
ptr: new(S10),
out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}},
},
}
// marshaledEmbeds (from marshal_test.go) plus some unexported fields.
// BUG 396
// var marshaledEmbedsPlus = func() types.Map {
// return marshaledEmbeds.Set(types.NewString("x"), types.Int32(4))
// }()
// allValue (from marshal_test.go) with some fields overridden to match how we handle nil in various cases.
var allValueUnmarshal = func() All {
local := allValue
local.MapP = map[string]*Small{
"19": {Tag: "tag19"},
// Note: We skip nil fields in maps.
}
// Note: we marshal nil map values, nil slices and nil slice entries entries as empty slices, maps, etc.
local.NilMap = map[string]Small{}
local.SliceP = []*Small{{Tag: "tag22"}, {Tag: "tag23"}}
local.NilSlice = []Small{}
// Note: Noms sets are marshaled from/to map[interface{}]bool, and only non-nil keys that map to true are included.
local.Set = map[Small]bool{Small{Tag: "tag33"}: true}
local.NilSet = map[Small]bool{}
return local
}()
func TestUnmarshal(t *testing.T) {
assert := assert.New(t)
for i, tt := range unmarshalTests {
// v = new(right-type)
v := reflect.New(reflect.TypeOf(tt.ptr).Elem())
err := d.Try(func() { Unmarshal(tt.in, v.Interface()) })
if tt.err != "" {
if assert.NotNil(err) {
assert.Contains(err.Error(), tt.err, "#%d: %v not in %s", i, err, tt.err)
}
continue
}
assert.NoError(err, "error in test #%d", i)
assert.EqualValues(tt.out, v.Elem().Interface())
// Check round trip.
nom := Marshal(v.Interface())
vv := reflect.New(reflect.TypeOf(tt.ptr).Elem())
Unmarshal(nom, vv.Interface())
assert.EqualValues(v.Elem().Interface(), vv.Elem().Interface())
}
}
var unmarshalAsRefTests = []types.Value{
blob(0, 1, 2),
list(3, 4, 5),
types.NewSet(types.Int8(6), types.Int8(7), types.Int8(8)),
strStrMap("9", "10", "11", "12"),
types.NewString("13"),
types.UInt64(14),
}
func TestUnmarshalAsRef(t *testing.T) {
assert := assert.New(t)
for _, input := range unmarshalAsRefTests {
expected := input.Ref()
target := ref.Ref{}
Unmarshal(input, &target)
assert.EqualValues(expected, target)
// Check round trip.
nom := Marshal(target)
newTarget := ref.Ref{}
Unmarshal(nom, &newTarget)
assert.EqualValues(target, newTarget)
}
}
func TestUnmarshalSetP(t *testing.T) {
assert := assert.New(t)
expected := map[*Small]bool{
&Small{Tag: "tag"}: true,
}
set := types.NewSet(types.NewMap(types.NewString("Tag"), types.NewString("tag")))
target := map[*Small]bool{}
findValueInMapKeys := func(tk *Small, m map[*Small]bool) (found bool) {
for k := range m {
found = *k == *tk
}
return
}
Unmarshal(set, &target)
if !assert.Len(target, len(expected)) {
return
}
for tk := range target {
assert.True(findValueInMapKeys(tk, expected))
}
// Check round trip.
nom := Marshal(target)
newTarget := map[*Small]bool{}
Unmarshal(nom, &newTarget)
for ntk := range newTarget {
assert.True(findValueInMapKeys(ntk, target))
}
}
func TestUnmarshalBlobIntoWriter(t *testing.T) {
assert := assert.New(t)
expected := []byte("abc")
blob := blob(expected...)
target := &bytes.Buffer{}
Unmarshal(blob, &target)
targetBytes := target.Bytes()
assert.EqualValues(expected, targetBytes)
// Check round trip.
nom := Marshal(target)
newTarget := &bytes.Buffer{}
Unmarshal(nom, &newTarget)
assert.EqualValues(targetBytes, newTarget.Bytes())
}
func TestUnmarshalBlobIntoWriterPtr(t *testing.T) {
assert := assert.New(t)
expected := []byte("abc")
blob := blob(expected...)
target := &bytes.Buffer{}
targetP := &target
Unmarshal(blob, &targetP)
targetBytes := target.Bytes()
assert.EqualValues(expected, targetBytes)
}
// Helpers for building up unmarshalTests
func sliceAddr(x []int32) *[]int32 { return &x }
func mapAddr(x map[string]int32) *map[string]int32 { return &x }
func byteSlice(b byte, times int) (out []byte) {
out = make([]byte, times)
for i := range out {
out[i] = b
}
return
}
func blob(b ...byte) types.Blob {
return types.NewMemoryBlob(bytes.NewReader(b))
}
func list(l ...int) types.List {
out := make([]types.Value, len(l))
for i, e := range l {
out[i] = types.Int32(e)
}
return types.NewList(out...)
}
func strStrMap(s ...string) types.Map {
out := make([]types.Value, len(s))
for i, e := range s {
out[i] = types.NewString(e)
}
return types.NewMap(out...)
}
func strIntMap(pairs ...si) types.Map {
out := make([]types.Value, 0, 2*len(pairs))
for _, e := range pairs {
out = append(out, types.NewString(e.k))
out = append(out, types.Int32(e.v))
}
return types.NewMap(out...)
}
type si struct {
k string
v int32
}
/*
TODO(cmasone): Figure out how to generate a random, large noms Value so we can do a version of this that makes sense for us.
func TestUnmarshalMarshal(t *testing.T) {
initBig()
var v interface{}
if err := Unmarshal(jsonBig, &v); err != nil {
t.Fatalf("Unmarshal: %v", err)
}
b:=Marshal(v)
if err != nil {
t.Fatalf("Marshal: %v", err)
}
if !bytes.Equal(jsonBig, b) {
t.Errorf("Marshal jsonBig")
printDiff(t, b, jsonBig)
return
}
}
*/
func TestLargeByteSlice(t *testing.T) {
s0 := make([]byte, 2000)
for i := range s0 {
s0[i] = byte(i)
}
b := Marshal(s0)
var s1 []byte
Unmarshal(b, &s1)
if !assert.Equal(t, s0, s1, "Marshal large byte slice") {
printDiff(t, s0, s1)
}
}
func printDiff(t *testing.T, a, b []byte) {
for i := 0; ; i++ {
if i >= len(a) || i >= len(b) || a[i] != b[i] {
j := i - 10
if j < 0 {
j = 0
}
t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
return
}
}
}
func trim(b []byte) []byte {
if len(b) > 20 {
return b[0:20]
}
return b
}
type Xint struct {
X int
}
func TestUnmarshalInterface(t *testing.T) {
var xint Xint
var i interface{} = &xint
Unmarshal(strIntMap(si{"X", 1}), &i)
if xint.X != 1 {
t.Fatalf("Did not write to xint")
}
}
func TestUnmarshalPtrPtr(t *testing.T) {
var xint Xint
pxint := &xint
Unmarshal(strIntMap(si{"X", 1}), &pxint)
if xint.X != 1 {
t.Fatalf("Did not write to xint")
}
}
func intp(x int) *int {
p := new(int)
*p = x
return p
}
func intpp(x *int) **int {
pp := new(*int)
*pp = x
return pp
}
var interfaceSetTests = []struct {
pre interface{}
nom types.Value
post interface{}
}{
{"foo", types.NewString("bar"), "bar"},
{nil, types.NewString("bar"), "bar"},
{"foo", types.Int32(2), int32(2)},
{"foo", types.Bool(true), true},
{"foo", list(1, 2, 3), []interface{}{int32(1), int32(2), int32(3)}},
{"foo", strStrMap("4", "5"), map[interface{}]interface{}{"4": "5"}},
{"foo", types.NewSet(types.Int8(6)), map[interface{}]bool{int8(6): true}},
{intp(1), types.Int64(7), intp(7)},
{intpp(intp(1)), types.Int64(8), intpp(intp(8))},
}
func TestInterfaceSet(t *testing.T) {
for _, tt := range interfaceSetTests {
native := struct{ X interface{} }{tt.pre}
nom := types.NewMap(types.NewString("X"), tt.nom)
Unmarshal(nom, &native)
assert.EqualValues(t, tt.post, native.X, "Unmarshal %v over %#v: X=%#v, want %#v", nom, tt.pre, native.X, tt.post)
}
}
// TODO: enable blobs to be unmarshaled to *io.Reader, then add test cases here (BUG 160)
var blobInterfaceSetTests = []struct {
pre interface{}
nom types.Blob
post []byte
}{
{"foo", blob(1, 2, 3), []byte{1, 2, 3}},
{bytes.NewBuffer([]byte{7}), blob(4, 5, 6), []byte{7, 4, 5, 6}},
}
func TestBlobInterfaceSet(t *testing.T) {
for _, tt := range blobInterfaceSetTests {
native := struct{ X interface{} }{tt.pre}
nom := types.NewMap(types.NewString("X"), tt.nom)
Unmarshal(nom, &native)
bytes, err := ioutil.ReadAll(native.X.(io.Reader))
if assert.NoError(t, err) {
assert.Equal(t, tt.post, bytes)
}
}
}
func TestStringKind(t *testing.T) {
assert := assert.New(t)
type stringKind string
var m1, m2 map[stringKind]int
m1 = map[stringKind]int{
"foo": 42,
}
data := Marshal(m1)
Unmarshal(data, &m2)
assert.EqualValues(m1, m2)
}
// Custom types with []byte as underlying type could not be marshalled
// and then unmarshalled.
// Issue 8962.
func TestByteKind(t *testing.T) {
assert := assert.New(t)
type byteKind []byte
a := byteKind("hello")
data := Marshal(a)
var b byteKind
Unmarshal(data, &b)
assert.EqualValues(a, b)
}
var decodeTypeErrorTests = []struct {
dest interface{}
src types.Value
}{
{new(string), types.NewMap(types.NewString("guy"), types.NewString("friend"))}, // issue 4628.
{new(error), types.NewMap()}, // issue 4222
{new(error), types.NewList()},
{new(error), types.NewString("")},
{new(error), types.UInt64(123)},
{new(error), types.Bool(true)},
}
func TestUnmarshalTypeError(t *testing.T) {
for _, item := range decodeTypeErrorTests {
err := d.Try(func() { Unmarshal(item.src, item.dest) })
assert.IsType(t, d.UsageError{}, err, "expected type error for Unmarshal(%v, type %T): got %T (%v)",
item.src, item.dest, err, err)
}
}
type unexportedFields struct {
Name string
m map[string]interface{} `noms:"-"`
m2 map[string]interface{} `noms:"abcd"`
}
func TestUnmarshalUnexported(t *testing.T) {
input := types.NewMap(
types.NewString("Name"), types.NewString("Bob"),
types.NewString("m"), types.NewMap(types.NewString("x"), types.Int64(123)),
types.NewString("m2"), types.NewMap(types.NewString("y"), types.Int64(456)),
types.NewString("abcd"), types.NewMap(types.NewString("z"), types.Int64(789)))
expected := &unexportedFields{Name: "Bob"}
out := &unexportedFields{}
Unmarshal(input, out)
assert.EqualValues(t, expected, out)
}
// Test semantics of pre-filled struct fields and pre-filled map fields.
// Issue 4900.
func TestPrefilled(t *testing.T) {
ptrToMap := func(m map[string]interface{}) *map[string]interface{} { return &m }
// Values here change, cannot reuse table across runs.
var prefillTests = []struct {
in types.Map
ptr interface{}
out interface{}
}{
{
in: strIntMap(si{"X", 1}, si{"Y", 2}),
ptr: &XYZ{X: float32(3), Y: int16(4), Z: 1.5},
out: &XYZ{X: int32(1), Y: int32(2), Z: 1.5},
},
{
in: strIntMap(si{"X", 1}, si{"Y", 2}),
ptr: ptrToMap(map[string]interface{}{"X": float32(3), "Y": int16(4), "Z": 1.5}),
out: ptrToMap(map[string]interface{}{"X": int32(1), "Y": int32(2), "Z": 1.5}),
},
}
for _, tt := range prefillTests {
ptrstr := fmt.Sprintf("%v", tt.ptr)
Unmarshal(tt.in, tt.ptr) // tt.ptr edited here
assert.EqualValues(t, tt.ptr, tt.out, "Target should have been overwritten, was originally %s", ptrstr)
}
}
var invalidUnmarshalTests = []struct {
v interface{}
want string
}{
{nil, "noms: Unmarshal(nil)"},
{struct{}{}, "noms: Unmarshal(non-pointer struct {})"},
{(*int)(nil), "noms: Unmarshal(nil *int)"},
{new(net.IP), "noms: cannot unmarshal noms String into Go value of type *net.IP"},
}
func TestInvalidUnmarshal(t *testing.T) {
nom := types.NewString("hello")
for _, tt := range invalidUnmarshalTests {
if err := d.Try(func() { Unmarshal(nom, tt.v) }); assert.Error(t, err) {
assert.Contains(t, err.Error(), tt.want)
} else {
assert.Fail(t, "Expecting error!")
}
}
}