diff --git a/go/types/codec.go b/go/types/codec.go index 356e33a6fc..e136984fd3 100644 --- a/go/types/codec.go +++ b/go/types/codec.go @@ -144,8 +144,6 @@ func (b *binaryNomsReader) readString() string { return v } -var createCount = uint64(0) - // Note: It's somewhat of a layering violation that a nomsReaders knows about a TypeCache. The reason why the code is structured this way is that the go compiler can stack-allocate the string which is created from the byte slice, which is a fairly large perf gain. func (b *binaryNomsReader) readIdent(tc *TypeCache) uint32 { size := b.readUint32() diff --git a/go/types/encode_human_readable_test.go b/go/types/encode_human_readable_test.go index 94905280c8..a069274674 100644 --- a/go/types/encode_human_readable_test.go +++ b/go/types/encode_human_readable_test.go @@ -360,7 +360,7 @@ func TestRecursiveStruct(t *testing.T) { })`, d) } -func TestUnserolvedRecursiveStruct(t *testing.T) { +func TestUnresolvedRecursiveStruct(t *testing.T) { // struct A { // a: A // b: Cycle<1> (unresolved) diff --git a/go/types/type.go b/go/types/type.go index 635e25408d..050870817e 100644 --- a/go/types/type.go +++ b/go/types/type.go @@ -30,7 +30,11 @@ type Type struct { const initialTypeBufferSize = 128 func newType(desc TypeDesc, id uint32) *Type { - t := &Type{desc, &hash.Hash{}, id, nil} + return &Type{desc, &hash.Hash{}, id, nil} +} + +func buildType(desc TypeDesc, id uint32) *Type { + t := newType(desc, id) if !t.HasUnresolvedCycle() { serializeType(t) } @@ -133,10 +137,6 @@ func MakePrimitiveType(k NomsKind) *Type { return nil } -func makePrimitiveType(k NomsKind) *Type { - return newType(PrimitiveDesc(k), uint32(k)) -} - func MakePrimitiveTypeByString(p string) *Type { switch p { case "Bool": diff --git a/go/types/type_cache.go b/go/types/type_cache.go index 580a8e1546..ec815e6f83 100644 --- a/go/types/type_cache.go +++ b/go/types/type_cache.go @@ -15,13 +15,16 @@ import ( type TypeCache struct { identTable *identTable trieRoots map[NomsKind]*typeTrie - typeBytes map[uint32][]byte nextId uint32 mu *sync.Mutex } var staticTypeCache = NewTypeCache() +func makePrimitiveType(k NomsKind) *Type { + return buildType(PrimitiveDesc(k), uint32(k)) +} + var BoolType = makePrimitiveType(BoolKind) var NumberType = makePrimitiveType(NumberKind) var StringType = makePrimitiveType(StringKind) @@ -41,7 +44,6 @@ func NewTypeCache() *TypeCache { CycleKind: newTypeTrie(), UnionKind: newTypeTrie(), }, - map[uint32][]byte{}, 256, // The first 255 type ids are reserved for the 8bit space of NomsKinds. &sync.Mutex{}, } @@ -68,7 +70,7 @@ func (tc *TypeCache) getCompoundType(kind NomsKind, elemTypes ...*Type) *Type { } if trie.t == nil { - trie.t = newType(CompoundDesc{kind, elemTypes}, tc.nextTypeId()) + trie.t = buildType(CompoundDesc{kind, elemTypes}, tc.nextTypeId()) } return trie.t @@ -93,7 +95,7 @@ func (tc *TypeCache) makeStructType(name string, fieldNames []string, fieldTypes i++ } - t := newType(StructDesc{name, fs}, 0) + t := buildType(StructDesc{name, fs}, 0) if t.serialization == nil { // HasUnresolvedCycle t, _ = toUnresolvedType(t, tc, -1, nil) @@ -122,8 +124,7 @@ func indexOfType(t *Type, tl []*Type) (uint32, bool) { func toUnresolvedType(t *Type, tc *TypeCache, level int, parentStructTypes []*Type) (*Type, bool) { i, found := indexOfType(t, parentStructTypes) if found { - cycle := CycleDesc(uint32(len(parentStructTypes)) - i - 1) - return &Type{cycle, &hash.Hash{}, 0, nil}, true // This type is just a placeholder. It doesn't need an id + return newType(CycleDesc(uint32(len(parentStructTypes))-i-1), 0), true // This type is just a placeholder. It doesn't need an id } switch desc := t.Desc.(type) { @@ -140,7 +141,7 @@ func toUnresolvedType(t *Type, tc *TypeCache, level int, parentStructTypes []*Ty return t, false } - return &Type{CompoundDesc{t.Kind(), ts}, &hash.Hash{}, tc.nextTypeId(), nil}, true + return newType(CompoundDesc{t.Kind(), ts}, tc.nextTypeId()), true case StructDesc: fs := make(fieldSlice, len(desc.fields)) didChange := false @@ -155,7 +156,7 @@ func toUnresolvedType(t *Type, tc *TypeCache, level int, parentStructTypes []*Ty return t, false } - return &Type{StructDesc{desc.Name, fs}, &hash.Hash{}, tc.nextTypeId(), nil}, true + return newType(StructDesc{desc.Name, fs}, tc.nextTypeId()), true case CycleDesc: cycleLevel := int(desc) return t, cycleLevel <= level // Only cycles which can be resolved in the current struct. @@ -224,11 +225,11 @@ func (tc *TypeCache) makeUnionType(elemTypes ...*Type) *Type { return tc.getCompoundType(UnionKind, ts...) } -func (tc *TypeCache) getCyclicType(level uint32) *Type { +func (tc *TypeCache) getCycleType(level uint32) *Type { trie := tc.trieRoots[CycleKind].Traverse(level) if trie.t == nil { - trie.t = newType(CycleDesc(level), tc.nextTypeId()) + trie.t = buildType(CycleDesc(level), tc.nextTypeId()) } return trie.t @@ -292,7 +293,7 @@ func MakeUnionType(elemTypes ...*Type) *Type { func MakeCycleType(level uint32) *Type { staticTypeCache.Lock() defer staticTypeCache.Unlock() - return staticTypeCache.getCyclicType(level) + return staticTypeCache.getCycleType(level) } // All types in noms are created in a deterministic order. A typeTrie stores types within a typeCache and allows construction of a prexisting type to return the already existing one rather than allocate a new one. @@ -306,13 +307,12 @@ func newTypeTrie() *typeTrie { } func (tct *typeTrie) Traverse(typeId uint32) *typeTrie { - if t, ok := tct.entries[typeId]; ok { - return t + next, ok := tct.entries[typeId] + if !ok { + // Insert edge + next = newTypeTrie() + tct.entries[typeId] = next } - - // Insert edge - next := newTypeTrie() - tct.entries[typeId] = next return next } @@ -326,12 +326,12 @@ func newIdentTable() *identTable { } func (it *identTable) GetId(ident string) uint32 { - if id, ok := it.entries[ident]; ok { - return id + id, ok := it.entries[ident] + if !ok { + id = it.nextId + it.nextId++ + it.entries[ident] = id } - id := it.nextId - it.nextId++ - it.entries[ident] = id return id } diff --git a/go/types/type_cache_test.go b/go/types/type_cache_test.go index ac2c0e512b..9359416c40 100644 --- a/go/types/type_cache_test.go +++ b/go/types/type_cache_test.go @@ -68,10 +68,15 @@ func TestTypeCacheRef(t *testing.T) { lst := MakeRefType(StringType) lnt := MakeRefType(NumberType) assert.False(lst == lnt) - assert.NotNil(lnt.serialization) lst2 := MakeRefType(StringType) assert.True(lst == lst2) + + lnt2 := MakeRefType(NumberType) + assert.True(lnt == lnt2) + + lbt3 := MakeRefType(BoolType) + assert.True(lbt == lbt3) } func TestTypeCacheStruct(t *testing.T) { diff --git a/go/types/value_decoder.go b/go/types/value_decoder.go index 0adfa9c7f8..000f90159e 100644 --- a/go/types/value_decoder.go +++ b/go/types/value_decoder.go @@ -53,7 +53,7 @@ func (r *valueDecoder) readType() *Type { } return r.tc.getCompoundType(UnionKind, elemTypes...) case CycleKind: - return r.tc.getCyclicType(r.readUint32()) + return r.tc.getCycleType(r.readUint32()) } d.Chk.True(IsPrimitiveKind(k)) diff --git a/js/src/assert-type-test.js b/js/src/assert-type-test.js index 6b8db9524d..0e25d2ba8f 100644 --- a/js/src/assert-type-test.js +++ b/js/src/assert-type-test.js @@ -16,16 +16,13 @@ import type {Type} from './type.js'; import { blobType, boolType, - listOfValueType, makeListType, makeMapType, makeRefType, makeSetType, makeStructType, makeUnionType, - mapOfValueType, numberType, - setOfValueType, stringType, typeType, valueType, @@ -89,7 +86,7 @@ suite('validate type', () => { assertSubtype(listOfNumberType, l); assertAll(listOfNumberType, l); - assertSubtype(listOfValueType, l); + assertSubtype(makeListType(valueType), l); }); test('map', () => { @@ -98,7 +95,7 @@ suite('validate type', () => { assertSubtype(mapOfNumberToStringType, m); assertAll(mapOfNumberToStringType, m); - assertSubtype(mapOfValueType, m); + assertSubtype(makeMapType(valueType, valueType), m); }); test('set', () => { @@ -107,7 +104,7 @@ suite('validate type', () => { assertSubtype(setOfNumberType, s); assertAll(setOfNumberType, s); - assertSubtype(setOfValueType, s); + assertSubtype(makeSetType(valueType), s); }); test('type', () => { @@ -233,11 +230,10 @@ suite('validate type', () => { const t11 = makeStructType('Commit', ['parents', 'value'], [ + makeSetType(makeRefType(t1)), numberType, - numberType, // placeholder ] ); - t11.desc.setField('parents', makeSetType(makeRefType(t11))); assertSubtype(t11, c1); const c2 = newStruct('Commit', { diff --git a/js/src/blob-test.js b/js/src/blob-test.js index 0023568991..bc179831b2 100644 --- a/js/src/blob-test.js +++ b/js/src/blob-test.js @@ -4,7 +4,7 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -import {blobType, refOfBlobType} from './type.js'; +import {blobType, makeRefType} from './type.js'; import {assert} from 'chai'; import Blob, {BlobReader, BlobWriter} from './blob.js'; import {suite, test, setup, teardown} from 'mocha'; @@ -118,7 +118,7 @@ suite('Blob', () => { assertValueHash(expectHashStr, blob); assertValueType(blobType, blob); assert.strictEqual(length, blob.length); - assertChunkCountAndType(expectChunkCount, refOfBlobType, blob); + assertChunkCountAndType(expectChunkCount, makeRefType(blobType), blob); await testRoundTripAndValidate(blob, async(b2) => { await assertReadFull(buff, b2.getReader()); diff --git a/js/src/codec.js b/js/src/codec.js index 2ced2a4bab..88f6804cbf 100644 --- a/js/src/codec.js +++ b/js/src/codec.js @@ -4,18 +4,20 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 +import * as Bytes from './bytes.js'; import Chunk from './chunk.js'; import Hash, {sha1Size} from './hash.js'; import ValueDecoder from './value-decoder.js'; import ValueEncoder from './value-encoder.js'; -import {invariant} from './assert.js'; +import svarint from 'signed-varint'; +import type Value from './value.js'; +import type {Type} from './type.js'; +import type {ValueReader, ValueWriter} from './value-store.js'; +import {default as TypeCache, staticTypeCache} from './type-cache.js'; +import {floatToIntExp, intExpToFloat} from './number-util.js'; +import {invariant, notNull} from './assert.js'; import {setEncodeValue} from './get-hash.js'; import {setHash, ValueBase} from './value.js'; -import type Value from './value.js'; -import type {ValueReader, ValueWriter} from './value-store.js'; -import * as Bytes from './bytes.js'; -import {floatToIntExp, intExpToFloat} from './number-util.js'; -import svarint from 'signed-varint'; export function encodeValue(v: Value, vw: ?ValueWriter): Chunk { const w = new BinaryNomsWriter(); @@ -33,7 +35,7 @@ setEncodeValue(encodeValue); export function decodeValue(chunk: Chunk, vr: ValueReader): Value { const data = chunk.data; - const dec = new ValueDecoder(new BinaryNomsReader(data), vr); + const dec = new ValueDecoder(new BinaryNomsReader(data), vr, staticTypeCache); const v = dec.readValue(); if (v instanceof ValueBase) { @@ -43,11 +45,21 @@ export function decodeValue(chunk: Chunk, vr: ValueReader): Value { return v; } +function ensureTypeSerialization(t: Type) { + if (!t.serialization) { + const w = new BinaryNomsWriter(); + const enc = new ValueEncoder(w, null); + enc.writeType(t, []); + t.serialization = w.data; + } +} const maxUInt32 = Math.pow(2, 32); const littleEndian = true; export interface NomsReader { + pos(): number; + seek(pos: number): void; readBytes(): Uint8Array; readUint8(): number; readUint32(): number; @@ -55,6 +67,7 @@ export interface NomsReader { readNumber(): number; readBool(): boolean; readString(): string; + readIdent(tc: TypeCache): number; readHash(): Hash; } @@ -67,6 +80,7 @@ export interface NomsWriter { writeBool(v:boolean): void; writeString(v: string): void; writeHash(h: Hash): void; + appendType(t: Type): void; } export class BinaryNomsReader { @@ -80,6 +94,14 @@ export class BinaryNomsReader { this.offset = 0; } + pos(): number { + return this.offset; + } + + seek(pos: number): void { + this.offset = pos; + } + readBytes(): Uint8Array { const size = this.readUint32(); // Make a copy of the buffer to return @@ -129,6 +151,16 @@ export class BinaryNomsReader { return str; } + readIdent(tc: TypeCache): number { + const str = this.readString(); // TODO: Figure out how to do this without allocating. + let id = tc.identTable.entries.get(str); + if (id === undefined) { + id = tc.identTable.getId(str); + } + + return id; + } + readHash(): Hash { // Make a copy of the data. const digest = Bytes.slice(this.buff, this.offset, this.offset + sha1Size); @@ -225,4 +257,16 @@ export class BinaryNomsWriter { Bytes.copy(h.digest, this.buff, this.offset); this.offset += sha1Size; } + + appendType(t: Type): void { + // Note: The JS & Go impls differ here. The Go impl eagerly serializes types as they are + // constructed. The JS does it lazily so as to avoid cyclic package dependencies. + ensureTypeSerialization(t); + const data = notNull(t.serialization); + const size = data.byteLength; + this.ensureCapacity(size); + + Bytes.copy(data, this.buff, this.offset); + this.offset += size; + } } diff --git a/js/src/commit.js b/js/src/commit.js index 09e79d4e20..bae02b1578 100644 --- a/js/src/commit.js +++ b/js/src/commit.js @@ -5,16 +5,28 @@ // http://www.apache.org/licenses/LICENSE-2.0 import {invariant} from './assert.js'; -import {getDatasTypes} from './database.js'; import Struct from './struct.js'; import type Value from './value.js'; import type Ref from './ref.js'; import Set from './set.js'; +import { + makeCycleType, + makeRefType, + makeStructType, + makeSetType, + valueType, +} from './type.js'; +export const commitType = makeStructType('Commit', + ['parents', 'value'], + [ + makeSetType(makeRefType(makeCycleType(0))), + valueType, + ] +); export default class Commit extends Struct { constructor(value: T, parents: Set> = new Set()) { - const {commitType} = getDatasTypes(); super(commitType, [parents, value]); } diff --git a/js/src/database.js b/js/src/database.js index 30c198571b..cc518225ea 100644 --- a/js/src/database.js +++ b/js/src/database.js @@ -12,54 +12,9 @@ import type Value from './value.js'; import type {RootTracker} from './chunk-store.js'; import ValueStore from './value-store.js'; import type {BatchStore} from './batch-store.js'; -import { - makeRefType, - makeStructType, - makeSetType, - makeMapType, - Type, - stringType, - valueType, -} from './type.js'; import Commit from './commit.js'; import {equals} from './compare.js'; -type DatasTypes = { - commitType: Type, - commitSetType: Type, - refOfCommitType: Type, - commitMapType: Type, -}; - -let datasTypes: DatasTypes; -export function getDatasTypes(): DatasTypes { - if (!datasTypes) { - // struct Commit { - // value: Value - // parents: Set> - // } - const commitType = makeStructType('Commit', - ['parents', 'value'], - [ - valueType, - valueType, - ] - ); - const refOfCommitType = makeRefType(commitType); - const commitSetType = makeSetType(refOfCommitType); - commitType.desc.setField('parents', commitSetType); - const commitMapType = makeMapType(stringType, refOfCommitType); - datasTypes = { - commitType, - refOfCommitType, - commitSetType, - commitMapType, - }; - } - - return datasTypes; -} - export default class Database { _vs: ValueStore; _rt: RootTracker; diff --git a/js/src/encode-human-readable-test.js b/js/src/encode-human-readable-test.js index 04c938f46d..73305c82dc 100644 --- a/js/src/encode-human-readable-test.js +++ b/js/src/encode-human-readable-test.js @@ -7,11 +7,13 @@ import {assert} from 'chai'; import {suite, test} from 'mocha'; +import {invariant, notNull} from './assert.js'; import {TypeWriter} from './encode-human-readable.js'; import { blobType, boolType, numberType, + makeCycleType, makeRefType, makeListType, makeMapType, @@ -19,7 +21,7 @@ import { makeStructType, makeUnionType, stringType, - valueType, + StructDesc, Type, } from './type.js'; @@ -86,23 +88,17 @@ suite('Encode human readable types', () => { const a = makeStructType('A', ['b', 'c', 'd'], [ - valueType, // placeholder - valueType, // placeholder - valueType, // placeholder + makeCycleType(0), + makeListType(makeCycleType(0)), + makeStructType('D', + ['e', 'f'], + [ + makeCycleType(0), + makeCycleType(1), + ] + ), ] ); - const d = makeStructType('D', - ['e', 'f'], - [ - valueType, // placeholder - a, - ] - ); - a.desc.setField('b', a); - a.desc.setField('d', d); - d.desc.setField('e', d); - d.desc.setField('f', a); - a.desc.setField('c', makeListType(a)); assertWriteType(`struct A { b: Cycle<0>, @@ -113,6 +109,9 @@ suite('Encode human readable types', () => { }, }`, a); + invariant(a.desc instanceof StructDesc); + const d = notNull(a.desc.getField('d')); + assertWriteType(`struct D { e: Cycle<0>, f: struct A { @@ -122,4 +121,24 @@ suite('Encode human readable types', () => { }, }`, d); }); + + test('recursive unresolved struct', () => { + // struct A { + // a: A + // b: Cycle<1> + // } + + const a = makeStructType('A', + ['a', 'b'], + [ + makeCycleType(0), + makeCycleType(1), + ] + ); + + assertWriteType(`struct A { + a: Cycle<0>, + b: Cycle<1>, +}`, a); + }); }); diff --git a/js/src/encode-human-readable.js b/js/src/encode-human-readable.js index 855ef9d408..fa06f59e96 100644 --- a/js/src/encode-human-readable.js +++ b/js/src/encode-human-readable.js @@ -4,7 +4,7 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -import {getTypeOfValue, CompoundDesc} from './type.js'; +import {getTypeOfValue, CompoundDesc, CycleDesc} from './type.js'; import type {Type} from './type.js'; import {Kind, kindToString} from './noms-kind.js'; import type {NomsKind} from './noms-kind.js'; @@ -109,6 +109,9 @@ export class TypeWriter { this._writeStructType(t, parentStructTypes); break; case Kind.Cycle: + invariant(t.desc instanceof CycleDesc); + this._w.write(`Cycle<${t.desc.level}>`); + break; default: throw new Error('unreachable'); } diff --git a/js/src/encoding-test.js b/js/src/encoding-test.js index c3f7a84f3a..ed982407e8 100644 --- a/js/src/encoding-test.js +++ b/js/src/encoding-test.js @@ -34,16 +34,18 @@ import { } from './meta-sequence.js'; import { boolType, + blobType, + makeCycleType, makeListType, makeMapType, makeRefType, makeSetType, makeStructType, numberType, - refOfBlobType, stringType, typeType, } from './type.js'; +import {staticTypeCache} from './type-cache.js'; function assertRoundTrips(v: Value) { const db = new TestDatabase(); @@ -121,6 +123,14 @@ suite('Encoding', () => { this.i = 0; } + pos(): number { + return this.i; + } + + seek(pos: number): void { + this.i = pos; + } + atEnd(): boolean { return this.i === this.a.length; } @@ -172,6 +182,16 @@ suite('Encoding', () => { return v; } + readIdent(tc: TypeCache): number { + const s = this.readString(); + let id = tc.identTable.entries.get(s); + if (id === undefined) { + id = tc.identTable.getId(s); + } + + return id; + } + readHash(): Hash { return Hash.parse(this.readString()); } @@ -221,6 +241,11 @@ suite('Encoding', () => { this.writeString(h.toString()); } + appendType(t: Type): void { + const enc = new ValueEncoder(this, null); + enc.writeType(t, []); + } + toArray(): any[] { return this.a; } @@ -246,7 +271,7 @@ suite('Encoding', () => { assert.deepEqual(encoding, w.toArray()); const r = new TestReader(encoding); - const dec = new ValueDecoder(r, null); + const dec = new ValueDecoder(r, null, staticTypeCache); const v2 = dec.readValue(); assert.isTrue(equals(v, v2)); } @@ -340,9 +365,9 @@ suite('Encoding', () => { uint8(RefKind), uint8(BlobKind), r3.toString(), uint64(33), uint8(NumberKind), float64(60), uint64(60), ], Blob.fromSequence(newBlobMetaSequence(null, [ - new MetaTuple(constructRef(refOfBlobType, r1, 11), new OrderedKey(20), 20, null), - new MetaTuple(constructRef(refOfBlobType, r2, 22), new OrderedKey(40), 40, null), - new MetaTuple(constructRef(refOfBlobType, r3, 33), new OrderedKey(60), 60, null), + new MetaTuple(constructRef(makeRefType(blobType), r1, 11), new OrderedKey(20), 20, null), + new MetaTuple(constructRef(makeRefType(blobType), r2, 22), new OrderedKey(40), 40, null), + new MetaTuple(constructRef(makeRefType(blobType), r3, 33), new OrderedKey(60), 60, null), ])) ); }); @@ -498,12 +523,10 @@ suite('Encoding', () => { const structType = makeStructType('A6', ['cs', 'v'], [ - numberType, // placeholder + makeListType(makeCycleType(0)), numberType, ] ); - const listType = makeListType(structType); - structType.desc.setField('cs', listType); assertEncoding([ uint8(StructKind), 'A6', uint32(2) /* len */, 'cs', uint8(ListKind), uint8(CycleKind), uint32(0), 'v', uint8(NumberKind), diff --git a/js/src/struct-test.js b/js/src/struct-test.js index c9524515f8..b4470d0877 100644 --- a/js/src/struct-test.js +++ b/js/src/struct-test.js @@ -16,11 +16,11 @@ import Struct, { import {assert} from 'chai'; import { boolType, + makeCycleType, makeListType, makeStructType, numberType, stringType, - valueType, } from './type.js'; import {suite, test} from 'mocha'; import {equals} from './compare.js'; @@ -126,13 +126,11 @@ suite('Struct', () => { ['b', 'l'], [ boolType, - valueType, // placeholder + makeListType(makeCycleType(0)), ] ); - const listType = makeListType(type); - type.desc.setField('l', listType); - const emptyList = new List([], listType); + const emptyList = new List([]); newStructWithType(type, [true, emptyList]); newStructWithType(type, [ diff --git a/js/src/type-cache-test.js b/js/src/type-cache-test.js new file mode 100644 index 0000000000..a70871aa77 --- /dev/null +++ b/js/src/type-cache-test.js @@ -0,0 +1,124 @@ +// @flow + +// Copyright 2016 Attic Labs, Inc. All rights reserved. +// Licensed under the Apache License, version 2.0: +// http://www.apache.org/licenses/LICENSE-2.0 + +import {assert} from 'chai'; +import {suite, test} from 'mocha'; +import { + boolType, + makeCycleType, + makeListType, + makeRefType, + makeSetType, + makeStructType, + makeUnionType, + numberType, + stringType, +} from './type.js'; + +suite('TypeCache', () => { + test('list', () => { + const lbt = makeListType(boolType); + const lbt2 = makeListType(boolType); + assert.strictEqual(lbt, lbt2); + + const lst = makeListType(stringType); + const lnt = makeListType(numberType); + assert.notEqual(lst, lnt); + + const lst2 = makeListType(stringType); + assert.strictEqual(lst, lst2); + + const lnt2 = makeListType(numberType); + assert.strictEqual(lnt, lnt2); + + const lbt3 = makeListType(boolType); + assert.strictEqual(lbt, lbt3); + }); + + test('set', () => { + const lbt = makeSetType(boolType); + const lbt2 = makeSetType(boolType); + assert.strictEqual(lbt, lbt2); + + const lst = makeSetType(stringType); + const lnt = makeSetType(numberType); + assert.notEqual(lst, lnt); + + const lst2 = makeSetType(stringType); + assert.strictEqual(lst, lst2); + + const lnt2 = makeSetType(numberType); + assert.strictEqual(lnt, lnt2); + + const lbt3 = makeSetType(boolType); + assert.strictEqual(lbt, lbt3); + }); + + test('ref', () => { + const lbt = makeRefType(boolType); + const lbt2 = makeRefType(boolType); + assert.strictEqual(lbt, lbt2); + + const lst = makeRefType(stringType); + const lnt = makeRefType(numberType); + assert.notEqual(lst, lnt); + + const lst2 = makeRefType(stringType); + assert.strictEqual(lst, lst2); + + const lnt2 = makeRefType(numberType); + assert.strictEqual(lnt, lnt2); + + const lbt3 = makeRefType(boolType); + assert.strictEqual(lbt, lbt3); + }); + + test('struct', () => { + const st = makeStructType('Foo', + ['bar', 'foo'], + [stringType, numberType] + ); + const st2 = makeStructType('Foo', + ['bar', 'foo'], + [stringType, numberType] + ); + + assert.strictEqual(st, st2); + }); + + test('union', () => { + let ut = makeUnionType([numberType]); + let ut2 = makeUnionType([numberType]); + assert.strictEqual(ut, ut2); + assert.strictEqual(ut2, numberType); + + ut = makeUnionType([numberType, stringType]); + ut2 = makeUnionType([stringType, numberType]); + assert.strictEqual(ut, ut2); + + ut = makeUnionType([stringType, boolType, numberType]); + ut2 = makeUnionType([numberType, stringType, boolType]); + assert.strictEqual(ut, ut2); + }); + + test('Cyclic Struct', () => { + const st = makeStructType('Foo', + ['foo'], + [ + makeRefType(makeCycleType(0)), + ]); + assert.isFalse(st.hasUnresolvedCycle([])); + assert.strictEqual(st, st.desc.fields[0].type.desc.elemTypes[0]); + + const st2 = makeStructType('Foo', + ['foo'], + [ + makeRefType(makeCycleType(0)), + ]); + assert.isFalse(st2.hasUnresolvedCycle([])); + assert.strictEqual(st, st2); + }); +}); diff --git a/js/src/type-cache.js b/js/src/type-cache.js new file mode 100644 index 0000000000..0a9f89649d --- /dev/null +++ b/js/src/type-cache.js @@ -0,0 +1,257 @@ +// @flow + +// Copyright 2016 The Noms Authors. All rights reserved. +// Licensed under the Apache License, version 2.0: +// http://www.apache.org/licenses/LICENSE-2.0 + +import type Hash from './hash.js'; +import type {NomsKind} from './noms-kind.js'; +import {Kind} from './noms-kind.js'; +import {CompoundDesc, CycleDesc, StructDesc, Type} from './type.js'; +import {compare} from './compare.js'; +import {notNull} from './assert.js'; + +class IdentTable { + entries: Map; + nextId: number; + + constructor() { + this.entries = new Map(); + this.nextId = 0; + } + + getId(ident: string): number { + let id = this.entries.get(ident); + if (id === undefined) { + id = this.nextId++; + this.entries.set(ident, id); + } + + return id; + } +} + +class TypeTrie { + t: Type; + entries: Map; + + constructor() { + this.entries = new Map(); + } + + traverse(typeId: number): TypeTrie { + let next = this.entries.get(typeId); + if (!next) { + // Insert edge + next = new TypeTrie(); + this.entries.set(typeId, next); + } + + return next; + } +} + +export default class TypeCache { + identTable: IdentTable; + trieRoots: Map; + nextId: number; + + constructor() { + this.identTable = new IdentTable(); + this.trieRoots = new Map(); + this.trieRoots.set(Kind.List, new TypeTrie()); + this.trieRoots.set(Kind.Set, new TypeTrie()); + this.trieRoots.set(Kind.Ref, new TypeTrie()); + this.trieRoots.set(Kind.Map, new TypeTrie()); + this.trieRoots.set(Kind.Struct, new TypeTrie()); + this.trieRoots.set(Kind.Cycle, new TypeTrie()); + this.trieRoots.set(Kind.Union, new TypeTrie()); + this.nextId = 256; // The first 255 type ids are reserved for the 8bit space of NomsKinds. + } + + nextTypeId(): number { + return this.nextId++; + } + + getCompoundType(kind: NomsKind, ...elemTypes: Type[]): Type { + let trie = notNull(this.trieRoots.get(kind)); + elemTypes.forEach(t => trie = notNull(trie).traverse(t.id)); + if (!notNull(trie).t) { + trie.t = new Type(new CompoundDesc(kind, elemTypes), this.nextTypeId()); + } + + return trie.t; + } + + makeStructType(name: string, fieldNames: string[], fieldTypes: Type[]): Type { + if (fieldNames.length !== fieldTypes.length) { + throw new Error('Field names and types must be of equal length'); + } + verifyStructName(name); + verifyFieldNames(fieldNames); + + let trie = notNull(this.trieRoots.get(Kind.Struct)).traverse(this.identTable.getId(name)); + fieldNames.forEach((fn, i) => { + const ft = fieldTypes[i]; + trie = trie.traverse(this.identTable.getId(fn)); + trie = trie.traverse(ft.id); + }); + + if (trie.t === undefined) { + const fs = fieldNames.map((name, i) => { + const type = fieldTypes[i]; + return {name, type}; + }); + + let t = new Type(new StructDesc(name, fs), 0); + if (t.hasUnresolvedCycle([])) { + t = notNull(this._toUnresolvedType(t, -1)); + t = this._resolveStructCycles(t); + } + t.id = this.nextTypeId(); + trie.t = t; + } + + return trie.t; + } + + _toUnresolvedType(t: Type, level: number, parentStructTypes: Type[] = []): ?Type { + const idx = parentStructTypes.indexOf(t); + if (idx >= 0) { + // This type is just a placeholder. It doesn't need an id + return new Type(new CycleDesc(parentStructTypes.length - idx - 1), 0); + } + + const desc = t.desc; + if (desc instanceof CompoundDesc) { + const elemTypes = desc.elemTypes; + let sts = elemTypes.map(t => this._toUnresolvedType(t, level, parentStructTypes)); + if (sts.some(t => t)) { + sts = sts.map((t, i) => t ? t : elemTypes[i]); + return new Type(new CompoundDesc(t.kind, sts), this.nextTypeId()); + } + return; + } + + if (desc instanceof StructDesc) { + const fields = desc.fields; + const outerType = t; // TODO: Stupid babel bug. + const sts = fields.map(f => { + parentStructTypes.push(outerType); + const t = this._toUnresolvedType(f.type, level + 1, parentStructTypes); + parentStructTypes.pop(); + return t ? t : undefined; + }); + if (sts.some(t => t)) { + const fs = sts.map((t, i) => ({name: fields[i].name, type: t ? t : fields[i].type})); + return new Type(new StructDesc(desc.name, fs), this.nextTypeId()); + } + return; + } + + if (desc instanceof CycleDesc) { + const cycleLevel = desc.level; + return cycleLevel <= level ? t : undefined; + } + } + + _resolveStructCycles(t: Type, parentStructTypes: Type[] = []): Type { + const desc = t.desc; + if (desc instanceof CompoundDesc) { + for (let i = 0; i < desc.elemTypes.length; i++) { + desc.elemTypes[i] = this._resolveStructCycles(desc.elemTypes[i], parentStructTypes); + } + } else if (desc instanceof StructDesc) { + for (let i = 0; i < desc.fields.length; i++) { + parentStructTypes.push(t); + desc.fields[i].type = this._resolveStructCycles(desc.fields[i].type, parentStructTypes); + parentStructTypes.pop(); + } + } else if (desc instanceof CycleDesc) { + const level = desc.level; + if (level < parentStructTypes.length) { + return parentStructTypes[parentStructTypes.length - 1 - level]; + } + } + + return t; + } + + + // Creates a new union type unless the elemTypes can be folded into a single non union type. + makeUnionType(types: Type[]): Type { + types = flattenUnionTypes(types, Object.create(null)); + if (types.length === 1) { + return types[0]; + } + types.sort(compare); + return this.getCompoundType(Kind.Union, ...types); + } + + getCycleType(level: number): Type { + const trie = notNull(this.trieRoots.get(Kind.Cycle)).traverse(level); + + if (trie.t === undefined) { + trie.t = new Type(new CycleDesc(level), this.nextTypeId()); + } + + return trie.t; + } +} + +export const staticTypeCache = new TypeCache(); + +function flattenUnionTypes(types: Type[], seenTypes: {[key: Hash]: boolean}): Type[] { + if (types.length === 0) { + return types; + } + + const newTypes = []; + for (let i = 0; i < types.length; i++) { + if (types[i].kind === Kind.Union) { + newTypes.push(...flattenUnionTypes(types[i].desc.elemTypes, seenTypes)); + } else { + if (!seenTypes[types[i].hash]) { + seenTypes[types[i].hash] = true; + newTypes.push(types[i]); + } + } + } + return newTypes; +} + +const fieldNameRe = /^[a-zA-Z][a-zA-Z0-9_]*$/; + +function verifyFieldNames(names: string[]) { + if (names.length === 0) { + return; + } + + let last = names[0]; + verifyFieldName(last); + + for (let i = 1; i < names.length; i++) { + verifyFieldName(names[i]); + if (last >= names[i]) { + throw new Error('Field names must be unique and ordered alphabetically'); + } + last = names[i]; + } +} + +function verifyName(name: string, kind: '' | ' field') { + if (!fieldNameRe.test(name)) { + throw new Error(`Invalid struct${kind} name: '${name}'`); + } +} + +function verifyFieldName(name: string) { + verifyName(name, ' field'); +} + +function verifyStructName(name: string) { + if (name !== '') { + verifyName(name, ''); + } +} + diff --git a/js/src/type.js b/js/src/type.js index 6f947b10d4..d196c6ccac 100644 --- a/js/src/type.js +++ b/js/src/type.js @@ -4,20 +4,21 @@ // Licensed under the Apache License, version 2.0: // http://www.apache.org/licenses/LICENSE-2.0 -import type Hash from './hash.js'; import Ref from './ref.js'; import type {NomsKind} from './noms-kind.js'; import {invariant} from './assert.js'; import {isPrimitiveKind, Kind} from './noms-kind.js'; import {ValueBase} from './value.js'; import type Value from './value.js'; -import {compare, equals} from './compare.js'; +import {equals} from './compare.js'; import {describeType} from './encode-human-readable.js'; import search from './binary-search.js'; +import {staticTypeCache} from './type-cache.js'; export interface TypeDesc { kind: NomsKind; equals(other: TypeDesc): boolean; + hasUnresolvedCycle(visited: Type[]): boolean; } export class PrimitiveDesc { @@ -30,6 +31,10 @@ export class PrimitiveDesc { equals(other: TypeDesc): boolean { return other instanceof PrimitiveDesc && other.kind === this.kind; } + + hasUnresolvedCycle(visited: Type[]): boolean { // eslint-disable-line no-unused-vars + return false; + } } export class CompoundDesc { @@ -41,7 +46,6 @@ export class CompoundDesc { this.elemTypes = elemTypes; } - equals(other: TypeDesc): boolean { if (other instanceof CompoundDesc) { if (this.kind !== other.kind || this.elemTypes.length !== other.elemTypes.length) { @@ -59,6 +63,10 @@ export class CompoundDesc { return false; } + + hasUnresolvedCycle(visited: Type[]): boolean { + return this.elemTypes.some(t => t.hasUnresolvedCycle(visited)); + } } export type Field = { @@ -109,6 +117,10 @@ export class StructDesc { return true; } + hasUnresolvedCycle(visited: Type[]): boolean { + return this.fields.some(f => f.type.hasUnresolvedCycle(visited)); + } + forEachField(cb: (name: string, type: Type) => void) { const fields = this.fields; for (let i = 0; i < fields.length; i++) { @@ -120,14 +132,6 @@ export class StructDesc { const f = findField(name, this.fields); return f && f.type; } - - setField(name: string, type: Type) { - const f = findField(name, this.fields); - if (!f) { - throw new Error(`No such field "${name}"`); - } - f.type = type; - } } function findField(name: string, fields: Field[]): ?Field { @@ -146,13 +150,36 @@ export function findFieldIndex(name: string, fields: Field[]): number { return i === fields.length || fields[i].name !== name ? -1 : i; } +export class CycleDesc { + level: number; + + constructor(level: number) { + this.level = level; + } + + get kind(): NomsKind { + return Kind.Cycle; + } + + equals(other: TypeDesc): boolean { + return other instanceof CycleDesc && other.level === this.level; + } + + hasUnresolvedCycle(visited: Type[]): boolean { // eslint-disable-line no-unused-vars + return true; + } +} export class Type extends ValueBase { _desc: T; + id: number; + serialization: ?Uint8Array; - constructor(desc: T) { + constructor(desc: T, id: number) { super(); this._desc = desc; + this.id = id; + this.serialization = null; } get type(): Type { @@ -176,6 +203,15 @@ export class Type extends ValueBase { return this._desc.name; } + hasUnresolvedCycle(visited: Type[]): boolean { + if (visited.indexOf(this) >= 0) { + return false; + } + + visited.push(this); + return this._desc.hasUnresolvedCycle(visited); + } + get elemTypes(): Array { invariant(this._desc instanceof CompoundDesc); return this._desc.elemTypes; @@ -186,123 +222,39 @@ export class Type extends ValueBase { } } -function buildType(desc: T): Type { - return new Type(desc); -} - function makePrimitiveType(k: NomsKind): Type { - return buildType(new PrimitiveDesc(k)); + return new Type(new PrimitiveDesc(k), k); } export function makeListType(elemType: Type): Type { - return buildType(new CompoundDesc(Kind.List, [elemType])); + return staticTypeCache.getCompoundType(Kind.List, elemType); } export function makeSetType(elemType: Type): Type { - return buildType(new CompoundDesc(Kind.Set, [elemType])); + return staticTypeCache.getCompoundType(Kind.Set, elemType); } export function makeMapType(keyType: Type, valueType: Type): Type { - return buildType(new CompoundDesc(Kind.Map, [keyType, valueType])); + return staticTypeCache.getCompoundType(Kind.Map, keyType, valueType); } export function makeRefType(elemType: Type): Type { - return buildType(new CompoundDesc(Kind.Ref, [elemType])); + return staticTypeCache.getCompoundType(Kind.Ref, elemType); } export function makeStructType(name: string, fieldNames: string[], fieldTypes: Type[]): Type { - verifyStructName(name); - verifyFieldNames(fieldNames); - - const fs = fieldNames.map((name, i) => { - const type = fieldTypes[i]; - return {name, type}; - }); - - return buildType(new StructDesc(name, fs)); + return staticTypeCache.makeStructType(name, fieldNames, fieldTypes); } -/** - * makeUnionType creates a new union type unless the elemTypes can be folded into a single non - * union type. - */ export function makeUnionType(types: Type[]): Type { - types = flattenUnionTypes(types, Object.create(null)); - if (types.length === 1) { - return types[0]; - } - types.sort(compare); - return buildType(new CompoundDesc(Kind.Union, types)); + return staticTypeCache.makeUnionType(types); } -const fieldNameRe = /^[a-zA-Z][a-zA-Z0-9_]*$/; - -function verifyFieldNames(names: string[]) { - if (names.length === 0) { - return; - } - - let last = names[0]; - verifyFieldName(last); - - for (let i = 1; i < names.length; i++) { - verifyFieldName(names[i]); - if (last >= names[i]) { - throw new Error('Field names must be unique and ordered alphabetically'); - } - last = names[i]; - } +export function makeCycleType(level: number): Type { + return staticTypeCache.getCycleType(level); } -function verifyName(name: string, kind: '' | ' field') { - if (!fieldNameRe.test(name)) { - throw new Error(`Invalid struct${kind} name: "${name}"`); - } -} - -function verifyFieldName(name: string) { - verifyName(name, ' field'); -} - -function verifyStructName(name: string) { - if (name !== '') { - verifyName(name, ''); - } -} - -function flattenUnionTypes(types: Type[], seenTypes: {[key: Hash]: boolean}): Type[] { - if (types.length === 0) { - return types; - } - - const newTypes = []; - for (let i = 0; i < types.length; i++) { - if (types[i].kind === Kind.Union) { - newTypes.push(...flattenUnionTypes(types[i].desc.elemTypes, seenTypes)); - } else { - if (!seenTypes[types[i].hash]) { - seenTypes[types[i].hash] = true; - newTypes.push(types[i]); - } - } - } - return newTypes; -} - -export const boolType = makePrimitiveType(Kind.Bool); -export const numberType = makePrimitiveType(Kind.Number); -export const stringType = makePrimitiveType(Kind.String); -export const blobType = makePrimitiveType(Kind.Blob); -export const typeType = makePrimitiveType(Kind.Type); -export const valueType = makePrimitiveType(Kind.Value); - -export const refOfBlobType = makeRefType(blobType); -export const refOfValueType = makeRefType(valueType); -export const listOfValueType = makeListType(valueType); -export const setOfValueType = makeSetType(valueType); -export const mapOfValueType = makeMapType(valueType, valueType); - /** * Gives the existing primitive Type value for a NomsKind. */ @@ -344,3 +296,10 @@ export function getTypeOfValue(v: Value): Type { throw new Error('Unknown type'); } } + +export const boolType = makePrimitiveType(Kind.Bool); +export const numberType = makePrimitiveType(Kind.Number); +export const stringType = makePrimitiveType(Kind.String); +export const blobType = makePrimitiveType(Kind.Blob); +export const typeType = makePrimitiveType(Kind.Type); +export const valueType = makePrimitiveType(Kind.Value); diff --git a/js/src/value-decoder.js b/js/src/value-decoder.js index 1c66636fe6..233c3d908f 100644 --- a/js/src/value-decoder.js +++ b/js/src/value-decoder.js @@ -11,16 +11,11 @@ import type Struct from './struct.js'; import type {NomsKind} from './noms-kind.js'; import { getPrimitiveType, - makeListType, - makeMapType, - makeRefType, - makeSetType, - makeUnionType, StructDesc, Type, } from './type.js'; import {OrderedKey, MetaTuple} from './meta-sequence.js'; -import {invariant} from './assert.js'; +import {invariant, notNull} from './assert.js'; import {isPrimitiveKind, kindToString, Kind} from './noms-kind.js'; import List, {ListLeafSequence} from './list.js'; import Map, {MapLeafSequence} from './map.js'; @@ -29,14 +24,17 @@ import {IndexedMetaSequence, OrderedMetaSequence} from './meta-sequence.js'; import type Value from './value.js'; import type {ValueReader} from './value-store.js'; import type {NomsReader} from './codec.js'; +import type TypeCache from './type-cache.js'; export default class ValueDecoder { _r: NomsReader; _ds: ValueReader; + _tc: TypeCache; - constructor(r: NomsReader, ds: ValueReader) { + constructor(r: NomsReader, ds: ValueReader, tc: TypeCache) { this._r = r; this._ds = ds; + this._tc = tc; } readKind(): NomsKind { @@ -49,31 +47,29 @@ export default class ValueDecoder { return constructRef(t, hash, height); } - readType(parentStructTypes: Type[]): Type { + readType(): Type { const k = this.readKind(); switch (k) { case Kind.List: - return makeListType(this.readType(parentStructTypes)); + return this._tc.getCompoundType(k, this.readType()); case Kind.Map: - return makeMapType(this.readType(parentStructTypes), - this.readType(parentStructTypes)); + return this._tc.getCompoundType(k, this.readType(), this.readType()); case Kind.Set: - return makeSetType(this.readType(parentStructTypes)); + return this._tc.getCompoundType(k, this.readType()); case Kind.Ref: - return makeRefType(this.readType(parentStructTypes)); + return this._tc.getCompoundType(k, this.readType()); case Kind.Struct: - return this.readStructType(parentStructTypes); + return this.readStructType(); case Kind.Union: { const len = this._r.readUint32(); const types: Type[] = new Array(len); for (let i = 0; i < len; i++) { - types[i] = this.readType(parentStructTypes); + types[i] = this.readType(); } - return makeUnionType(types); + return this._tc.getCompoundType(k, ...types); } case Kind.Cycle: { - const i = this._r.readUint32(); - return parentStructTypes[parentStructTypes.length - 1 - i]; + return this._tc.getCycleType(this._r.readUint32()); } } @@ -143,7 +139,7 @@ export default class ValueDecoder { } readValue(): any { - const t = this.readType([]); + const t = this.readType(); switch (t.kind) { case Kind.Blob: { const isMeta = this._r.readBool(); @@ -185,7 +181,7 @@ export default class ValueDecoder { case Kind.Struct: return this.readStruct(t); case Kind.Type: - return this.readType([]); + return this.readType(); case Kind.Cycle: case Kind.Union: case Kind.Value: @@ -208,24 +204,34 @@ export default class ValueDecoder { return newStructWithType(type, values); } - readStructType(parentStructTypes: Type[]): Type { + readCachedStructType(): Type { + let trie = notNull(this._tc.trieRoots.get(Kind.Struct)).traverse(this._r.readIdent(this._tc)); + const count = this._r.readUint32(); + for (let i = 0; i < count; i++) { + trie = trie.traverse(this._r.readIdent(this._tc)); + trie = trie.traverse(this.readType().id); + } + + return trie.t; + } + + readStructType(): Type { + const pos = this._r.pos(); + const t = this.readCachedStructType(); + if (t) { + return t; + } + this._r.seek(pos); + const name = this._r.readString(); const count = this._r.readUint32(); - const fields = new Array(count); - const desc = new StructDesc(name, fields); - const structType = new Type(desc); - parentStructTypes.push(structType); - + const fieldNames = new Array(count); + const fieldTypes = new Array(count); for (let i = 0; i < count; i++) { - const name = this._r.readString(); - const type = this.readType(parentStructTypes); - // Mutate the already created structType since when looking for the cycle we compare - // by identity. - fields[i] = {name, type}; + fieldNames[i] = this._r.readString(); + fieldTypes[i] = this.readType(); } - - parentStructTypes.pop(); - return structType; + return this._tc.makeStructType(name, fieldNames, fieldTypes); } } diff --git a/js/src/value-encoder.js b/js/src/value-encoder.js index 0d6b17a3f2..bf233664c6 100644 --- a/js/src/value-encoder.js +++ b/js/src/value-encoder.js @@ -131,7 +131,7 @@ export default class ValueEncoder { writeValue(v: Value) { const t = getTypeOfValue(v); - this.writeType(t, []); + this._w.appendType(t); switch (t.kind) { case Kind.Blob: { invariant(v instanceof Blob, @@ -205,7 +205,7 @@ export default class ValueEncoder { case Kind.Type: invariant(v instanceof Type, () => `Failed to write Type. Invalid type: ${describeTypeOfValue(v)}`); - this.writeType(v, []); + this._w.appendType(v); break; case Kind.Struct: invariant(v instanceof Struct,