mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-02-09 18:59:12 -06:00
Code Issues Packages Projects Releases 127 Wiki Activity

Js type cache (#2011)

Browse Source 
Js type cache
This commit is contained in:
Rafael Weinstein
2016-07-11 16:45:35 -07:00
committed by GitHub
parent 5483383acc
commit 4e10ae63f4
20 changed files with 666 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go/types/codec.go
View File

@@ -144,8 +144,6 @@ func (b *binaryNomsReader) readString() string {
return v
}
var createCount = uint64(0)
// Note: It's somewhat of a layering violation that a nomsReaders knows about a TypeCache. The reason why the code is structured this way is that the go compiler can stack-allocate the string which is created from the byte slice, which is a fairly large perf gain.
func (b *binaryNomsReader) readIdent(tc *TypeCache) uint32 {
size := b.readUint32()

2
go/types/encode_human_readable_test.go
View File

@@ -360,7 +360,7 @@ func TestRecursiveStruct(t *testing.T) {
})`, d)
}
func TestUnserolvedRecursiveStruct(t *testing.T) {
func TestUnresolvedRecursiveStruct(t *testing.T) {
// struct A {
// a: A
// b: Cycle<1> (unresolved)

10
go/types/type.go
View File

@@ -30,7 +30,11 @@ type Type struct {
const initialTypeBufferSize = 128
func newType(desc TypeDesc, id uint32) *Type {
t := &Type{desc, &hash.Hash{}, id, nil}
return &Type{desc, &hash.Hash{}, id, nil}
}
func buildType(desc TypeDesc, id uint32) *Type {
t := newType(desc, id)
if !t.HasUnresolvedCycle() {
serializeType(t)
}
@@ -133,10 +137,6 @@ func MakePrimitiveType(k NomsKind) *Type {
return nil
}
func makePrimitiveType(k NomsKind) *Type {
return newType(PrimitiveDesc(k), uint32(k))
}
func MakePrimitiveTypeByString(p string) *Type {
switch p {
case "Bool":

44
go/types/type_cache.go
View File

@@ -15,13 +15,16 @@ import (
type TypeCache struct {
identTable *identTable
trieRoots map[NomsKind]*typeTrie
typeBytes map[uint32][]byte
nextId uint32
mu *sync.Mutex
}
var staticTypeCache = NewTypeCache()
func makePrimitiveType(k NomsKind) *Type {
return buildType(PrimitiveDesc(k), uint32(k))
}
var BoolType = makePrimitiveType(BoolKind)
var NumberType = makePrimitiveType(NumberKind)
var StringType = makePrimitiveType(StringKind)
@@ -41,7 +44,6 @@ func NewTypeCache() *TypeCache {
CycleKind: newTypeTrie(),
UnionKind: newTypeTrie(),
},
map[uint32][]byte{},
256, // The first 255 type ids are reserved for the 8bit space of NomsKinds.
&sync.Mutex{},
}
@@ -68,7 +70,7 @@ func (tc *TypeCache) getCompoundType(kind NomsKind, elemTypes ...*Type) *Type {
}
if trie.t == nil {
trie.t = newType(CompoundDesc{kind, elemTypes}, tc.nextTypeId())
trie.t = buildType(CompoundDesc{kind, elemTypes}, tc.nextTypeId())
}
return trie.t
@@ -93,7 +95,7 @@ func (tc *TypeCache) makeStructType(name string, fieldNames []string, fieldTypes
i++
}
t := newType(StructDesc{name, fs}, 0)
t := buildType(StructDesc{name, fs}, 0)
if t.serialization == nil {
// HasUnresolvedCycle
t, _ = toUnresolvedType(t, tc, -1, nil)
@@ -122,8 +124,7 @@ func indexOfType(t *Type, tl []*Type) (uint32, bool) {
func toUnresolvedType(t *Type, tc *TypeCache, level int, parentStructTypes []*Type) (*Type, bool) {
i, found := indexOfType(t, parentStructTypes)
if found {
cycle := CycleDesc(uint32(len(parentStructTypes)) - i - 1)
return &Type{cycle, &hash.Hash{}, 0, nil}, true // This type is just a placeholder. It doesn't need an id
return newType(CycleDesc(uint32(len(parentStructTypes))-i-1), 0), true // This type is just a placeholder. It doesn't need an id
}
switch desc := t.Desc.(type) {
@@ -140,7 +141,7 @@ func toUnresolvedType(t *Type, tc *TypeCache, level int, parentStructTypes []*Ty
return t, false
}
return &Type{CompoundDesc{t.Kind(), ts}, &hash.Hash{}, tc.nextTypeId(), nil}, true
return newType(CompoundDesc{t.Kind(), ts}, tc.nextTypeId()), true
case StructDesc:
fs := make(fieldSlice, len(desc.fields))
didChange := false
@@ -155,7 +156,7 @@ func toUnresolvedType(t *Type, tc *TypeCache, level int, parentStructTypes []*Ty
return t, false
}
return &Type{StructDesc{desc.Name, fs}, &hash.Hash{}, tc.nextTypeId(), nil}, true
return newType(StructDesc{desc.Name, fs}, tc.nextTypeId()), true
case CycleDesc:
cycleLevel := int(desc)
return t, cycleLevel <= level // Only cycles which can be resolved in the current struct.
@@ -224,11 +225,11 @@ func (tc *TypeCache) makeUnionType(elemTypes ...*Type) *Type {
return tc.getCompoundType(UnionKind, ts...)
}
func (tc *TypeCache) getCyclicType(level uint32) *Type {
func (tc *TypeCache) getCycleType(level uint32) *Type {
trie := tc.trieRoots[CycleKind].Traverse(level)
if trie.t == nil {
trie.t = newType(CycleDesc(level), tc.nextTypeId())
trie.t = buildType(CycleDesc(level), tc.nextTypeId())
}
return trie.t
@@ -292,7 +293,7 @@ func MakeUnionType(elemTypes ...*Type) *Type {
func MakeCycleType(level uint32) *Type {
staticTypeCache.Lock()
defer staticTypeCache.Unlock()
return staticTypeCache.getCyclicType(level)
return staticTypeCache.getCycleType(level)
}
// All types in noms are created in a deterministic order. A typeTrie stores types within a typeCache and allows construction of a prexisting type to return the already existing one rather than allocate a new one.
@@ -306,13 +307,12 @@ func newTypeTrie() *typeTrie {
}
func (tct *typeTrie) Traverse(typeId uint32) *typeTrie {
if t, ok := tct.entries[typeId]; ok {
return t
next, ok := tct.entries[typeId]
if !ok {
// Insert edge
next = newTypeTrie()
tct.entries[typeId] = next
}
// Insert edge
next := newTypeTrie()
tct.entries[typeId] = next
return next
}
@@ -326,12 +326,12 @@ func newIdentTable() *identTable {
}
func (it *identTable) GetId(ident string) uint32 {
if id, ok := it.entries[ident]; ok {
return id
id, ok := it.entries[ident]
if !ok {
id = it.nextId
it.nextId++
it.entries[ident] = id
}
id := it.nextId
it.nextId++
it.entries[ident] = id
return id
}

7
go/types/type_cache_test.go
View File

@@ -68,10 +68,15 @@ func TestTypeCacheRef(t *testing.T) {
lst := MakeRefType(StringType)
lnt := MakeRefType(NumberType)
assert.False(lst == lnt)
assert.NotNil(lnt.serialization)
lst2 := MakeRefType(StringType)
assert.True(lst == lst2)
lnt2 := MakeRefType(NumberType)
assert.True(lnt == lnt2)
lbt3 := MakeRefType(BoolType)
assert.True(lbt == lbt3)
}
func TestTypeCacheStruct(t *testing.T) {

2
go/types/value_decoder.go
View File

@@ -53,7 +53,7 @@ func (r *valueDecoder) readType() *Type {
}
return r.tc.getCompoundType(UnionKind, elemTypes...)
case CycleKind:
return r.tc.getCyclicType(r.readUint32())
return r.tc.getCycleType(r.readUint32())
}
d.Chk.True(IsPrimitiveKind(k))

12
js/src/assert-type-test.js
View File

@@ -16,16 +16,13 @@ import type {Type} from './type.js';
import {
blobType,
boolType,
listOfValueType,
makeListType,
makeMapType,
makeRefType,
makeSetType,
makeStructType,
makeUnionType,
mapOfValueType,
numberType,
setOfValueType,
stringType,
typeType,
valueType,
@@ -89,7 +86,7 @@ suite('validate type', () => {
assertSubtype(listOfNumberType, l);
assertAll(listOfNumberType, l);
assertSubtype(listOfValueType, l);
assertSubtype(makeListType(valueType), l);
});
test('map', () => {
@@ -98,7 +95,7 @@ suite('validate type', () => {
assertSubtype(mapOfNumberToStringType, m);
assertAll(mapOfNumberToStringType, m);
assertSubtype(mapOfValueType, m);
assertSubtype(makeMapType(valueType, valueType), m);
});
test('set', () => {
@@ -107,7 +104,7 @@ suite('validate type', () => {
assertSubtype(setOfNumberType, s);
assertAll(setOfNumberType, s);
assertSubtype(setOfValueType, s);
assertSubtype(makeSetType(valueType), s);
});
test('type', () => {
@@ -233,11 +230,10 @@ suite('validate type', () => {
const t11 = makeStructType('Commit',
['parents', 'value'],
[
makeSetType(makeRefType(t1)),
numberType,
numberType, // placeholder
]
);
t11.desc.setField('parents', makeSetType(makeRefType(t11)));
assertSubtype(t11, c1);
const c2 = newStruct('Commit', {

4
js/src/blob-test.js
View File

@@ -4,7 +4,7 @@
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import {blobType, refOfBlobType} from './type.js';
import {blobType, makeRefType} from './type.js';
import {assert} from 'chai';
import Blob, {BlobReader, BlobWriter} from './blob.js';
import {suite, test, setup, teardown} from 'mocha';
@@ -118,7 +118,7 @@ suite('Blob', () => {
assertValueHash(expectHashStr, blob);
assertValueType(blobType, blob);
assert.strictEqual(length, blob.length);
assertChunkCountAndType(expectChunkCount, refOfBlobType, blob);
assertChunkCountAndType(expectChunkCount, makeRefType(blobType), blob);
await testRoundTripAndValidate(blob, async(b2) => {
await assertReadFull(buff, b2.getReader());

58
js/src/codec.js
View File

@@ -4,18 +4,20 @@
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import * as Bytes from './bytes.js';
import Chunk from './chunk.js';
import Hash, {sha1Size} from './hash.js';
import ValueDecoder from './value-decoder.js';
import ValueEncoder from './value-encoder.js';
import {invariant} from './assert.js';
import svarint from 'signed-varint';
import type Value from './value.js';
import type {Type} from './type.js';
import type {ValueReader, ValueWriter} from './value-store.js';
import {default as TypeCache, staticTypeCache} from './type-cache.js';
import {floatToIntExp, intExpToFloat} from './number-util.js';
import {invariant, notNull} from './assert.js';
import {setEncodeValue} from './get-hash.js';
import {setHash, ValueBase} from './value.js';
import type Value from './value.js';
import type {ValueReader, ValueWriter} from './value-store.js';
import * as Bytes from './bytes.js';
import {floatToIntExp, intExpToFloat} from './number-util.js';
import svarint from 'signed-varint';
export function encodeValue(v: Value, vw: ?ValueWriter): Chunk {
const w = new BinaryNomsWriter();
@@ -33,7 +35,7 @@ setEncodeValue(encodeValue);
export function decodeValue(chunk: Chunk, vr: ValueReader): Value {
const data = chunk.data;
const dec = new ValueDecoder(new BinaryNomsReader(data), vr);
const dec = new ValueDecoder(new BinaryNomsReader(data), vr, staticTypeCache);
const v = dec.readValue();
if (v instanceof ValueBase) {
@@ -43,11 +45,21 @@ export function decodeValue(chunk: Chunk, vr: ValueReader): Value {
return v;
}
function ensureTypeSerialization(t: Type) {
if (!t.serialization) {
const w = new BinaryNomsWriter();
const enc = new ValueEncoder(w, null);
enc.writeType(t, []);
t.serialization = w.data;
}
}
const maxUInt32 = Math.pow(2, 32);
const littleEndian = true;
export interface NomsReader {
pos(): number;
seek(pos: number): void;
readBytes(): Uint8Array;
readUint8(): number;
readUint32(): number;
@@ -55,6 +67,7 @@ export interface NomsReader {
readNumber(): number;
readBool(): boolean;
readString(): string;
readIdent(tc: TypeCache): number;
readHash(): Hash;
}
@@ -67,6 +80,7 @@ export interface NomsWriter {
writeBool(v:boolean): void;
writeString(v: string): void;
writeHash(h: Hash): void;
appendType(t: Type): void;
}
export class BinaryNomsReader {
@@ -80,6 +94,14 @@ export class BinaryNomsReader {
this.offset = 0;
}
pos(): number {
return this.offset;
}
seek(pos: number): void {
this.offset = pos;
}
readBytes(): Uint8Array {
const size = this.readUint32();
// Make a copy of the buffer to return
@@ -129,6 +151,16 @@ export class BinaryNomsReader {
return str;
}
readIdent(tc: TypeCache): number {
const str = this.readString(); // TODO: Figure out how to do this without allocating.
let id = tc.identTable.entries.get(str);
if (id === undefined) {
id = tc.identTable.getId(str);
}
return id;
}
readHash(): Hash {
// Make a copy of the data.
const digest = Bytes.slice(this.buff, this.offset, this.offset + sha1Size);
@@ -225,4 +257,16 @@ export class BinaryNomsWriter {
Bytes.copy(h.digest, this.buff, this.offset);
this.offset += sha1Size;
}
appendType(t: Type): void {
// Note: The JS & Go impls differ here. The Go impl eagerly serializes types as they are
// constructed. The JS does it lazily so as to avoid cyclic package dependencies.
ensureTypeSerialization(t);
const data = notNull(t.serialization);
const size = data.byteLength;
this.ensureCapacity(size);
Bytes.copy(data, this.buff, this.offset);
this.offset += size;
}
}

16
js/src/commit.js
View File

@@ -5,16 +5,28 @@
// http://www.apache.org/licenses/LICENSE-2.0
import {invariant} from './assert.js';
import {getDatasTypes} from './database.js';
import Struct from './struct.js';
import type Value from './value.js';
import type Ref from './ref.js';
import Set from './set.js';
import {
makeCycleType,
makeRefType,
makeStructType,
makeSetType,
valueType,
} from './type.js';
export const commitType = makeStructType('Commit',
['parents', 'value'],
[
makeSetType(makeRefType(makeCycleType(0))),
valueType,
]
);
export default class Commit<T: Value> extends Struct {
constructor(value: T, parents: Set<Ref<Commit>> = new Set()) {
const {commitType} = getDatasTypes();
super(commitType, [parents, value]);
}

45
js/src/database.js
View File

@@ -12,54 +12,9 @@ import type Value from './value.js';
import type {RootTracker} from './chunk-store.js';
import ValueStore from './value-store.js';
import type {BatchStore} from './batch-store.js';
import {
makeRefType,
makeStructType,
makeSetType,
makeMapType,
Type,
stringType,
valueType,
} from './type.js';
import Commit from './commit.js';
import {equals} from './compare.js';
type DatasTypes = {
commitType: Type,
commitSetType: Type,
refOfCommitType: Type,
commitMapType: Type,
};
let datasTypes: DatasTypes;
export function getDatasTypes(): DatasTypes {
if (!datasTypes) {
// struct Commit {
// value: Value
// parents: Set<Ref<Commit>>
// }
const commitType = makeStructType('Commit',
['parents', 'value'],
[
valueType,
valueType,
]
);
const refOfCommitType = makeRefType(commitType);
const commitSetType = makeSetType(refOfCommitType);
commitType.desc.setField('parents', commitSetType);
const commitMapType = makeMapType(stringType, refOfCommitType);
datasTypes = {
commitType,
refOfCommitType,
commitSetType,
commitMapType,
};
}
return datasTypes;
}
export default class Database {
_vs: ValueStore;
_rt: RootTracker;

51
js/src/encode-human-readable-test.js
View File

@@ -7,11 +7,13 @@
import {assert} from 'chai';
import {suite, test} from 'mocha';
import {invariant, notNull} from './assert.js';
import {TypeWriter} from './encode-human-readable.js';
import {
blobType,
boolType,
numberType,
makeCycleType,
makeRefType,
makeListType,
makeMapType,
@@ -19,7 +21,7 @@ import {
makeStructType,
makeUnionType,
stringType,
valueType,
StructDesc,
Type,
} from './type.js';
@@ -86,23 +88,17 @@ suite('Encode human readable types', () => {
const a = makeStructType('A',
['b', 'c', 'd'],
[
valueType, // placeholder
valueType, // placeholder
valueType, // placeholder
makeCycleType(0),
makeListType(makeCycleType(0)),
makeStructType('D',
['e', 'f'],
[
makeCycleType(0),
makeCycleType(1),
]
),
]
);
const d = makeStructType('D',
['e', 'f'],
[
valueType, // placeholder
a,
]
);
a.desc.setField('b', a);
a.desc.setField('d', d);
d.desc.setField('e', d);
d.desc.setField('f', a);
a.desc.setField('c', makeListType(a));
assertWriteType(`struct A {
b: Cycle<0>,
@@ -113,6 +109,9 @@ suite('Encode human readable types', () => {
},
}`, a);
invariant(a.desc instanceof StructDesc);
const d = notNull(a.desc.getField('d'));
assertWriteType(`struct D {
e: Cycle<0>,
f: struct A {
@@ -122,4 +121,24 @@ suite('Encode human readable types', () => {
},
}`, d);
});
test('recursive unresolved struct', () => {
// struct A {
// a: A
// b: Cycle<1>
// }
const a = makeStructType('A',
['a', 'b'],
[
makeCycleType(0),
makeCycleType(1),
]
);
assertWriteType(`struct A {
a: Cycle<0>,
b: Cycle<1>,
}`, a);
});
});

5
js/src/encode-human-readable.js
View File

@@ -4,7 +4,7 @@
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import {getTypeOfValue, CompoundDesc} from './type.js';
import {getTypeOfValue, CompoundDesc, CycleDesc} from './type.js';
import type {Type} from './type.js';
import {Kind, kindToString} from './noms-kind.js';
import type {NomsKind} from './noms-kind.js';
@@ -109,6 +109,9 @@ export class TypeWriter {
this._writeStructType(t, parentStructTypes);
break;
case Kind.Cycle:
invariant(t.desc instanceof CycleDesc);
this._w.write(`Cycle<${t.desc.level}>`);
break;
default:
throw new Error('unreachable');
}

39
js/src/encoding-test.js
View File

@@ -34,16 +34,18 @@ import {
} from './meta-sequence.js';
import {
boolType,
blobType,
makeCycleType,
makeListType,
makeMapType,
makeRefType,
makeSetType,
makeStructType,
numberType,
refOfBlobType,
stringType,
typeType,
} from './type.js';
import {staticTypeCache} from './type-cache.js';
function assertRoundTrips(v: Value) {
const db = new TestDatabase();
@@ -121,6 +123,14 @@ suite('Encoding', () => {
this.i = 0;
}
pos(): number {
return this.i;
}
seek(pos: number): void {
this.i = pos;
}
atEnd(): boolean {
return this.i === this.a.length;
}
@@ -172,6 +182,16 @@ suite('Encoding', () => {
return v;
}
readIdent(tc: TypeCache): number {
const s = this.readString();
let id = tc.identTable.entries.get(s);
if (id === undefined) {
id = tc.identTable.getId(s);
}
return id;
}
readHash(): Hash {
return Hash.parse(this.readString());
}
@@ -221,6 +241,11 @@ suite('Encoding', () => {
this.writeString(h.toString());
}
appendType(t: Type): void {
const enc = new ValueEncoder(this, null);
enc.writeType(t, []);
}
toArray(): any[] {
return this.a;
}
@@ -246,7 +271,7 @@ suite('Encoding', () => {
assert.deepEqual(encoding, w.toArray());
const r = new TestReader(encoding);
const dec = new ValueDecoder(r, null);
const dec = new ValueDecoder(r, null, staticTypeCache);
const v2 = dec.readValue();
assert.isTrue(equals(v, v2));
}
@@ -340,9 +365,9 @@ suite('Encoding', () => {
uint8(RefKind), uint8(BlobKind), r3.toString(), uint64(33), uint8(NumberKind), float64(60), uint64(60),
],
Blob.fromSequence(newBlobMetaSequence(null, [
new MetaTuple(constructRef(refOfBlobType, r1, 11), new OrderedKey(20), 20, null),
new MetaTuple(constructRef(refOfBlobType, r2, 22), new OrderedKey(40), 40, null),
new MetaTuple(constructRef(refOfBlobType, r3, 33), new OrderedKey(60), 60, null),
new MetaTuple(constructRef(makeRefType(blobType), r1, 11), new OrderedKey(20), 20, null),
new MetaTuple(constructRef(makeRefType(blobType), r2, 22), new OrderedKey(40), 40, null),
new MetaTuple(constructRef(makeRefType(blobType), r3, 33), new OrderedKey(60), 60, null),
]))
);
});
@@ -498,12 +523,10 @@ suite('Encoding', () => {
const structType = makeStructType('A6',
['cs', 'v'],
[
numberType, // placeholder
makeListType(makeCycleType(0)),
numberType,
]
);
const listType = makeListType(structType);
structType.desc.setField('cs', listType);
assertEncoding([
uint8(StructKind), 'A6', uint32(2) /* len */, 'cs', uint8(ListKind), uint8(CycleKind), uint32(0), 'v', uint8(NumberKind),

8
js/src/struct-test.js
View File

@@ -16,11 +16,11 @@ import Struct, {
import {assert} from 'chai';
import {
boolType,
makeCycleType,
makeListType,
makeStructType,
numberType,
stringType,
valueType,
} from './type.js';
import {suite, test} from 'mocha';
import {equals} from './compare.js';
@@ -126,13 +126,11 @@ suite('Struct', () => {
['b', 'l'],
[
boolType,
valueType, // placeholder
makeListType(makeCycleType(0)),
]
);
const listType = makeListType(type);
type.desc.setField('l', listType);
const emptyList = new List([], listType);
const emptyList = new List([]);
newStructWithType(type, [true, emptyList]);
newStructWithType(type,
[

124
js/src/type-cache-test.js Normal file
View File

@@ -0,0 +1,124 @@
// @flow
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import {assert} from 'chai';
import {suite, test} from 'mocha';
import {
boolType,
makeCycleType,
makeListType,
makeRefType,
makeSetType,
makeStructType,
makeUnionType,
numberType,
stringType,
} from './type.js';
suite('TypeCache', () => {
test('list', () => {
const lbt = makeListType(boolType);
const lbt2 = makeListType(boolType);
assert.strictEqual(lbt, lbt2);
const lst = makeListType(stringType);
const lnt = makeListType(numberType);
assert.notEqual(lst, lnt);
const lst2 = makeListType(stringType);
assert.strictEqual(lst, lst2);
const lnt2 = makeListType(numberType);
assert.strictEqual(lnt, lnt2);
const lbt3 = makeListType(boolType);
assert.strictEqual(lbt, lbt3);
});
test('set', () => {
const lbt = makeSetType(boolType);
const lbt2 = makeSetType(boolType);
assert.strictEqual(lbt, lbt2);
const lst = makeSetType(stringType);
const lnt = makeSetType(numberType);
assert.notEqual(lst, lnt);
const lst2 = makeSetType(stringType);
assert.strictEqual(lst, lst2);
const lnt2 = makeSetType(numberType);
assert.strictEqual(lnt, lnt2);
const lbt3 = makeSetType(boolType);
assert.strictEqual(lbt, lbt3);
});
test('ref', () => {
const lbt = makeRefType(boolType);
const lbt2 = makeRefType(boolType);
assert.strictEqual(lbt, lbt2);
const lst = makeRefType(stringType);
const lnt = makeRefType(numberType);
assert.notEqual(lst, lnt);
const lst2 = makeRefType(stringType);
assert.strictEqual(lst, lst2);
const lnt2 = makeRefType(numberType);
assert.strictEqual(lnt, lnt2);
const lbt3 = makeRefType(boolType);
assert.strictEqual(lbt, lbt3);
});
test('struct', () => {
const st = makeStructType('Foo',
['bar', 'foo'],
[stringType, numberType]
);
const st2 = makeStructType('Foo',
['bar', 'foo'],
[stringType, numberType]
);
assert.strictEqual(st, st2);
});
test('union', () => {
let ut = makeUnionType([numberType]);
let ut2 = makeUnionType([numberType]);
assert.strictEqual(ut, ut2);
assert.strictEqual(ut2, numberType);
ut = makeUnionType([numberType, stringType]);
ut2 = makeUnionType([stringType, numberType]);
assert.strictEqual(ut, ut2);
ut = makeUnionType([stringType, boolType, numberType]);
ut2 = makeUnionType([numberType, stringType, boolType]);
assert.strictEqual(ut, ut2);
});
test('Cyclic Struct', () => {
const st = makeStructType('Foo',
['foo'],
[
makeRefType(makeCycleType(0)),
]);
assert.isFalse(st.hasUnresolvedCycle([]));
assert.strictEqual(st, st.desc.fields[0].type.desc.elemTypes[0]);
const st2 = makeStructType('Foo',
['foo'],
[
makeRefType(makeCycleType(0)),
]);
assert.isFalse(st2.hasUnresolvedCycle([]));
assert.strictEqual(st, st2);
});
});

257
js/src/type-cache.js Normal file
View File

@@ -0,0 +1,257 @@
// @flow
// Copyright 2016 The Noms Authors. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import type Hash from './hash.js';
import type {NomsKind} from './noms-kind.js';
import {Kind} from './noms-kind.js';
import {CompoundDesc, CycleDesc, StructDesc, Type} from './type.js';
import {compare} from './compare.js';
import {notNull} from './assert.js';
class IdentTable {
entries: Map<string, number>;
nextId: number;
constructor() {
this.entries = new Map();
this.nextId = 0;
}
getId(ident: string): number {
let id = this.entries.get(ident);
if (id === undefined) {
id = this.nextId++;
this.entries.set(ident, id);
}
return id;
}
}
class TypeTrie {
t: Type;
entries: Map<number, TypeTrie>;
constructor() {
this.entries = new Map();
}
traverse(typeId: number): TypeTrie {
let next = this.entries.get(typeId);
if (!next) {
// Insert edge
next = new TypeTrie();
this.entries.set(typeId, next);
}
return next;
}
}
export default class TypeCache {
identTable: IdentTable;
trieRoots: Map<NomsKind, TypeTrie>;
nextId: number;
constructor() {
this.identTable = new IdentTable();
this.trieRoots = new Map();
this.trieRoots.set(Kind.List, new TypeTrie());
this.trieRoots.set(Kind.Set, new TypeTrie());
this.trieRoots.set(Kind.Ref, new TypeTrie());
this.trieRoots.set(Kind.Map, new TypeTrie());
this.trieRoots.set(Kind.Struct, new TypeTrie());
this.trieRoots.set(Kind.Cycle, new TypeTrie());
this.trieRoots.set(Kind.Union, new TypeTrie());
this.nextId = 256; // The first 255 type ids are reserved for the 8bit space of NomsKinds.
}
nextTypeId(): number {
return this.nextId++;
}
getCompoundType(kind: NomsKind, ...elemTypes: Type[]): Type {
let trie = notNull(this.trieRoots.get(kind));
elemTypes.forEach(t => trie = notNull(trie).traverse(t.id));
if (!notNull(trie).t) {
trie.t = new Type(new CompoundDesc(kind, elemTypes), this.nextTypeId());
}
return trie.t;
}
makeStructType(name: string, fieldNames: string[], fieldTypes: Type[]): Type<StructDesc> {
if (fieldNames.length !== fieldTypes.length) {
throw new Error('Field names and types must be of equal length');
}
verifyStructName(name);
verifyFieldNames(fieldNames);
let trie = notNull(this.trieRoots.get(Kind.Struct)).traverse(this.identTable.getId(name));
fieldNames.forEach((fn, i) => {
const ft = fieldTypes[i];
trie = trie.traverse(this.identTable.getId(fn));
trie = trie.traverse(ft.id);
});
if (trie.t === undefined) {
const fs = fieldNames.map((name, i) => {
const type = fieldTypes[i];
return {name, type};
});
let t = new Type(new StructDesc(name, fs), 0);
if (t.hasUnresolvedCycle([])) {
t = notNull(this._toUnresolvedType(t, -1));
t = this._resolveStructCycles(t);
}
t.id = this.nextTypeId();
trie.t = t;
}
return trie.t;
}
_toUnresolvedType(t: Type, level: number, parentStructTypes: Type[] = []): ?Type {
const idx = parentStructTypes.indexOf(t);
if (idx >= 0) {
// This type is just a placeholder. It doesn't need an id
return new Type(new CycleDesc(parentStructTypes.length - idx - 1), 0);
}
const desc = t.desc;
if (desc instanceof CompoundDesc) {
const elemTypes = desc.elemTypes;
let sts = elemTypes.map(t => this._toUnresolvedType(t, level, parentStructTypes));
if (sts.some(t => t)) {
sts = sts.map((t, i) => t ? t : elemTypes[i]);
return new Type(new CompoundDesc(t.kind, sts), this.nextTypeId());
}
return;
}
if (desc instanceof StructDesc) {
const fields = desc.fields;
const outerType = t; // TODO: Stupid babel bug.
const sts = fields.map(f => {
parentStructTypes.push(outerType);
const t = this._toUnresolvedType(f.type, level + 1, parentStructTypes);
parentStructTypes.pop();
return t ? t : undefined;
});
if (sts.some(t => t)) {
const fs = sts.map((t, i) => ({name: fields[i].name, type: t ? t : fields[i].type}));
return new Type(new StructDesc(desc.name, fs), this.nextTypeId());
}
return;
}
if (desc instanceof CycleDesc) {
const cycleLevel = desc.level;
return cycleLevel <= level ? t : undefined;
}
}
_resolveStructCycles(t: Type, parentStructTypes: Type[] = []): Type {
const desc = t.desc;
if (desc instanceof CompoundDesc) {
for (let i = 0; i < desc.elemTypes.length; i++) {
desc.elemTypes[i] = this._resolveStructCycles(desc.elemTypes[i], parentStructTypes);
}
} else if (desc instanceof StructDesc) {
for (let i = 0; i < desc.fields.length; i++) {
parentStructTypes.push(t);
desc.fields[i].type = this._resolveStructCycles(desc.fields[i].type, parentStructTypes);
parentStructTypes.pop();
}
} else if (desc instanceof CycleDesc) {
const level = desc.level;
if (level < parentStructTypes.length) {
return parentStructTypes[parentStructTypes.length - 1 - level];
}
}
return t;
}
// Creates a new union type unless the elemTypes can be folded into a single non union type.
makeUnionType(types: Type[]): Type {
types = flattenUnionTypes(types, Object.create(null));
if (types.length === 1) {
return types[0];
}
types.sort(compare);
return this.getCompoundType(Kind.Union, ...types);
}
getCycleType(level: number): Type {
const trie = notNull(this.trieRoots.get(Kind.Cycle)).traverse(level);
if (trie.t === undefined) {
trie.t = new Type(new CycleDesc(level), this.nextTypeId());
}
return trie.t;
}
}
export const staticTypeCache = new TypeCache();
function flattenUnionTypes(types: Type[], seenTypes: {[key: Hash]: boolean}): Type[] {
if (types.length === 0) {
return types;
}
const newTypes = [];
for (let i = 0; i < types.length; i++) {
if (types[i].kind === Kind.Union) {
newTypes.push(...flattenUnionTypes(types[i].desc.elemTypes, seenTypes));
} else {
if (!seenTypes[types[i].hash]) {
seenTypes[types[i].hash] = true;
newTypes.push(types[i]);
}
}
}
return newTypes;
}
const fieldNameRe = /^[a-zA-Z][a-zA-Z0-9_]*$/;
function verifyFieldNames(names: string[]) {
if (names.length === 0) {
return;
}
let last = names[0];
verifyFieldName(last);
for (let i = 1; i < names.length; i++) {
verifyFieldName(names[i]);
if (last >= names[i]) {
throw new Error('Field names must be unique and ordered alphabetically');
}
last = names[i];
}
}
function verifyName(name: string, kind: '' | ' field') {
if (!fieldNameRe.test(name)) {
throw new Error(`Invalid struct${kind} name: '${name}'`);
}
}
function verifyFieldName(name: string) {
verifyName(name, ' field');
}
function verifyStructName(name: string) {
if (name !== '') {
verifyName(name, '');
}
}

169
js/src/type.js
View File

@@ -4,20 +4,21 @@
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import type Hash from './hash.js';
import Ref from './ref.js';
import type {NomsKind} from './noms-kind.js';
import {invariant} from './assert.js';
import {isPrimitiveKind, Kind} from './noms-kind.js';
import {ValueBase} from './value.js';
import type Value from './value.js';
import {compare, equals} from './compare.js';
import {equals} from './compare.js';
import {describeType} from './encode-human-readable.js';
import search from './binary-search.js';
import {staticTypeCache} from './type-cache.js';
export interface TypeDesc {
kind: NomsKind;
equals(other: TypeDesc): boolean;
hasUnresolvedCycle(visited: Type[]): boolean;
}
export class PrimitiveDesc {
@@ -30,6 +31,10 @@ export class PrimitiveDesc {
equals(other: TypeDesc): boolean {
return other instanceof PrimitiveDesc && other.kind === this.kind;
}
hasUnresolvedCycle(visited: Type[]): boolean { // eslint-disable-line no-unused-vars
return false;
}
}
export class CompoundDesc {
@@ -41,7 +46,6 @@ export class CompoundDesc {
this.elemTypes = elemTypes;
}
equals(other: TypeDesc): boolean {
if (other instanceof CompoundDesc) {
if (this.kind !== other.kind || this.elemTypes.length !== other.elemTypes.length) {
@@ -59,6 +63,10 @@ export class CompoundDesc {
return false;
}
hasUnresolvedCycle(visited: Type[]): boolean {
return this.elemTypes.some(t => t.hasUnresolvedCycle(visited));
}
}
export type Field = {
@@ -109,6 +117,10 @@ export class StructDesc {
return true;
}
hasUnresolvedCycle(visited: Type[]): boolean {
return this.fields.some(f => f.type.hasUnresolvedCycle(visited));
}
forEachField(cb: (name: string, type: Type) => void) {
const fields = this.fields;
for (let i = 0; i < fields.length; i++) {
@@ -120,14 +132,6 @@ export class StructDesc {
const f = findField(name, this.fields);
return f && f.type;
}
setField(name: string, type: Type) {
const f = findField(name, this.fields);
if (!f) {
throw new Error(`No such field "${name}"`);
}
f.type = type;
}
}
function findField(name: string, fields: Field[]): ?Field {
@@ -146,13 +150,36 @@ export function findFieldIndex(name: string, fields: Field[]): number {
return i === fields.length || fields[i].name !== name ? -1 : i;
}
export class CycleDesc {
level: number;
constructor(level: number) {
this.level = level;
}
get kind(): NomsKind {
return Kind.Cycle;
}
equals(other: TypeDesc): boolean {
return other instanceof CycleDesc && other.level === this.level;
}
hasUnresolvedCycle(visited: Type[]): boolean { // eslint-disable-line no-unused-vars
return true;
}
}
export class Type<T: TypeDesc> extends ValueBase {
_desc: T;
id: number;
serialization: ?Uint8Array;
constructor(desc: T) {
constructor(desc: T, id: number) {
super();
this._desc = desc;
this.id = id;
this.serialization = null;
}
get type(): Type {
@@ -176,6 +203,15 @@ export class Type<T: TypeDesc> extends ValueBase {
return this._desc.name;
}
hasUnresolvedCycle(visited: Type[]): boolean {
if (visited.indexOf(this) >= 0) {
return false;
}
visited.push(this);
return this._desc.hasUnresolvedCycle(visited);
}
get elemTypes(): Array<Type> {
invariant(this._desc instanceof CompoundDesc);
return this._desc.elemTypes;
@@ -186,123 +222,39 @@ export class Type<T: TypeDesc> extends ValueBase {
}
}
function buildType<T: TypeDesc>(desc: T): Type<T> {
return new Type(desc);
}
function makePrimitiveType(k: NomsKind): Type<PrimitiveDesc> {
return buildType(new PrimitiveDesc(k));
return new Type(new PrimitiveDesc(k), k);
}
export function makeListType(elemType: Type): Type<CompoundDesc> {
return buildType(new CompoundDesc(Kind.List, [elemType]));
return staticTypeCache.getCompoundType(Kind.List, elemType);
}
export function makeSetType(elemType: Type): Type<CompoundDesc> {
return buildType(new CompoundDesc(Kind.Set, [elemType]));
return staticTypeCache.getCompoundType(Kind.Set, elemType);
}
export function makeMapType(keyType: Type, valueType: Type): Type<CompoundDesc> {
return buildType(new CompoundDesc(Kind.Map, [keyType, valueType]));
return staticTypeCache.getCompoundType(Kind.Map, keyType, valueType);
}
export function makeRefType(elemType: Type): Type<CompoundDesc> {
return buildType(new CompoundDesc(Kind.Ref, [elemType]));
return staticTypeCache.getCompoundType(Kind.Ref, elemType);
}
export function makeStructType(name: string, fieldNames: string[], fieldTypes: Type[]):
Type<StructDesc> {
verifyStructName(name);
verifyFieldNames(fieldNames);
const fs = fieldNames.map((name, i) => {
const type = fieldTypes[i];
return {name, type};
});
return buildType(new StructDesc(name, fs));
return staticTypeCache.makeStructType(name, fieldNames, fieldTypes);
}
/**
* makeUnionType creates a new union type unless the elemTypes can be folded into a single non
* union type.
*/
export function makeUnionType(types: Type[]): Type {
types = flattenUnionTypes(types, Object.create(null));
if (types.length === 1) {
return types[0];
}
types.sort(compare);
return buildType(new CompoundDesc(Kind.Union, types));
return staticTypeCache.makeUnionType(types);
}
const fieldNameRe = /^[a-zA-Z][a-zA-Z0-9_]*$/;
function verifyFieldNames(names: string[]) {
if (names.length === 0) {
return;
}
let last = names[0];
verifyFieldName(last);
for (let i = 1; i < names.length; i++) {
verifyFieldName(names[i]);
if (last >= names[i]) {
throw new Error('Field names must be unique and ordered alphabetically');
}
last = names[i];
}
export function makeCycleType(level: number): Type {
return staticTypeCache.getCycleType(level);
}
function verifyName(name: string, kind: '' | ' field') {
if (!fieldNameRe.test(name)) {
throw new Error(`Invalid struct${kind} name: "${name}"`);
}
}
function verifyFieldName(name: string) {
verifyName(name, ' field');
}
function verifyStructName(name: string) {
if (name !== '') {
verifyName(name, '');
}
}
function flattenUnionTypes(types: Type[], seenTypes: {[key: Hash]: boolean}): Type[] {
if (types.length === 0) {
return types;
}
const newTypes = [];
for (let i = 0; i < types.length; i++) {
if (types[i].kind === Kind.Union) {
newTypes.push(...flattenUnionTypes(types[i].desc.elemTypes, seenTypes));
} else {
if (!seenTypes[types[i].hash]) {
seenTypes[types[i].hash] = true;
newTypes.push(types[i]);
}
}
}
return newTypes;
}
export const boolType = makePrimitiveType(Kind.Bool);
export const numberType = makePrimitiveType(Kind.Number);
export const stringType = makePrimitiveType(Kind.String);
export const blobType = makePrimitiveType(Kind.Blob);
export const typeType = makePrimitiveType(Kind.Type);
export const valueType = makePrimitiveType(Kind.Value);
export const refOfBlobType = makeRefType(blobType);
export const refOfValueType = makeRefType(valueType);
export const listOfValueType = makeListType(valueType);
export const setOfValueType = makeSetType(valueType);
export const mapOfValueType = makeMapType(valueType, valueType);
/**
* Gives the existing primitive Type value for a NomsKind.
*/
@@ -344,3 +296,10 @@ export function getTypeOfValue(v: Value): Type {
throw new Error('Unknown type');
}
}
export const boolType = makePrimitiveType(Kind.Bool);
export const numberType = makePrimitiveType(Kind.Number);
export const stringType = makePrimitiveType(Kind.String);
export const blobType = makePrimitiveType(Kind.Blob);
export const typeType = makePrimitiveType(Kind.Type);
export const valueType = makePrimitiveType(Kind.Value);

74
js/src/value-decoder.js
View File

@@ -11,16 +11,11 @@ import type Struct from './struct.js';
import type {NomsKind} from './noms-kind.js';
import {
getPrimitiveType,
makeListType,
makeMapType,
makeRefType,
makeSetType,
makeUnionType,
StructDesc,
Type,
} from './type.js';
import {OrderedKey, MetaTuple} from './meta-sequence.js';
import {invariant} from './assert.js';
import {invariant, notNull} from './assert.js';
import {isPrimitiveKind, kindToString, Kind} from './noms-kind.js';
import List, {ListLeafSequence} from './list.js';
import Map, {MapLeafSequence} from './map.js';
@@ -29,14 +24,17 @@ import {IndexedMetaSequence, OrderedMetaSequence} from './meta-sequence.js';
import type Value from './value.js';
import type {ValueReader} from './value-store.js';
import type {NomsReader} from './codec.js';
import type TypeCache from './type-cache.js';
export default class ValueDecoder {
_r: NomsReader;
_ds: ValueReader;
_tc: TypeCache;
constructor(r: NomsReader, ds: ValueReader) {
constructor(r: NomsReader, ds: ValueReader, tc: TypeCache) {
this._r = r;
this._ds = ds;
this._tc = tc;
}
readKind(): NomsKind {
@@ -49,31 +47,29 @@ export default class ValueDecoder {
return constructRef(t, hash, height);
}
readType(parentStructTypes: Type[]): Type {
readType(): Type {
const k = this.readKind();
switch (k) {
case Kind.List:
return makeListType(this.readType(parentStructTypes));
return this._tc.getCompoundType(k, this.readType());
case Kind.Map:
return makeMapType(this.readType(parentStructTypes),
this.readType(parentStructTypes));
return this._tc.getCompoundType(k, this.readType(), this.readType());
case Kind.Set:
return makeSetType(this.readType(parentStructTypes));
return this._tc.getCompoundType(k, this.readType());
case Kind.Ref:
return makeRefType(this.readType(parentStructTypes));
return this._tc.getCompoundType(k, this.readType());
case Kind.Struct:
return this.readStructType(parentStructTypes);
return this.readStructType();
case Kind.Union: {
const len = this._r.readUint32();
const types: Type[] = new Array(len);
for (let i = 0; i < len; i++) {
types[i] = this.readType(parentStructTypes);
types[i] = this.readType();
}
return makeUnionType(types);
return this._tc.getCompoundType(k, ...types);
}
case Kind.Cycle: {
const i = this._r.readUint32();
return parentStructTypes[parentStructTypes.length - 1 - i];
return this._tc.getCycleType(this._r.readUint32());
}
}
@@ -143,7 +139,7 @@ export default class ValueDecoder {
}
readValue(): any {
const t = this.readType([]);
const t = this.readType();
switch (t.kind) {
case Kind.Blob: {
const isMeta = this._r.readBool();
@@ -185,7 +181,7 @@ export default class ValueDecoder {
case Kind.Struct:
return this.readStruct(t);
case Kind.Type:
return this.readType([]);
return this.readType();
case Kind.Cycle:
case Kind.Union:
case Kind.Value:
@@ -208,24 +204,34 @@ export default class ValueDecoder {
return newStructWithType(type, values);
}
readStructType(parentStructTypes: Type[]): Type {
readCachedStructType(): Type {
let trie = notNull(this._tc.trieRoots.get(Kind.Struct)).traverse(this._r.readIdent(this._tc));
const count = this._r.readUint32();
for (let i = 0; i < count; i++) {
trie = trie.traverse(this._r.readIdent(this._tc));
trie = trie.traverse(this.readType().id);
}
return trie.t;
}
readStructType(): Type {
const pos = this._r.pos();
const t = this.readCachedStructType();
if (t) {
return t;
}
this._r.seek(pos);
const name = this._r.readString();
const count = this._r.readUint32();
const fields = new Array(count);
const desc = new StructDesc(name, fields);
const structType = new Type(desc);
parentStructTypes.push(structType);
const fieldNames = new Array(count);
const fieldTypes = new Array(count);
for (let i = 0; i < count; i++) {
const name = this._r.readString();
const type = this.readType(parentStructTypes);
// Mutate the already created structType since when looking for the cycle we compare
// by identity.
fields[i] = {name, type};
fieldNames[i] = this._r.readString();
fieldTypes[i] = this.readType();
}
parentStructTypes.pop();
return structType;
return this._tc.makeStructType(name, fieldNames, fieldTypes);
}
}

4
js/src/value-encoder.js
View File

@@ -131,7 +131,7 @@ export default class ValueEncoder {
writeValue(v: Value) {
const t = getTypeOfValue(v);
this.writeType(t, []);
this._w.appendType(t);
switch (t.kind) {
case Kind.Blob: {
invariant(v instanceof Blob,
@@ -205,7 +205,7 @@ export default class ValueEncoder {
case Kind.Type:
invariant(v instanceof Type,
() => `Failed to write Type. Invalid type: ${describeTypeOfValue(v)}`);
this.writeType(v, []);
this._w.appendType(v);
break;
case Kind.Struct:
invariant(v instanceof Struct,