mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-02-11 02:59:34 -06:00
Code Issues Packages Projects Releases 128 Wiki Activity

Revert "Js type cache (#2011)"

Browse Source 
This reverts commit 4e10ae63f4.
This commit is contained in:
Rafael Weinstein
2016-07-11 18:36:17 -07:00
parent 48ab930ed2
commit 8a5f3853b7
20 changed files with 267 additions and 666 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go/types/codec.go
View File

@@ -144,6 +144,8 @@ func (b *binaryNomsReader) readString() string {
return v
}
var createCount = uint64(0)
// Note: It's somewhat of a layering violation that a nomsReaders knows about a TypeCache. The reason why the code is structured this way is that the go compiler can stack-allocate the string which is created from the byte slice, which is a fairly large perf gain.
func (b *binaryNomsReader) readIdent(tc *TypeCache) uint32 {
size := b.readUint32()

2
go/types/encode_human_readable_test.go
View File

@@ -360,7 +360,7 @@ func TestRecursiveStruct(t *testing.T) {
})`, d)
}
func TestUnresolvedRecursiveStruct(t *testing.T) {
func TestUnserolvedRecursiveStruct(t *testing.T) {
// struct A {
// a: A
// b: Cycle<1> (unresolved)

10
go/types/type.go
View File

@@ -30,11 +30,7 @@ type Type struct {
const initialTypeBufferSize = 128
func newType(desc TypeDesc, id uint32) *Type {
return &Type{desc, &hash.Hash{}, id, nil}
}
func buildType(desc TypeDesc, id uint32) *Type {
t := newType(desc, id)
t := &Type{desc, &hash.Hash{}, id, nil}
if !t.HasUnresolvedCycle() {
serializeType(t)
}
@@ -137,6 +133,10 @@ func MakePrimitiveType(k NomsKind) *Type {
return nil
}
func makePrimitiveType(k NomsKind) *Type {
return newType(PrimitiveDesc(k), uint32(k))
}
func MakePrimitiveTypeByString(p string) *Type {
switch p {
case "Bool":

44
go/types/type_cache.go
View File

@@ -15,16 +15,13 @@ import (
type TypeCache struct {
identTable *identTable
trieRoots map[NomsKind]*typeTrie
typeBytes map[uint32][]byte
nextId uint32
mu *sync.Mutex
}
var staticTypeCache = NewTypeCache()
func makePrimitiveType(k NomsKind) *Type {
return buildType(PrimitiveDesc(k), uint32(k))
}
var BoolType = makePrimitiveType(BoolKind)
var NumberType = makePrimitiveType(NumberKind)
var StringType = makePrimitiveType(StringKind)
@@ -44,6 +41,7 @@ func NewTypeCache() *TypeCache {
CycleKind: newTypeTrie(),
UnionKind: newTypeTrie(),
},
map[uint32][]byte{},
256, // The first 255 type ids are reserved for the 8bit space of NomsKinds.
&sync.Mutex{},
}
@@ -70,7 +68,7 @@ func (tc *TypeCache) getCompoundType(kind NomsKind, elemTypes ...*Type) *Type {
}
if trie.t == nil {
trie.t = buildType(CompoundDesc{kind, elemTypes}, tc.nextTypeId())
trie.t = newType(CompoundDesc{kind, elemTypes}, tc.nextTypeId())
}
return trie.t
@@ -95,7 +93,7 @@ func (tc *TypeCache) makeStructType(name string, fieldNames []string, fieldTypes
i++
}
t := buildType(StructDesc{name, fs}, 0)
t := newType(StructDesc{name, fs}, 0)
if t.serialization == nil {
// HasUnresolvedCycle
t, _ = toUnresolvedType(t, tc, -1, nil)
@@ -124,7 +122,8 @@ func indexOfType(t *Type, tl []*Type) (uint32, bool) {
func toUnresolvedType(t *Type, tc *TypeCache, level int, parentStructTypes []*Type) (*Type, bool) {
i, found := indexOfType(t, parentStructTypes)
if found {
return newType(CycleDesc(uint32(len(parentStructTypes))-i-1), 0), true // This type is just a placeholder. It doesn't need an id
cycle := CycleDesc(uint32(len(parentStructTypes)) - i - 1)
return &Type{cycle, &hash.Hash{}, 0, nil}, true // This type is just a placeholder. It doesn't need an id
}
switch desc := t.Desc.(type) {
@@ -141,7 +140,7 @@ func toUnresolvedType(t *Type, tc *TypeCache, level int, parentStructTypes []*Ty
return t, false
}
return newType(CompoundDesc{t.Kind(), ts}, tc.nextTypeId()), true
return &Type{CompoundDesc{t.Kind(), ts}, &hash.Hash{}, tc.nextTypeId(), nil}, true
case StructDesc:
fs := make(fieldSlice, len(desc.fields))
didChange := false
@@ -156,7 +155,7 @@ func toUnresolvedType(t *Type, tc *TypeCache, level int, parentStructTypes []*Ty
return t, false
}
return newType(StructDesc{desc.Name, fs}, tc.nextTypeId()), true
return &Type{StructDesc{desc.Name, fs}, &hash.Hash{}, tc.nextTypeId(), nil}, true
case CycleDesc:
cycleLevel := int(desc)
return t, cycleLevel <= level // Only cycles which can be resolved in the current struct.
@@ -225,11 +224,11 @@ func (tc *TypeCache) makeUnionType(elemTypes ...*Type) *Type {
return tc.getCompoundType(UnionKind, ts...)
}
func (tc *TypeCache) getCycleType(level uint32) *Type {
func (tc *TypeCache) getCyclicType(level uint32) *Type {
trie := tc.trieRoots[CycleKind].Traverse(level)
if trie.t == nil {
trie.t = buildType(CycleDesc(level), tc.nextTypeId())
trie.t = newType(CycleDesc(level), tc.nextTypeId())
}
return trie.t
@@ -293,7 +292,7 @@ func MakeUnionType(elemTypes ...*Type) *Type {
func MakeCycleType(level uint32) *Type {
staticTypeCache.Lock()
defer staticTypeCache.Unlock()
return staticTypeCache.getCycleType(level)
return staticTypeCache.getCyclicType(level)
}
// All types in noms are created in a deterministic order. A typeTrie stores types within a typeCache and allows construction of a prexisting type to return the already existing one rather than allocate a new one.
@@ -307,12 +306,13 @@ func newTypeTrie() *typeTrie {
}
func (tct *typeTrie) Traverse(typeId uint32) *typeTrie {
next, ok := tct.entries[typeId]
if !ok {
// Insert edge
next = newTypeTrie()
tct.entries[typeId] = next
if t, ok := tct.entries[typeId]; ok {
return t
}
// Insert edge
next := newTypeTrie()
tct.entries[typeId] = next
return next
}
@@ -326,12 +326,12 @@ func newIdentTable() *identTable {
}
func (it *identTable) GetId(ident string) uint32 {
id, ok := it.entries[ident]
if !ok {
id = it.nextId
it.nextId++
it.entries[ident] = id
if id, ok := it.entries[ident]; ok {
return id
}
id := it.nextId
it.nextId++
it.entries[ident] = id
return id
}

7
go/types/type_cache_test.go
View File

@@ -68,15 +68,10 @@ func TestTypeCacheRef(t *testing.T) {
lst := MakeRefType(StringType)
lnt := MakeRefType(NumberType)
assert.False(lst == lnt)
assert.NotNil(lnt.serialization)
lst2 := MakeRefType(StringType)
assert.True(lst == lst2)
lnt2 := MakeRefType(NumberType)
assert.True(lnt == lnt2)
lbt3 := MakeRefType(BoolType)
assert.True(lbt == lbt3)
}
func TestTypeCacheStruct(t *testing.T) {

2
go/types/value_decoder.go
View File

@@ -53,7 +53,7 @@ func (r *valueDecoder) readType() *Type {
}
return r.tc.getCompoundType(UnionKind, elemTypes...)
case CycleKind:
return r.tc.getCycleType(r.readUint32())
return r.tc.getCyclicType(r.readUint32())
}
d.Chk.True(IsPrimitiveKind(k))

12
js/src/assert-type-test.js
View File

@@ -16,13 +16,16 @@ import type {Type} from './type.js';
import {
blobType,
boolType,
listOfValueType,
makeListType,
makeMapType,
makeRefType,
makeSetType,
makeStructType,
makeUnionType,
mapOfValueType,
numberType,
setOfValueType,
stringType,
typeType,
valueType,
@@ -86,7 +89,7 @@ suite('validate type', () => {
assertSubtype(listOfNumberType, l);
assertAll(listOfNumberType, l);
assertSubtype(makeListType(valueType), l);
assertSubtype(listOfValueType, l);
});
test('map', () => {
@@ -95,7 +98,7 @@ suite('validate type', () => {
assertSubtype(mapOfNumberToStringType, m);
assertAll(mapOfNumberToStringType, m);
assertSubtype(makeMapType(valueType, valueType), m);
assertSubtype(mapOfValueType, m);
});
test('set', () => {
@@ -104,7 +107,7 @@ suite('validate type', () => {
assertSubtype(setOfNumberType, s);
assertAll(setOfNumberType, s);
assertSubtype(makeSetType(valueType), s);
assertSubtype(setOfValueType, s);
});
test('type', () => {
@@ -230,10 +233,11 @@ suite('validate type', () => {
const t11 = makeStructType('Commit',
['parents', 'value'],
[
makeSetType(makeRefType(t1)),
numberType,
numberType, // placeholder
]
);
t11.desc.setField('parents', makeSetType(makeRefType(t11)));
assertSubtype(t11, c1);
const c2 = newStruct('Commit', {

4
js/src/blob-test.js
View File

@@ -4,7 +4,7 @@
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import {blobType, makeRefType} from './type.js';
import {blobType, refOfBlobType} from './type.js';
import {assert} from 'chai';
import Blob, {BlobReader, BlobWriter} from './blob.js';
import {suite, test, setup, teardown} from 'mocha';
@@ -118,7 +118,7 @@ suite('Blob', () => {
assertValueHash(expectHashStr, blob);
assertValueType(blobType, blob);
assert.strictEqual(length, blob.length);
assertChunkCountAndType(expectChunkCount, makeRefType(blobType), blob);
assertChunkCountAndType(expectChunkCount, refOfBlobType, blob);
await testRoundTripAndValidate(blob, async(b2) => {
await assertReadFull(buff, b2.getReader());

58
js/src/codec.js
View File

@@ -4,20 +4,18 @@
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import * as Bytes from './bytes.js';
import Chunk from './chunk.js';
import Hash, {sha1Size} from './hash.js';
import ValueDecoder from './value-decoder.js';
import ValueEncoder from './value-encoder.js';
import svarint from 'signed-varint';
import type Value from './value.js';
import type {Type} from './type.js';
import type {ValueReader, ValueWriter} from './value-store.js';
import {default as TypeCache, staticTypeCache} from './type-cache.js';
import {floatToIntExp, intExpToFloat} from './number-util.js';
import {invariant, notNull} from './assert.js';
import {invariant} from './assert.js';
import {setEncodeValue} from './get-hash.js';
import {setHash, ValueBase} from './value.js';
import type Value from './value.js';
import type {ValueReader, ValueWriter} from './value-store.js';
import * as Bytes from './bytes.js';
import {floatToIntExp, intExpToFloat} from './number-util.js';
import svarint from 'signed-varint';
export function encodeValue(v: Value, vw: ?ValueWriter): Chunk {
const w = new BinaryNomsWriter();
@@ -35,7 +33,7 @@ setEncodeValue(encodeValue);
export function decodeValue(chunk: Chunk, vr: ValueReader): Value {
const data = chunk.data;
const dec = new ValueDecoder(new BinaryNomsReader(data), vr, staticTypeCache);
const dec = new ValueDecoder(new BinaryNomsReader(data), vr);
const v = dec.readValue();
if (v instanceof ValueBase) {
@@ -45,21 +43,11 @@ export function decodeValue(chunk: Chunk, vr: ValueReader): Value {
return v;
}
function ensureTypeSerialization(t: Type) {
if (!t.serialization) {
const w = new BinaryNomsWriter();
const enc = new ValueEncoder(w, null);
enc.writeType(t, []);
t.serialization = w.data;
}
}
const maxUInt32 = Math.pow(2, 32);
const littleEndian = true;
export interface NomsReader {
pos(): number;
seek(pos: number): void;
readBytes(): Uint8Array;
readUint8(): number;
readUint32(): number;
@@ -67,7 +55,6 @@ export interface NomsReader {
readNumber(): number;
readBool(): boolean;
readString(): string;
readIdent(tc: TypeCache): number;
readHash(): Hash;
}
@@ -80,7 +67,6 @@ export interface NomsWriter {
writeBool(v:boolean): void;
writeString(v: string): void;
writeHash(h: Hash): void;
appendType(t: Type): void;
}
export class BinaryNomsReader {
@@ -94,14 +80,6 @@ export class BinaryNomsReader {
this.offset = 0;
}
pos(): number {
return this.offset;
}
seek(pos: number): void {
this.offset = pos;
}
readBytes(): Uint8Array {
const size = this.readUint32();
// Make a copy of the buffer to return
@@ -151,16 +129,6 @@ export class BinaryNomsReader {
return str;
}
readIdent(tc: TypeCache): number {
const str = this.readString(); // TODO: Figure out how to do this without allocating.
let id = tc.identTable.entries.get(str);
if (id === undefined) {
id = tc.identTable.getId(str);
}
return id;
}
readHash(): Hash {
// Make a copy of the data.
const digest = Bytes.slice(this.buff, this.offset, this.offset + sha1Size);
@@ -257,16 +225,4 @@ export class BinaryNomsWriter {
Bytes.copy(h.digest, this.buff, this.offset);
this.offset += sha1Size;
}
appendType(t: Type): void {
// Note: The JS & Go impls differ here. The Go impl eagerly serializes types as they are
// constructed. The JS does it lazily so as to avoid cyclic package dependencies.
ensureTypeSerialization(t);
const data = notNull(t.serialization);
const size = data.byteLength;
this.ensureCapacity(size);
Bytes.copy(data, this.buff, this.offset);
this.offset += size;
}
}

16
js/src/commit.js
View File

@@ -5,28 +5,16 @@
// http://www.apache.org/licenses/LICENSE-2.0
import {invariant} from './assert.js';
import {getDatasTypes} from './database.js';
import Struct from './struct.js';
import type Value from './value.js';
import type Ref from './ref.js';
import Set from './set.js';
import {
makeCycleType,
makeRefType,
makeStructType,
makeSetType,
valueType,
} from './type.js';
export const commitType = makeStructType('Commit',
['parents', 'value'],
[
makeSetType(makeRefType(makeCycleType(0))),
valueType,
]
);
export default class Commit<T: Value> extends Struct {
constructor(value: T, parents: Set<Ref<Commit>> = new Set()) {
const {commitType} = getDatasTypes();
super(commitType, [parents, value]);
}

45
js/src/database.js
View File

@@ -12,9 +12,54 @@ import type Value from './value.js';
import type {RootTracker} from './chunk-store.js';
import ValueStore from './value-store.js';
import type {BatchStore} from './batch-store.js';
import {
makeRefType,
makeStructType,
makeSetType,
makeMapType,
Type,
stringType,
valueType,
} from './type.js';
import Commit from './commit.js';
import {equals} from './compare.js';
type DatasTypes = {
commitType: Type,
commitSetType: Type,
refOfCommitType: Type,
commitMapType: Type,
};
let datasTypes: DatasTypes;
export function getDatasTypes(): DatasTypes {
if (!datasTypes) {
// struct Commit {
// value: Value
// parents: Set<Ref<Commit>>
// }
const commitType = makeStructType('Commit',
['parents', 'value'],
[
valueType,
valueType,
]
);
const refOfCommitType = makeRefType(commitType);
const commitSetType = makeSetType(refOfCommitType);
commitType.desc.setField('parents', commitSetType);
const commitMapType = makeMapType(stringType, refOfCommitType);
datasTypes = {
commitType,
refOfCommitType,
commitSetType,
commitMapType,
};
}
return datasTypes;
}
export default class Database {
_vs: ValueStore;
_rt: RootTracker;

51
js/src/encode-human-readable-test.js
View File

@@ -7,13 +7,11 @@
import {assert} from 'chai';
import {suite, test} from 'mocha';
import {invariant, notNull} from './assert.js';
import {TypeWriter} from './encode-human-readable.js';
import {
blobType,
boolType,
numberType,
makeCycleType,
makeRefType,
makeListType,
makeMapType,
@@ -21,7 +19,7 @@ import {
makeStructType,
makeUnionType,
stringType,
StructDesc,
valueType,
Type,
} from './type.js';
@@ -88,17 +86,23 @@ suite('Encode human readable types', () => {
const a = makeStructType('A',
['b', 'c', 'd'],
[
makeCycleType(0),
makeListType(makeCycleType(0)),
makeStructType('D',
['e', 'f'],
[
makeCycleType(0),
makeCycleType(1),
]
),
valueType, // placeholder
valueType, // placeholder
valueType, // placeholder
]
);
const d = makeStructType('D',
['e', 'f'],
[
valueType, // placeholder
a,
]
);
a.desc.setField('b', a);
a.desc.setField('d', d);
d.desc.setField('e', d);
d.desc.setField('f', a);
a.desc.setField('c', makeListType(a));
assertWriteType(`struct A {
b: Cycle<0>,
@@ -109,9 +113,6 @@ suite('Encode human readable types', () => {
},
}`, a);
invariant(a.desc instanceof StructDesc);
const d = notNull(a.desc.getField('d'));
assertWriteType(`struct D {
e: Cycle<0>,
f: struct A {
@@ -121,24 +122,4 @@ suite('Encode human readable types', () => {
},
}`, d);
});
test('recursive unresolved struct', () => {
// struct A {
// a: A
// b: Cycle<1>
// }
const a = makeStructType('A',
['a', 'b'],
[
makeCycleType(0),
makeCycleType(1),
]
);
assertWriteType(`struct A {
a: Cycle<0>,
b: Cycle<1>,
}`, a);
});
});

5
js/src/encode-human-readable.js
View File

@@ -4,7 +4,7 @@
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import {getTypeOfValue, CompoundDesc, CycleDesc} from './type.js';
import {getTypeOfValue, CompoundDesc} from './type.js';
import type {Type} from './type.js';
import {Kind, kindToString} from './noms-kind.js';
import type {NomsKind} from './noms-kind.js';
@@ -109,9 +109,6 @@ export class TypeWriter {
this._writeStructType(t, parentStructTypes);
break;
case Kind.Cycle:
invariant(t.desc instanceof CycleDesc);
this._w.write(`Cycle<${t.desc.level}>`);
break;
default:
throw new Error('unreachable');
}

39
js/src/encoding-test.js
View File

@@ -34,18 +34,16 @@ import {
} from './meta-sequence.js';
import {
boolType,
blobType,
makeCycleType,
makeListType,
makeMapType,
makeRefType,
makeSetType,
makeStructType,
numberType,
refOfBlobType,
stringType,
typeType,
} from './type.js';
import {staticTypeCache} from './type-cache.js';
function assertRoundTrips(v: Value) {
const db = new TestDatabase();
@@ -123,14 +121,6 @@ suite('Encoding', () => {
this.i = 0;
}
pos(): number {
return this.i;
}
seek(pos: number): void {
this.i = pos;
}
atEnd(): boolean {
return this.i === this.a.length;
}
@@ -182,16 +172,6 @@ suite('Encoding', () => {
return v;
}
readIdent(tc: TypeCache): number {
const s = this.readString();
let id = tc.identTable.entries.get(s);
if (id === undefined) {
id = tc.identTable.getId(s);
}
return id;
}
readHash(): Hash {
return Hash.parse(this.readString());
}
@@ -241,11 +221,6 @@ suite('Encoding', () => {
this.writeString(h.toString());
}
appendType(t: Type): void {
const enc = new ValueEncoder(this, null);
enc.writeType(t, []);
}
toArray(): any[] {
return this.a;
}
@@ -271,7 +246,7 @@ suite('Encoding', () => {
assert.deepEqual(encoding, w.toArray());
const r = new TestReader(encoding);
const dec = new ValueDecoder(r, null, staticTypeCache);
const dec = new ValueDecoder(r, null);
const v2 = dec.readValue();
assert.isTrue(equals(v, v2));
}
@@ -365,9 +340,9 @@ suite('Encoding', () => {
uint8(RefKind), uint8(BlobKind), r3.toString(), uint64(33), uint8(NumberKind), float64(60), uint64(60),
],
Blob.fromSequence(newBlobMetaSequence(null, [
new MetaTuple(constructRef(makeRefType(blobType), r1, 11), new OrderedKey(20), 20, null),
new MetaTuple(constructRef(makeRefType(blobType), r2, 22), new OrderedKey(40), 40, null),
new MetaTuple(constructRef(makeRefType(blobType), r3, 33), new OrderedKey(60), 60, null),
new MetaTuple(constructRef(refOfBlobType, r1, 11), new OrderedKey(20), 20, null),
new MetaTuple(constructRef(refOfBlobType, r2, 22), new OrderedKey(40), 40, null),
new MetaTuple(constructRef(refOfBlobType, r3, 33), new OrderedKey(60), 60, null),
]))
);
});
@@ -523,10 +498,12 @@ suite('Encoding', () => {
const structType = makeStructType('A6',
['cs', 'v'],
[
makeListType(makeCycleType(0)),
numberType, // placeholder
numberType,
]
);
const listType = makeListType(structType);
structType.desc.setField('cs', listType);
assertEncoding([
uint8(StructKind), 'A6', uint32(2) /* len */, 'cs', uint8(ListKind), uint8(CycleKind), uint32(0), 'v', uint8(NumberKind),

8
js/src/struct-test.js
View File

@@ -16,11 +16,11 @@ import Struct, {
import {assert} from 'chai';
import {
boolType,
makeCycleType,
makeListType,
makeStructType,
numberType,
stringType,
valueType,
} from './type.js';
import {suite, test} from 'mocha';
import {equals} from './compare.js';
@@ -126,11 +126,13 @@ suite('Struct', () => {
['b', 'l'],
[
boolType,
makeListType(makeCycleType(0)),
valueType, // placeholder
]
);
const listType = makeListType(type);
type.desc.setField('l', listType);
const emptyList = new List([]);
const emptyList = new List([], listType);
newStructWithType(type, [true, emptyList]);
newStructWithType(type,
[

124
js/src/type-cache-test.js
View File

@@ -1,124 +0,0 @@
// @flow
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import {assert} from 'chai';
import {suite, test} from 'mocha';
import {
boolType,
makeCycleType,
makeListType,
makeRefType,
makeSetType,
makeStructType,
makeUnionType,
numberType,
stringType,
} from './type.js';
suite('TypeCache', () => {
test('list', () => {
const lbt = makeListType(boolType);
const lbt2 = makeListType(boolType);
assert.strictEqual(lbt, lbt2);
const lst = makeListType(stringType);
const lnt = makeListType(numberType);
assert.notEqual(lst, lnt);
const lst2 = makeListType(stringType);
assert.strictEqual(lst, lst2);
const lnt2 = makeListType(numberType);
assert.strictEqual(lnt, lnt2);
const lbt3 = makeListType(boolType);
assert.strictEqual(lbt, lbt3);
});
test('set', () => {
const lbt = makeSetType(boolType);
const lbt2 = makeSetType(boolType);
assert.strictEqual(lbt, lbt2);
const lst = makeSetType(stringType);
const lnt = makeSetType(numberType);
assert.notEqual(lst, lnt);
const lst2 = makeSetType(stringType);
assert.strictEqual(lst, lst2);
const lnt2 = makeSetType(numberType);
assert.strictEqual(lnt, lnt2);
const lbt3 = makeSetType(boolType);
assert.strictEqual(lbt, lbt3);
});
test('ref', () => {
const lbt = makeRefType(boolType);
const lbt2 = makeRefType(boolType);
assert.strictEqual(lbt, lbt2);
const lst = makeRefType(stringType);
const lnt = makeRefType(numberType);
assert.notEqual(lst, lnt);
const lst2 = makeRefType(stringType);
assert.strictEqual(lst, lst2);
const lnt2 = makeRefType(numberType);
assert.strictEqual(lnt, lnt2);
const lbt3 = makeRefType(boolType);
assert.strictEqual(lbt, lbt3);
});
test('struct', () => {
const st = makeStructType('Foo',
['bar', 'foo'],
[stringType, numberType]
);
const st2 = makeStructType('Foo',
['bar', 'foo'],
[stringType, numberType]
);
assert.strictEqual(st, st2);
});
test('union', () => {
let ut = makeUnionType([numberType]);
let ut2 = makeUnionType([numberType]);
assert.strictEqual(ut, ut2);
assert.strictEqual(ut2, numberType);
ut = makeUnionType([numberType, stringType]);
ut2 = makeUnionType([stringType, numberType]);
assert.strictEqual(ut, ut2);
ut = makeUnionType([stringType, boolType, numberType]);
ut2 = makeUnionType([numberType, stringType, boolType]);
assert.strictEqual(ut, ut2);
});
test('Cyclic Struct', () => {
const st = makeStructType('Foo',
['foo'],
[
makeRefType(makeCycleType(0)),
]);
assert.isFalse(st.hasUnresolvedCycle([]));
assert.strictEqual(st, st.desc.fields[0].type.desc.elemTypes[0]);
const st2 = makeStructType('Foo',
['foo'],
[
makeRefType(makeCycleType(0)),
]);
assert.isFalse(st2.hasUnresolvedCycle([]));
assert.strictEqual(st, st2);
});
});

257
js/src/type-cache.js
View File

@@ -1,257 +0,0 @@
// @flow
// Copyright 2016 The Noms Authors. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import type Hash from './hash.js';
import type {NomsKind} from './noms-kind.js';
import {Kind} from './noms-kind.js';
import {CompoundDesc, CycleDesc, StructDesc, Type} from './type.js';
import {compare} from './compare.js';
import {notNull} from './assert.js';
class IdentTable {
entries: Map<string, number>;
nextId: number;
constructor() {
this.entries = new Map();
this.nextId = 0;
}
getId(ident: string): number {
let id = this.entries.get(ident);
if (id === undefined) {
id = this.nextId++;
this.entries.set(ident, id);
}
return id;
}
}
class TypeTrie {
t: Type;
entries: Map<number, TypeTrie>;
constructor() {
this.entries = new Map();
}
traverse(typeId: number): TypeTrie {
let next = this.entries.get(typeId);
if (!next) {
// Insert edge
next = new TypeTrie();
this.entries.set(typeId, next);
}
return next;
}
}
export default class TypeCache {
identTable: IdentTable;
trieRoots: Map<NomsKind, TypeTrie>;
nextId: number;
constructor() {
this.identTable = new IdentTable();
this.trieRoots = new Map();
this.trieRoots.set(Kind.List, new TypeTrie());
this.trieRoots.set(Kind.Set, new TypeTrie());
this.trieRoots.set(Kind.Ref, new TypeTrie());
this.trieRoots.set(Kind.Map, new TypeTrie());
this.trieRoots.set(Kind.Struct, new TypeTrie());
this.trieRoots.set(Kind.Cycle, new TypeTrie());
this.trieRoots.set(Kind.Union, new TypeTrie());
this.nextId = 256; // The first 255 type ids are reserved for the 8bit space of NomsKinds.
}
nextTypeId(): number {
return this.nextId++;
}
getCompoundType(kind: NomsKind, ...elemTypes: Type[]): Type {
let trie = notNull(this.trieRoots.get(kind));
elemTypes.forEach(t => trie = notNull(trie).traverse(t.id));
if (!notNull(trie).t) {
trie.t = new Type(new CompoundDesc(kind, elemTypes), this.nextTypeId());
}
return trie.t;
}
makeStructType(name: string, fieldNames: string[], fieldTypes: Type[]): Type<StructDesc> {
if (fieldNames.length !== fieldTypes.length) {
throw new Error('Field names and types must be of equal length');
}
verifyStructName(name);
verifyFieldNames(fieldNames);
let trie = notNull(this.trieRoots.get(Kind.Struct)).traverse(this.identTable.getId(name));
fieldNames.forEach((fn, i) => {
const ft = fieldTypes[i];
trie = trie.traverse(this.identTable.getId(fn));
trie = trie.traverse(ft.id);
});
if (trie.t === undefined) {
const fs = fieldNames.map((name, i) => {
const type = fieldTypes[i];
return {name, type};
});
let t = new Type(new StructDesc(name, fs), 0);
if (t.hasUnresolvedCycle([])) {
t = notNull(this._toUnresolvedType(t, -1));
t = this._resolveStructCycles(t);
}
t.id = this.nextTypeId();
trie.t = t;
}
return trie.t;
}
_toUnresolvedType(t: Type, level: number, parentStructTypes: Type[] = []): ?Type {
const idx = parentStructTypes.indexOf(t);
if (idx >= 0) {
// This type is just a placeholder. It doesn't need an id
return new Type(new CycleDesc(parentStructTypes.length - idx - 1), 0);
}
const desc = t.desc;
if (desc instanceof CompoundDesc) {
const elemTypes = desc.elemTypes;
let sts = elemTypes.map(t => this._toUnresolvedType(t, level, parentStructTypes));
if (sts.some(t => t)) {
sts = sts.map((t, i) => t ? t : elemTypes[i]);
return new Type(new CompoundDesc(t.kind, sts), this.nextTypeId());
}
return;
}
if (desc instanceof StructDesc) {
const fields = desc.fields;
const outerType = t; // TODO: Stupid babel bug.
const sts = fields.map(f => {
parentStructTypes.push(outerType);
const t = this._toUnresolvedType(f.type, level + 1, parentStructTypes);
parentStructTypes.pop();
return t ? t : undefined;
});
if (sts.some(t => t)) {
const fs = sts.map((t, i) => ({name: fields[i].name, type: t ? t : fields[i].type}));
return new Type(new StructDesc(desc.name, fs), this.nextTypeId());
}
return;
}
if (desc instanceof CycleDesc) {
const cycleLevel = desc.level;
return cycleLevel <= level ? t : undefined;
}
}
_resolveStructCycles(t: Type, parentStructTypes: Type[] = []): Type {
const desc = t.desc;
if (desc instanceof CompoundDesc) {
for (let i = 0; i < desc.elemTypes.length; i++) {
desc.elemTypes[i] = this._resolveStructCycles(desc.elemTypes[i], parentStructTypes);
}
} else if (desc instanceof StructDesc) {
for (let i = 0; i < desc.fields.length; i++) {
parentStructTypes.push(t);
desc.fields[i].type = this._resolveStructCycles(desc.fields[i].type, parentStructTypes);
parentStructTypes.pop();
}
} else if (desc instanceof CycleDesc) {
const level = desc.level;
if (level < parentStructTypes.length) {
return parentStructTypes[parentStructTypes.length - 1 - level];
}
}
return t;
}
// Creates a new union type unless the elemTypes can be folded into a single non union type.
makeUnionType(types: Type[]): Type {
types = flattenUnionTypes(types, Object.create(null));
if (types.length === 1) {
return types[0];
}
types.sort(compare);
return this.getCompoundType(Kind.Union, ...types);
}
getCycleType(level: number): Type {
const trie = notNull(this.trieRoots.get(Kind.Cycle)).traverse(level);
if (trie.t === undefined) {
trie.t = new Type(new CycleDesc(level), this.nextTypeId());
}
return trie.t;
}
}
export const staticTypeCache = new TypeCache();
function flattenUnionTypes(types: Type[], seenTypes: {[key: Hash]: boolean}): Type[] {
if (types.length === 0) {
return types;
}
const newTypes = [];
for (let i = 0; i < types.length; i++) {
if (types[i].kind === Kind.Union) {
newTypes.push(...flattenUnionTypes(types[i].desc.elemTypes, seenTypes));
} else {
if (!seenTypes[types[i].hash]) {
seenTypes[types[i].hash] = true;
newTypes.push(types[i]);
}
}
}
return newTypes;
}
const fieldNameRe = /^[a-zA-Z][a-zA-Z0-9_]*$/;
function verifyFieldNames(names: string[]) {
if (names.length === 0) {
return;
}
let last = names[0];
verifyFieldName(last);
for (let i = 1; i < names.length; i++) {
verifyFieldName(names[i]);
if (last >= names[i]) {
throw new Error('Field names must be unique and ordered alphabetically');
}
last = names[i];
}
}
function verifyName(name: string, kind: '' | ' field') {
if (!fieldNameRe.test(name)) {
throw new Error(`Invalid struct${kind} name: '${name}'`);
}
}
function verifyFieldName(name: string) {
verifyName(name, ' field');
}
function verifyStructName(name: string) {
if (name !== '') {
verifyName(name, '');
}
}

169
js/src/type.js
View File

@@ -4,21 +4,20 @@
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
import type Hash from './hash.js';
import Ref from './ref.js';
import type {NomsKind} from './noms-kind.js';
import {invariant} from './assert.js';
import {isPrimitiveKind, Kind} from './noms-kind.js';
import {ValueBase} from './value.js';
import type Value from './value.js';
import {equals} from './compare.js';
import {compare, equals} from './compare.js';
import {describeType} from './encode-human-readable.js';
import search from './binary-search.js';
import {staticTypeCache} from './type-cache.js';
export interface TypeDesc {
kind: NomsKind;
equals(other: TypeDesc): boolean;
hasUnresolvedCycle(visited: Type[]): boolean;
}
export class PrimitiveDesc {
@@ -31,10 +30,6 @@ export class PrimitiveDesc {
equals(other: TypeDesc): boolean {
return other instanceof PrimitiveDesc && other.kind === this.kind;
}
hasUnresolvedCycle(visited: Type[]): boolean { // eslint-disable-line no-unused-vars
return false;
}
}
export class CompoundDesc {
@@ -46,6 +41,7 @@ export class CompoundDesc {
this.elemTypes = elemTypes;
}
equals(other: TypeDesc): boolean {
if (other instanceof CompoundDesc) {
if (this.kind !== other.kind || this.elemTypes.length !== other.elemTypes.length) {
@@ -63,10 +59,6 @@ export class CompoundDesc {
return false;
}
hasUnresolvedCycle(visited: Type[]): boolean {
return this.elemTypes.some(t => t.hasUnresolvedCycle(visited));
}
}
export type Field = {
@@ -117,10 +109,6 @@ export class StructDesc {
return true;
}
hasUnresolvedCycle(visited: Type[]): boolean {
return this.fields.some(f => f.type.hasUnresolvedCycle(visited));
}
forEachField(cb: (name: string, type: Type) => void) {
const fields = this.fields;
for (let i = 0; i < fields.length; i++) {
@@ -132,6 +120,14 @@ export class StructDesc {
const f = findField(name, this.fields);
return f && f.type;
}
setField(name: string, type: Type) {
const f = findField(name, this.fields);
if (!f) {
throw new Error(`No such field "${name}"`);
}
f.type = type;
}
}
function findField(name: string, fields: Field[]): ?Field {
@@ -150,36 +146,13 @@ export function findFieldIndex(name: string, fields: Field[]): number {
return i === fields.length || fields[i].name !== name ? -1 : i;
}
export class CycleDesc {
level: number;
constructor(level: number) {
this.level = level;
}
get kind(): NomsKind {
return Kind.Cycle;
}
equals(other: TypeDesc): boolean {
return other instanceof CycleDesc && other.level === this.level;
}
hasUnresolvedCycle(visited: Type[]): boolean { // eslint-disable-line no-unused-vars
return true;
}
}
export class Type<T: TypeDesc> extends ValueBase {
_desc: T;
id: number;
serialization: ?Uint8Array;
constructor(desc: T, id: number) {
constructor(desc: T) {
super();
this._desc = desc;
this.id = id;
this.serialization = null;
}
get type(): Type {
@@ -203,15 +176,6 @@ export class Type<T: TypeDesc> extends ValueBase {
return this._desc.name;
}
hasUnresolvedCycle(visited: Type[]): boolean {
if (visited.indexOf(this) >= 0) {
return false;
}
visited.push(this);
return this._desc.hasUnresolvedCycle(visited);
}
get elemTypes(): Array<Type> {
invariant(this._desc instanceof CompoundDesc);
return this._desc.elemTypes;
@@ -222,39 +186,123 @@ export class Type<T: TypeDesc> extends ValueBase {
}
}
function buildType<T: TypeDesc>(desc: T): Type<T> {
return new Type(desc);
}
function makePrimitiveType(k: NomsKind): Type<PrimitiveDesc> {
return new Type(new PrimitiveDesc(k), k);
return buildType(new PrimitiveDesc(k));
}
export function makeListType(elemType: Type): Type<CompoundDesc> {
return staticTypeCache.getCompoundType(Kind.List, elemType);
return buildType(new CompoundDesc(Kind.List, [elemType]));
}
export function makeSetType(elemType: Type): Type<CompoundDesc> {
return staticTypeCache.getCompoundType(Kind.Set, elemType);
return buildType(new CompoundDesc(Kind.Set, [elemType]));
}
export function makeMapType(keyType: Type, valueType: Type): Type<CompoundDesc> {
return staticTypeCache.getCompoundType(Kind.Map, keyType, valueType);
return buildType(new CompoundDesc(Kind.Map, [keyType, valueType]));
}
export function makeRefType(elemType: Type): Type<CompoundDesc> {
return staticTypeCache.getCompoundType(Kind.Ref, elemType);
return buildType(new CompoundDesc(Kind.Ref, [elemType]));
}
export function makeStructType(name: string, fieldNames: string[], fieldTypes: Type[]):
Type<StructDesc> {
return staticTypeCache.makeStructType(name, fieldNames, fieldTypes);
verifyStructName(name);
verifyFieldNames(fieldNames);
const fs = fieldNames.map((name, i) => {
const type = fieldTypes[i];
return {name, type};
});
return buildType(new StructDesc(name, fs));
}
/**
* makeUnionType creates a new union type unless the elemTypes can be folded into a single non
* union type.
*/
export function makeUnionType(types: Type[]): Type {
return staticTypeCache.makeUnionType(types);
types = flattenUnionTypes(types, Object.create(null));
if (types.length === 1) {
return types[0];
}
types.sort(compare);
return buildType(new CompoundDesc(Kind.Union, types));
}
export function makeCycleType(level: number): Type {
return staticTypeCache.getCycleType(level);
const fieldNameRe = /^[a-zA-Z][a-zA-Z0-9_]*$/;
function verifyFieldNames(names: string[]) {
if (names.length === 0) {
return;
}
let last = names[0];
verifyFieldName(last);
for (let i = 1; i < names.length; i++) {
verifyFieldName(names[i]);
if (last >= names[i]) {
throw new Error('Field names must be unique and ordered alphabetically');
}
last = names[i];
}
}
function verifyName(name: string, kind: '' | ' field') {
if (!fieldNameRe.test(name)) {
throw new Error(`Invalid struct${kind} name: "${name}"`);
}
}
function verifyFieldName(name: string) {
verifyName(name, ' field');
}
function verifyStructName(name: string) {
if (name !== '') {
verifyName(name, '');
}
}
function flattenUnionTypes(types: Type[], seenTypes: {[key: Hash]: boolean}): Type[] {
if (types.length === 0) {
return types;
}
const newTypes = [];
for (let i = 0; i < types.length; i++) {
if (types[i].kind === Kind.Union) {
newTypes.push(...flattenUnionTypes(types[i].desc.elemTypes, seenTypes));
} else {
if (!seenTypes[types[i].hash]) {
seenTypes[types[i].hash] = true;
newTypes.push(types[i]);
}
}
}
return newTypes;
}
export const boolType = makePrimitiveType(Kind.Bool);
export const numberType = makePrimitiveType(Kind.Number);
export const stringType = makePrimitiveType(Kind.String);
export const blobType = makePrimitiveType(Kind.Blob);
export const typeType = makePrimitiveType(Kind.Type);
export const valueType = makePrimitiveType(Kind.Value);
export const refOfBlobType = makeRefType(blobType);
export const refOfValueType = makeRefType(valueType);
export const listOfValueType = makeListType(valueType);
export const setOfValueType = makeSetType(valueType);
export const mapOfValueType = makeMapType(valueType, valueType);
/**
* Gives the existing primitive Type value for a NomsKind.
*/
@@ -296,10 +344,3 @@ export function getTypeOfValue(v: Value): Type {
throw new Error('Unknown type');
}
}
export const boolType = makePrimitiveType(Kind.Bool);
export const numberType = makePrimitiveType(Kind.Number);
export const stringType = makePrimitiveType(Kind.String);
export const blobType = makePrimitiveType(Kind.Blob);
export const typeType = makePrimitiveType(Kind.Type);
export const valueType = makePrimitiveType(Kind.Value);

74
js/src/value-decoder.js
View File

@@ -11,11 +11,16 @@ import type Struct from './struct.js';
import type {NomsKind} from './noms-kind.js';
import {
getPrimitiveType,
makeListType,
makeMapType,
makeRefType,
makeSetType,
makeUnionType,
StructDesc,
Type,
} from './type.js';
import {OrderedKey, MetaTuple} from './meta-sequence.js';
import {invariant, notNull} from './assert.js';
import {invariant} from './assert.js';
import {isPrimitiveKind, kindToString, Kind} from './noms-kind.js';
import List, {ListLeafSequence} from './list.js';
import Map, {MapLeafSequence} from './map.js';
@@ -24,17 +29,14 @@ import {IndexedMetaSequence, OrderedMetaSequence} from './meta-sequence.js';
import type Value from './value.js';
import type {ValueReader} from './value-store.js';
import type {NomsReader} from './codec.js';
import type TypeCache from './type-cache.js';
export default class ValueDecoder {
_r: NomsReader;
_ds: ValueReader;
_tc: TypeCache;
constructor(r: NomsReader, ds: ValueReader, tc: TypeCache) {
constructor(r: NomsReader, ds: ValueReader) {
this._r = r;
this._ds = ds;
this._tc = tc;
}
readKind(): NomsKind {
@@ -47,29 +49,31 @@ export default class ValueDecoder {
return constructRef(t, hash, height);
}
readType(): Type {
readType(parentStructTypes: Type[]): Type {
const k = this.readKind();
switch (k) {
case Kind.List:
return this._tc.getCompoundType(k, this.readType());
return makeListType(this.readType(parentStructTypes));
case Kind.Map:
return this._tc.getCompoundType(k, this.readType(), this.readType());
return makeMapType(this.readType(parentStructTypes),
this.readType(parentStructTypes));
case Kind.Set:
return this._tc.getCompoundType(k, this.readType());
return makeSetType(this.readType(parentStructTypes));
case Kind.Ref:
return this._tc.getCompoundType(k, this.readType());
return makeRefType(this.readType(parentStructTypes));
case Kind.Struct:
return this.readStructType();
return this.readStructType(parentStructTypes);
case Kind.Union: {
const len = this._r.readUint32();
const types: Type[] = new Array(len);
for (let i = 0; i < len; i++) {
types[i] = this.readType();
types[i] = this.readType(parentStructTypes);
}
return this._tc.getCompoundType(k, ...types);
return makeUnionType(types);
}
case Kind.Cycle: {
return this._tc.getCycleType(this._r.readUint32());
const i = this._r.readUint32();
return parentStructTypes[parentStructTypes.length - 1 - i];
}
}
@@ -139,7 +143,7 @@ export default class ValueDecoder {
}
readValue(): any {
const t = this.readType();
const t = this.readType([]);
switch (t.kind) {
case Kind.Blob: {
const isMeta = this._r.readBool();
@@ -181,7 +185,7 @@ export default class ValueDecoder {
case Kind.Struct:
return this.readStruct(t);
case Kind.Type:
return this.readType();
return this.readType([]);
case Kind.Cycle:
case Kind.Union:
case Kind.Value:
@@ -204,34 +208,24 @@ export default class ValueDecoder {
return newStructWithType(type, values);
}
readCachedStructType(): Type {
let trie = notNull(this._tc.trieRoots.get(Kind.Struct)).traverse(this._r.readIdent(this._tc));
const count = this._r.readUint32();
for (let i = 0; i < count; i++) {
trie = trie.traverse(this._r.readIdent(this._tc));
trie = trie.traverse(this.readType().id);
}
return trie.t;
}
readStructType(): Type {
const pos = this._r.pos();
const t = this.readCachedStructType();
if (t) {
return t;
}
this._r.seek(pos);
readStructType(parentStructTypes: Type[]): Type {
const name = this._r.readString();
const count = this._r.readUint32();
const fieldNames = new Array(count);
const fieldTypes = new Array(count);
const fields = new Array(count);
const desc = new StructDesc(name, fields);
const structType = new Type(desc);
parentStructTypes.push(structType);
for (let i = 0; i < count; i++) {
fieldNames[i] = this._r.readString();
fieldTypes[i] = this.readType();
const name = this._r.readString();
const type = this.readType(parentStructTypes);
// Mutate the already created structType since when looking for the cycle we compare
// by identity.
fields[i] = {name, type};
}
return this._tc.makeStructType(name, fieldNames, fieldTypes);
parentStructTypes.pop();
return structType;
}
}

4
js/src/value-encoder.js
View File

@@ -131,7 +131,7 @@ export default class ValueEncoder {
writeValue(v: Value) {
const t = getTypeOfValue(v);
this._w.appendType(t);
this.writeType(t, []);
switch (t.kind) {
case Kind.Blob: {
invariant(v instanceof Blob,
@@ -205,7 +205,7 @@ export default class ValueEncoder {
case Kind.Type:
invariant(v instanceof Type,
() => `Failed to write Type. Invalid type: ${describeTypeOfValue(v)}`);
this._w.appendType(v);
this.writeType(v, []);
break;
case Kind.Struct:
invariant(v instanceof Struct,