From fe01f333a1424ec121fd266df8db9d3ad910d16b Mon Sep 17 00:00:00 2001 From: Rafael Weinstein Date: Thu, 10 Dec 2015 15:03:32 -0800 Subject: [PATCH] JS Support for Compound Objects --- clients/pitchmap/ui/heat_map.js | 2 +- clients/splore/main.js | 48 ++++++++-- js/src/compound_list.js | 12 --- js/src/decode.js | 71 +++++--------- js/src/decode_test.js | 111 +++++++++++----------- js/src/encode.js | 65 ++++++------- js/src/encode_test.js | 46 ++++----- js/src/get_ref.js | 13 ++- js/src/indexed_sequence.js | 37 ++++++++ js/src/list.js | 69 ++++++++++++++ js/src/list_test.js | 76 +++++++++++++++ js/src/map.js | 80 ++++++++++++++++ js/src/map_test.js | 121 ++++++++++++++++++++++++ js/src/meta_sequence.js | 75 ++++++++++----- js/src/noms.js | 9 +- js/src/ordered_sequence.js | 87 +++++++++++++++++ js/src/package.js | 2 +- js/src/primitives.js | 26 ++++++ js/src/read_value.js | 22 +++++ js/src/sequence.js | 161 ++++++++++++++++++++++++++++++++ js/src/set.js | 58 ++++++++++++ js/src/set_test.js | 140 +++++++++++++++++++++++++++ js/src/struct.js | 18 +--- js/src/type.js | 30 +++++- js/src/type_test.js | 2 +- js/src/value.js | 69 ++++++++++++++ 26 files changed, 1222 insertions(+), 228 deletions(-) delete mode 100644 js/src/compound_list.js create mode 100644 js/src/indexed_sequence.js create mode 100644 js/src/list.js create mode 100644 js/src/list_test.js create mode 100644 js/src/map.js create mode 100644 js/src/map_test.js create mode 100644 js/src/ordered_sequence.js create mode 100644 js/src/primitives.js create mode 100644 js/src/read_value.js create mode 100644 js/src/sequence.js create mode 100644 js/src/set.js create mode 100644 js/src/set_test.js create mode 100644 js/src/value.js diff --git a/clients/pitchmap/ui/heat_map.js b/clients/pitchmap/ui/heat_map.js index 920ab30ff3..a92a52b529 100644 --- a/clients/pitchmap/ui/heat_map.js +++ b/clients/pitchmap/ui/heat_map.js @@ -1,6 +1,6 @@ // @flow -import React from 'react'; //eslint-disable-line no-unused-lets +import React from 'react'; import {readValue, HttpStore, Ref, Struct} from 'noms'; const IMAGE_WIDTH_PX = 286; diff --git a/clients/splore/main.js b/clients/splore/main.js index 3be040aceb..c53b98fb9b 100644 --- a/clients/splore/main.js +++ b/clients/splore/main.js @@ -2,9 +2,9 @@ import {layout, NodeGraph, TreeNode} from './buchheim.js'; import Layout from './layout.js'; -import React from 'react'; //eslint-disable-line no-unused-vars +import React from 'react'; // eslint-disable-line no-unused-vars import ReactDOM from 'react-dom'; -import {CompoundList, readValue, HttpStore, Ref, Struct} from 'noms'; +import {CompoundList, CompoundMap, CompoundSet, ListLeaf, MapLeaf, readValue, SetLeaf, HttpStore, Ref, Struct} from 'noms'; let data: NodeGraph = {nodes: {}, links: {}}; let rootRef: Ref; @@ -60,15 +60,17 @@ function handleChunkLoad(ref: Ref, val: any, fromRef: ?string) { if (val instanceof Blob) { data.nodes[id] = {name: `Blob (${val.size})`}; - } else if (Array.isArray(val)) { + } else if (val instanceof ListLeaf) { data.nodes[id] = {name: `List (${val.length})`}; - val.forEach(c => process(ref, c, id)); - } else if (val instanceof Set) { + val.items.forEach(c => process(ref, c, id)); + } else if (val instanceof SetLeaf) { data.nodes[id] = {name: `Set (${val.size})`}; - val.forEach(c => process(ref, c, id)); - } else if (val instanceof Map) { + val.items.forEach(c => process(ref, c, id)); + } else if (val instanceof MapLeaf) { data.nodes[id] = {name: `Map (${val.size})`}; - val.forEach((v, k) => { + val.items.forEach(entry => { + let k = entry.key; + let v = entry.value; // TODO: handle non-string keys let kid = process(ref, k, id); if (kid) { @@ -105,8 +107,34 @@ function handleChunkLoad(ref: Ref, val: any, fromRef: ?string) { } }); } else if (val instanceof CompoundList) { - data.nodes[id] = {name: 'MetaList'}; - val.tuples.forEach(tuple => { + data.nodes[id] = {name: 'ListNode'}; + val.items.forEach(tuple => { + let kid = process(ref, tuple.value, id); + if (kid) { + // Start map keys open, just makes it easier to use. + data.nodes[kid].isOpen = true; + + process(ref, tuple.ref, kid); + } else { + throw new Error('No kid id.'); + } + }); + } else if (val instanceof CompoundMap) { + data.nodes[id] = {name: 'MapNode'}; + val.items.forEach(tuple => { + let kid = process(ref, tuple.value, id); + if (kid) { + // Start map keys open, just makes it easier to use. + data.nodes[kid].isOpen = true; + + process(ref, tuple.ref, kid); + } else { + throw new Error('No kid id.'); + } + }); + } else if (val instanceof CompoundSet) { + data.nodes[id] = {name: 'SetNode'}; + val.items.forEach(tuple => { let kid = process(ref, tuple.value, id); if (kid) { // Start map keys open, just makes it easier to use. diff --git a/js/src/compound_list.js b/js/src/compound_list.js deleted file mode 100644 index 594263e800..0000000000 --- a/js/src/compound_list.js +++ /dev/null @@ -1,12 +0,0 @@ -// @flow - -import type {ChunkStore} from './chunk_store.js'; -import {MetaTuple, MetaSequence} from './meta_sequence.js'; -import {Type} from './type.js'; - -export default class CompoundList extends MetaSequence { - - constructor(cs: ChunkStore, type: Type, tuples: Array) { - super(cs, type, tuples); - } -} diff --git a/js/src/decode.js b/js/src/decode.js index c33fa57d51..7bfe6f476e 100644 --- a/js/src/decode.js +++ b/js/src/decode.js @@ -1,17 +1,20 @@ // @flow import Chunk from './chunk.js'; -import CompoundList from './compound_list.js'; import Ref from './ref.js'; import Struct from './struct.js'; import type {ChunkStore} from './chunk_store.js'; import type {NomsKind} from './noms_kind.js'; import {decode as decodeBase64} from './base64.js'; -import {CompoundDesc, Field, makeCompoundType, makeEnumType, makePrimitiveType, makeStructType, makeType, makeUnresolvedType, StructDesc, Type} from './type.js'; +import {Field, makeCompoundType, makeEnumType, makePrimitiveType, makeStructType, makeType, makeUnresolvedType, StructDesc, Type} from './type.js'; +import {indexTypeForMetaSequence, MetaTuple, newMetaSequenceFromData} from './meta_sequence.js'; import {invariant, notNull} from './assert.js'; import {isPrimitiveKind, Kind} from './noms_kind.js'; +import {ListLeaf} from './list.js'; import {lookupPackage, Package, readPackage} from './package.js'; -import {MetaTuple} from './meta_sequence.js'; +import {MapLeaf} from './map.js'; +import {setDecodeNomsValue} from './read_value.js'; +import {SetLeaf} from './set.js'; const typedTag = 't '; const blobTag = 'b '; @@ -107,7 +110,7 @@ class JsonArrayReader { return Promise.resolve(decodeBase64(s)); } - async readList(t: Type, pkg: ?Package): Promise> { + async readSequence(t: Type, pkg: ?Package): Promise> { let elemType = t.elemTypes[0]; let list = []; while (!this.atEnd()) { @@ -118,22 +121,27 @@ class JsonArrayReader { return list; } - async readSet(t: Type, pkg: ?Package): Promise { - let seq = await this.readList(t, pkg); - return new Set(seq); + async readListLeaf(t: Type, pkg: ?Package): Promise { + let seq = await this.readSequence(t, pkg); + return new ListLeaf(this._cs, t, seq); } - async readMap(t: Type, pkg: ?Package): Promise { + async readSetLeaf(t: Type, pkg: ?Package): Promise { + let seq = await this.readSequence(t, pkg); + return new SetLeaf(this._cs, t, seq); + } + + async readMapLeaf(t: Type, pkg: ?Package): Promise { let keyType = t.elemTypes[0]; let valueType = t.elemTypes[1]; - let m = new Map(); + let entries = []; while (!this.atEnd()) { let k = await this.readValueWithoutTag(keyType, pkg); let v = await this.readValueWithoutTag(valueType, pkg); - m.set(k, v); + entries.push({key: k, value: v}); } - return m; + return new MapLeaf(this._cs, t, entries); } readEnum(): number { @@ -154,13 +162,7 @@ class JsonArrayReader { data.push(new MetaTuple(ref, v)); } - switch (t.kind) { - // TODO: case Kind.Blob, Kind.Set, Kind.Map - case Kind.List: - return new CompoundList(this._cs, t, data); - default: - throw new Error('unreached'); - } + return newMetaSequenceFromData(this._cs, t, data); } readPackage(t: Type, pkg: ?Package): Package { @@ -221,7 +223,7 @@ class JsonArrayReader { } let r2 = new JsonArrayReader(this.readArray(), this._cs); - return r2.readList(t, pkg); + return r2.readListLeaf(t, pkg); } case Kind.Map: { let ms = await this.maybeReadMetaSequence(t, pkg); @@ -230,7 +232,7 @@ class JsonArrayReader { } let r2 = new JsonArrayReader(this.readArray(), this._cs); - return r2.readMap(t, pkg); + return r2.readMapLeaf(t, pkg); } case Kind.Package: return Promise.resolve(this.readPackage(t, pkg)); @@ -245,7 +247,7 @@ class JsonArrayReader { } let r2 = new JsonArrayReader(this.readArray(), this._cs); - return r2.readSet(t, pkg); + return r2.readSetLeaf(t, pkg); } case Kind.Enum: case Kind.Struct: @@ -374,22 +376,6 @@ class JsonArrayReader { } } -function indexTypeForMetaSequence(t: Type): Type { - switch (t.kind) { - case Kind.Map: - case Kind.Set: { - let desc = t.desc; - invariant(desc instanceof CompoundDesc); - return desc.elemTypes[0]; - } - case Kind.Blob: - case Kind.List: - return makePrimitiveType(Kind.Uint64); - } - - throw new Error('Not reached'); -} - function decodeNomsValue(chunk: Chunk, cs: ChunkStore): Promise { let tag = new Chunk(new Uint8Array(chunk.data.buffer, 0, 2)).toString(); @@ -407,13 +393,6 @@ function decodeNomsValue(chunk: Chunk, cs: ChunkStore): Promise { } } -export async function readValue(r: Ref, cs: ChunkStore): Promise { - let chunk = await cs.get(r); - if (chunk.isEmpty()) { - return null; - } +export {decodeNomsValue, indexTypeForMetaSequence, JsonArrayReader}; - return decodeNomsValue(chunk, cs); -} - -export {decodeNomsValue, indexTypeForMetaSequence, JsonArrayReader, readValue}; +setDecodeNomsValue(decodeNomsValue); // TODO: Avoid cyclic badness with commonjs. diff --git a/js/src/decode_test.js b/js/src/decode_test.js index 97827eeb24..826a006f62 100644 --- a/js/src/decode_test.js +++ b/js/src/decode_test.js @@ -1,19 +1,22 @@ // @flow import Chunk from './chunk.js'; -import CompoundList from './compound_list.js'; import MemoryStore from './memory_store.js'; import Ref from './ref.js'; import Struct from './struct.js'; import test from './async_test.js'; import type {TypeDesc} from './type.js'; import {assert} from 'chai'; -import {decodeNomsValue, JsonArrayReader, readValue} from './decode.js'; +import {decodeNomsValue, JsonArrayReader} from './decode.js'; import {Field, makeCompoundType, makeEnumType, makePrimitiveType, makeStructType, makeType, Type} from './type.js'; import {invariant} from './assert.js'; import {Kind} from './noms_kind.js'; +import {ListLeaf, CompoundList} from './list.js'; +import {MapLeaf} from './map.js'; import {MetaTuple} from './meta_sequence.js'; +import {readValue} from './read_value.js'; import {registerPackage, Package} from './package.js'; +import {SetLeaf} from './set.js'; import {suite} from 'mocha'; import {writeValue} from './encode.js'; @@ -85,15 +88,25 @@ suite('Decode', () => { let a = [Kind.List, Kind.Int32, false, [0, 1, 2, 3]]; let r = new JsonArrayReader(a, ms); let v = await r.readTopLevelValue(); - assert.deepEqual([0, 1, 2, 3], v); + invariant(v instanceof ListLeaf); + + let tr = makeCompoundType(Kind.List, makePrimitiveType(Kind.Int32)); + let l = new ListLeaf(ms, tr, [0, 1, 2, 3]); + assert.isTrue(l.equals(v)); }); + // TODO: Can't round-trip collections of value types. =-( test('read list of value', async () => { let ms = new MemoryStore(); let a = [Kind.List, Kind.Value, false, [Kind.Int32, 1, Kind.String, 'hi', Kind.Bool, true]]; let r = new JsonArrayReader(a, ms); let v = await r.readTopLevelValue(); - assert.deepEqual([1, 'hi', true], v); + invariant(v instanceof ListLeaf); + let tr = makeCompoundType(Kind.List, makePrimitiveType(Kind.Value)); + assert.isTrue(v.type.equals(tr)); + assert.strictEqual(1, await v.get(0)); + assert.strictEqual('hi', await v.get(1)); + assert.strictEqual(true, await v.get(2)); }); test('read value list of int8', async () => { @@ -101,16 +114,20 @@ suite('Decode', () => { let a = [Kind.Value, Kind.List, Kind.Int8, false, [0, 1, 2]]; let r = new JsonArrayReader(a, ms); let v = await r.readTopLevelValue(); - assert.deepEqual([0, 1, 2], v); + invariant(v instanceof ListLeaf); + + let tr = makeCompoundType(Kind.List, makePrimitiveType(Kind.Int8)); + let l = new ListLeaf(ms, tr, [0, 1, 2]); + assert.isTrue(l.equals(v)); }); test('read compound list', async () => { let ms = new MemoryStore(); let ltr = makeCompoundType(Kind.List, makePrimitiveType(Kind.Int32)); - let r1 = writeValue([0, 1], ltr, ms); - let r2 = writeValue([2, 3], ltr, ms); - let r3 = writeValue([4, 5], ltr, ms); + let r1 = writeValue(new ListLeaf(ms, ltr, [0, 1]), ltr, ms); + let r2 = writeValue(new ListLeaf(ms, ltr, [2, 3]), ltr, ms); + let r3 = writeValue(new ListLeaf(ms, ltr, [4, 5]), ltr, ms); let tuples = [ new MetaTuple(r1, 2), new MetaTuple(r2, 4), @@ -125,25 +142,16 @@ suite('Decode', () => { assert.isTrue(v.ref.equals(l.ref)); }); - function assertMapsEqual(expected: Map, actual: Map): void { - assert.strictEqual(expected.size, actual.size); - expected.forEach((v, k) => { - assert.isTrue(actual.has(k)); - assert.deepEqual(v, actual.get(k)); - }); - } - test('read map of int64 to float64', async () => { let ms = new MemoryStore(); let a = [Kind.Map, Kind.Int64, Kind.Float64, false, [0, 1, 2, 3]]; let r = new JsonArrayReader(a, ms); let v = await r.readTopLevelValue(); + invariant(v instanceof MapLeaf); - let m = new Map(); - m.set(0, 1); - m.set(2, 3); - - assertMapsEqual(m, v); + let t = makeCompoundType(Kind.Map, makePrimitiveType(Kind.Int64), makePrimitiveType(Kind.Float64)); + let m = new MapLeaf(ms, t, [{key: 0, value: 1}, {key: 2, value: 3}]); + assert.isTrue(v.equals(m)); }); test('read value map of uint64 to uint32', async () => { @@ -151,34 +159,23 @@ suite('Decode', () => { let a = [Kind.Value, Kind.Map, Kind.Uint64, Kind.Uint32, false, [0, 1, 2, 3]]; let r = new JsonArrayReader(a, ms); let v = await r.readTopLevelValue(); + invariant(v instanceof MapLeaf); - let m = new Map(); - m.set(0, 1); - m.set(2, 3); - - assertMapsEqual(m, v); + let t = makeCompoundType(Kind.Map, makePrimitiveType(Kind.Uint64), makePrimitiveType(Kind.Uint32)); + let m = new MapLeaf(ms, t, [{key: 0, value: 1}, {key: 2, value: 3}]); + assert.isTrue(v.equals(m)); }); - function assertSetsEqual(expected: Set, actual: Set): void { - assert.strictEqual(expected.size, actual.size); - expected.forEach((v) => { - assert.isTrue(actual.has(v)); - }); - } - test('read set of uint8', async () => { let ms = new MemoryStore(); let a = [Kind.Set, Kind.Uint8, false, [0, 1, 2, 3]]; let r = new JsonArrayReader(a, ms); let v = await r.readTopLevelValue(); + invariant(v instanceof SetLeaf); - let s = new Set(); - s.add(0); - s.add(1); - s.add(2); - s.add(3); - - assertSetsEqual(s, v); + let t = makeCompoundType(Kind.Set, makePrimitiveType(Kind.Uint8)); + let s = new SetLeaf(ms, t, [0, 1, 2, 3]); + assert.isTrue(v.equals(s)); }); test('read value set of uint16', async () => { @@ -187,12 +184,13 @@ suite('Decode', () => { let r = new JsonArrayReader(a, ms); let v = await r.readTopLevelValue(); - let s = new Set([0, 1, 2, 3]); - assertSetsEqual(s, v); + let t = makeCompoundType(Kind.Set, makePrimitiveType(Kind.Uint16)); + let s = new SetLeaf(ms, t, [0, 1, 2, 3]); + assert.isTrue(v.equals(s)); }); function assertStruct(s: Struct, desc: TypeDesc, data: {[key: string]: any}) { - invariant(s instanceof Struct); + invariant(s instanceof Struct, 'expected instanceof struct'); assert.deepEqual(desc, s.desc); for (let key in data) { @@ -267,9 +265,11 @@ suite('Decode', () => { test('test read struct with list', async () => { let ms = new MemoryStore(); + + let ltr = makeCompoundType(Kind.List, makePrimitiveType(Kind.Int32)); let tr = makeStructType('A4', [ new Field('b', makePrimitiveType(Kind.Bool), false), - new Field('l', makeCompoundType(Kind.List, makePrimitiveType(Kind.Int32)), false), + new Field('l', ltr, false), new Field('s', makePrimitiveType(Kind.String), false) ], []); @@ -282,7 +282,7 @@ suite('Decode', () => { assertStruct(v, tr.desc, { b: true, - l: [0, 1, 2], + l: new ListLeaf(ms, ltr, [0, 1, 2]), s: 'hi' }); }); @@ -391,24 +391,27 @@ suite('Decode', () => { let pkg = new Package([tr], []); registerPackage(pkg); - let a = [Kind.Value, Kind.Map, Kind.String, Kind.Unresolved, pkg.ref.toString(), 0, false, ['foo', true, 3, 'bar', false, 2, 'baz', false, 1]]; + let a = [Kind.Value, Kind.Map, Kind.String, Kind.Unresolved, pkg.ref.toString(), 0, false, ['bar', false, 2, 'baz', false, 1, 'foo', true, 3]]; let r = new JsonArrayReader(a, ms); let v = await r.readTopLevelValue(); - invariant(v instanceof Map); + invariant(v instanceof MapLeaf); assert.strictEqual(3, v.size); - assertStruct(v.get('foo'), tr.desc, {b: true, i: 3}); - assertStruct(v.get('bar'), tr.desc, {b: false, i: 2}); - assertStruct(v.get('baz'), tr.desc, {b: false, i: 1}); + assertStruct(await v.get('foo'), tr.desc, {b: true, i: 3}); + assertStruct(await v.get('bar'), tr.desc, {b: false, i: 2}); + assertStruct(await v.get('baz'), tr.desc, {b: false, i: 1}); }); test('decodeNomsValue', async () => { + let ms = new MemoryStore(); let chunk = Chunk.fromString(`t [${Kind.Value}, ${Kind.Set}, ${Kind.Uint16}, false, [0, 1, 2, 3]]`); let v = await decodeNomsValue(chunk, new MemoryStore()); - let s = new Set([0, 1, 2, 3]); - assertSetsEqual(s, v); + + let t = makeCompoundType(Kind.Set, makePrimitiveType(Kind.Uint16)); + let s = new SetLeaf(ms, t, [0, 1, 2, 3]); + assert.isTrue(v.equals(s)); }); test('decodeNomsValue: counter with one commit', async () => { @@ -419,9 +422,9 @@ suite('Decode', () => { ms.put(Chunk.fromString('t [21,"sha1-7546d804d845125bc42669c7a4c3f3fb909eca29",0,4,1,false,[]]')); // commit let rootMap = await readValue(root, ms); - let counterRef = rootMap.get('counter'); + let counterRef = await rootMap.get('counter'); let commit = await readValue(counterRef, ms); - assert.strictEqual(1, commit.get('value')); + assert.strictEqual(1, await commit.get('value')); }); test('top level blob', async () => { diff --git a/js/src/encode.js b/js/src/encode.js index d174ff797b..fef3babcf7 100644 --- a/js/src/encode.js +++ b/js/src/encode.js @@ -6,12 +6,16 @@ import Struct from './struct.js'; import type {ChunkStore} from './chunk_store.js'; import type {NomsKind} from './noms_kind.js'; import {encode as encodeBase64} from './base64.js'; -import {indexTypeForMetaSequence} from './decode.js'; +import {indexTypeForMetaSequence} from './meta_sequence.js'; import {invariant, notNull} from './assert.js'; import {isPrimitiveKind, Kind} from './noms_kind.js'; +import {ListLeaf} from './list.js'; import {lookupPackage, Package} from './package.js'; import {makePrimitiveType, EnumDesc, StructDesc, Type} from './type.js'; -import {MetaSequence} from './meta_sequence.js'; +import {MapLeaf} from './map.js'; +import {Sequence} from './sequence.js'; +import {setEncodeNomsValue} from './get_ref.js'; +import {SetLeaf} from './set.js'; const typedTag = 't '; @@ -78,8 +82,8 @@ class JsonArrayWriter { this.writeValue(v, t); } - maybeWriteMetaSequence(v: any, t: Type, pkg: ?Package): boolean { - if (!(v instanceof MetaSequence)) { + maybeWriteMetaSequence(v: Sequence, t: Type, pkg: ?Package): boolean { + if (!v.isMeta) { this.write(false); return false; } @@ -87,8 +91,8 @@ class JsonArrayWriter { this.write(true); let w2 = new JsonArrayWriter(this._cs); let indexType = indexTypeForMetaSequence(t); - for (let i = 0; i < v.tuples.length; i++) { - let tuple = v.tuples[i]; + for (let i = 0; i < v.items.length; i++) { + let tuple = v.items[i]; w2.writeRef(tuple.ref); w2.writeValue(tuple.value, indexType, pkg); } @@ -99,9 +103,12 @@ class JsonArrayWriter { writeValue(v: any, t: Type, pkg: ?Package) { switch (t.kind) { case Kind.Blob: - if (this.maybeWriteMetaSequence(v, t, pkg)) { - break; - } + this.write(false); + // TODO: When CompoundBlob is implemented... + // invariant(v instanceof Sequence); + // if (this.maybeWriteMetaSequence(v, t, pkg)) { + // break; + // } this.writeBlob(v); break; @@ -120,34 +127,31 @@ class JsonArrayWriter { this.write(v); // TODO: Verify value fits in type break; case Kind.List: { + invariant(v instanceof Sequence); if (this.maybeWriteMetaSequence(v, t, pkg)) { break; } - invariant(Array.isArray(v)); + invariant(v instanceof ListLeaf); let w2 = new JsonArrayWriter(this._cs); let elemType = t.elemTypes[0]; - v.forEach(sv => w2.writeValue(sv, elemType)); + v.items.forEach(sv => w2.writeValue(sv, elemType)); this.write(w2.array); break; } case Kind.Map: { + invariant(v instanceof Sequence); if (this.maybeWriteMetaSequence(v, t, pkg)) { break; } - invariant(v instanceof Map); + invariant(v instanceof MapLeaf); let w2 = new JsonArrayWriter(this._cs); let keyType = t.elemTypes[0]; let valueType = t.elemTypes[1]; - let elems = []; - v.forEach((v, k) => { - elems.push(k); - }); - elems = orderValuesByRef(keyType, elems); - elems.forEach(elem => { - w2.writeValue(elem, keyType); - w2.writeValue(v.get(elem), valueType); + v.items.forEach(entry => { + w2.writeValue(entry.key, keyType); + w2.writeValue(entry.value, valueType); }); this.write(w2.array); break; @@ -171,18 +175,18 @@ class JsonArrayWriter { break; } case Kind.Set: { + invariant(v instanceof Sequence); if (this.maybeWriteMetaSequence(v, t, pkg)) { break; } - invariant(v instanceof Set); + invariant(v instanceof SetLeaf); let w2 = new JsonArrayWriter(this._cs); let elemType = t.elemTypes[0]; let elems = []; - v.forEach(v => { + v.items.forEach(v => { elems.push(v); }); - elems = orderValuesByRef(elemType, elems); elems.forEach(elem => w2.writeValue(elem, elemType)); this.write(w2.array); break; @@ -324,19 +328,6 @@ class JsonArrayWriter { } } -function orderValuesByRef(t: Type, a: Array): Array { - return a.map(v => { - return { - v: v, - r: encodeEmbeddedNomsValue(v, t, null).ref - }; - }).sort((a, b) => { - return a.r.compare(b.r); - }).map(o => { - return o.v; - }); -} - function encodeEmbeddedNomsValue(v: any, t: Type, cs: ?ChunkStore): Chunk { if (v instanceof Package) { // if (v.dependencies.length > 0) { @@ -378,3 +369,5 @@ function writeValue(v: any, t: Type, cs: ChunkStore): Ref { } export {encodeNomsValue, JsonArrayWriter, writeValue}; + +setEncodeNomsValue(encodeNomsValue); diff --git a/js/src/encode_test.js b/js/src/encode_test.js index b0d652f99b..72d76c1c12 100644 --- a/js/src/encode_test.js +++ b/js/src/encode_test.js @@ -3,7 +3,6 @@ import {assert} from 'chai'; import {suite} from 'mocha'; -import CompoundList from './compound_list.js'; import MemoryStore from './memory_store.js'; import Ref from './ref.js'; import Struct from './struct.js'; @@ -12,8 +11,11 @@ import type {NomsKind} from './noms_kind.js'; import {Field, makeCompoundType, makeEnumType, makePrimitiveType, makeStructType, makeType, Type} from './type.js'; import {JsonArrayWriter, encodeNomsValue} from './encode.js'; import {Kind} from './noms_kind.js'; +import {ListLeaf, CompoundList} from './list.js'; +import {MapLeaf} from './map.js'; import {MetaTuple} from './meta_sequence.js'; import {Package, registerPackage} from './package.js'; +import {SetLeaf} from './set.js'; import {writeValue} from './encode.js'; suite('Encode', () => { @@ -54,7 +56,8 @@ suite('Encode', () => { let w = new JsonArrayWriter(ms); let tr = makeCompoundType(Kind.List, makePrimitiveType(Kind.Int32)); - w.writeTopLevel(tr, [0, 1, 2, 3]); + let l = new ListLeaf(ms, tr, [0, 1, 2, 3]); + w.writeTopLevel(tr, l); assert.deepEqual([Kind.List, Kind.Int32, false, [0, 1, 2, 3]], w.array); }); @@ -64,7 +67,7 @@ suite('Encode', () => { let it = makeCompoundType(Kind.List, makePrimitiveType(Kind.Int16)); let tr = makeCompoundType(Kind.List, it); - let v = [[0], [1, 2, 3]]; + let v = new ListLeaf(ms, tr, [new ListLeaf(ms, it, [0]), new ListLeaf(ms, it, [1, 2, 3])]); w.writeTopLevel(tr, v); assert.deepEqual([Kind.List, Kind.List, Kind.Int16, false, [false, [0], false, [1, 2, 3]]], w.array); }); @@ -74,9 +77,9 @@ suite('Encode', () => { let w = new JsonArrayWriter(ms); let tr = makeCompoundType(Kind.Set, makePrimitiveType(Kind.Uint32)); - let v = new Set([0, 1, 2, 3]); + let v = new SetLeaf(ms, tr, [0, 1, 2, 3]); w.writeTopLevel(tr, v); - assert.deepEqual([Kind.Set, Kind.Uint32, false, [1, 3, 0, 2]], w.array); + assert.deepEqual([Kind.Set, Kind.Uint32, false, [0, 1, 2, 3]], w.array); }); test('write set of set', async () => { @@ -85,9 +88,10 @@ suite('Encode', () => { let st = makeCompoundType(Kind.Set, makePrimitiveType(Kind.Int32)); let tr = makeCompoundType(Kind.Set, st); - let v = new Set([new Set([0]), new Set([1, 2, 3])]); + let v = new SetLeaf(ms, tr, [new SetLeaf(ms, st, [0]), new SetLeaf(ms, st, [1, 2, 3])]); + w.writeTopLevel(tr, v); - assert.deepEqual([Kind.Set, Kind.Set, Kind.Int32, false, [false, [1, 3, 2], false, [0]]], w.array); + assert.deepEqual([Kind.Set, Kind.Set, Kind.Int32, false, [false, [0], false, [1, 2, 3]]], w.array); }); test('write map', async() => { @@ -95,9 +99,7 @@ suite('Encode', () => { let w = new JsonArrayWriter(ms); let tr = makeCompoundType(Kind.Map, makePrimitiveType(Kind.String), makePrimitiveType(Kind.Bool)); - let v = new Map(); - v.set('a', false); - v.set('b', true); + let v = new MapLeaf(ms, tr, [{key: 'a', value: false}, {key:'b', value:true}]); w.writeTopLevel(tr, v); assert.deepEqual([Kind.Map, Kind.String, Kind.Bool, false, ['a', false, 'b', true]], w.array); }); @@ -110,11 +112,10 @@ suite('Encode', () => { let vt = makeCompoundType(Kind.Set, makePrimitiveType(Kind.Bool)); let tr = makeCompoundType(Kind.Map, kt, vt); - let v = new Map(); - let m1 = new Map(); - m1.set('a', 0); - let s = new Set([true]); - v.set(m1, s); + + let s = new SetLeaf(ms, vt, [true]); + let m1 = new MapLeaf(ms, kt, [{key: 'a', value: 0}]); + let v = new MapLeaf(ms, tr, [{key: m1, value: s}]); w.writeTopLevel(tr, v); assert.deepEqual([Kind.Map, Kind.Map, Kind.String, Kind.Int64, Kind.Set, Kind.Bool, false, [false, ['a', 0], false, [true]]], w.array); }); @@ -206,19 +207,20 @@ suite('Encode', () => { let ms = new MemoryStore(); let w = new JsonArrayWriter(ms); + let ltr = makeCompoundType(Kind.List, makePrimitiveType(Kind.String)); let typeDef = makeStructType('S', [ - new Field('l', makeCompoundType(Kind.List, makePrimitiveType(Kind.String)), false) + new Field('l', ltr, false) ], []); let pkg = new Package([typeDef], []); registerPackage(pkg); let pkgRef = pkg.ref; let type = makeType(pkgRef, 0); - let v = new Struct(type, typeDef, {l: ['a', 'b']}); + let v = new Struct(type, typeDef, {l: new ListLeaf(ms, ltr, ['a', 'b'])}); w.writeTopLevel(type, v); assert.deepEqual([Kind.Unresolved, pkgRef.toString(), 0, false, ['a', 'b']], w.array); - v = new Struct(type, typeDef, {l: []}); + v = new Struct(type, typeDef, {l: new ListLeaf(ms, ltr, [])}); w = new JsonArrayWriter(ms); w.writeTopLevel(type, v); assert.deepEqual([Kind.Unresolved, pkgRef.toString(), 0, false, []], w.array); @@ -268,7 +270,7 @@ suite('Encode', () => { let pkgRef = pkg.ref; let typ = makeType(pkgRef, 0); let listType = makeCompoundType(Kind.List, typ); - let l = [0, 1, 2]; + let l = new ListLeaf(ms, listType, [0, 1, 2]); w.writeTopLevel(listType, l); assert.deepEqual([Kind.List, Kind.Unresolved, pkgRef.toString(), 0, false, [0, 1, 2]], w.array); @@ -279,9 +281,9 @@ suite('Encode', () => { let w = new JsonArrayWriter(ms); let ltr = makeCompoundType(Kind.List, makePrimitiveType(Kind.Int32)); - let r1 = writeValue([0, 1], ltr, ms); - let r2 = writeValue([2, 3], ltr, ms); - let r3 = writeValue([4, 5], ltr, ms); + let r1 = writeValue(new ListLeaf(ms, ltr, [0, 1]), ltr, ms); + let r2 = writeValue(new ListLeaf(ms, ltr, [2, 3]), ltr, ms); + let r3 = writeValue(new ListLeaf(ms, ltr, [4, 5]), ltr, ms); let tuples = [ new MetaTuple(r1, 2), new MetaTuple(r2, 4), diff --git a/js/src/get_ref.js b/js/src/get_ref.js index a84d400a4f..2fd2d882d8 100644 --- a/js/src/get_ref.js +++ b/js/src/get_ref.js @@ -1,11 +1,16 @@ // @flow +import Chunk from './chunk.js'; import Ref from './ref.js'; -import {encodeNomsValue} from './encode.js'; +import type {ChunkStore} from './chunk_store.js'; +import {notNull} from './assert.js'; import {Type} from './type.js'; +type encodeFn = (v: any, t: Type, cs: ?ChunkStore) => Chunk; +let encodeNomsValue: ?encodeFn = null; + export function getRef(v: any, t: Type): Ref { - return encodeNomsValue(v, t, null).ref; + return notNull(encodeNomsValue)(v, t, null).ref; } export function ensureRef(r: ?Ref, v: any, t: Type): Ref { @@ -15,3 +20,7 @@ export function ensureRef(r: ?Ref, v: any, t: Type): Ref { return getRef(v, t); } + +export function setEncodeNomsValue(encode: encodeFn) { + encodeNomsValue = encode; +} diff --git a/js/src/indexed_sequence.js b/js/src/indexed_sequence.js new file mode 100644 index 0000000000..6756643723 --- /dev/null +++ b/js/src/indexed_sequence.js @@ -0,0 +1,37 @@ +// @flow + +import {notNull} from './assert.js'; +import {search, Sequence, SequenceCursor} from './sequence.js'; + +export class IndexedSequence extends Sequence { + getOffset(idx: number): number { // eslint-disable-line no-unused-vars + throw new Error('override'); + } + + async newCursorAt(idx: number): Promise { + let cursor: ?IndexedSequenceCursor = null; + let sequence: ?IndexedSequence = this; + + while (sequence) { + cursor = new IndexedSequenceCursor(cursor, sequence, 0); + idx -= cursor.advanceToOffset(idx); + sequence = await cursor.getChildSequence(); + } + + return notNull(cursor); + } +} + +export class IndexedSequenceCursor extends SequenceCursor { + advanceToOffset(idx: number): number { + this.idx = search(this.length, (i: number) => { + return idx <= this.sequence.getOffset(i); + }); + + if (this.idx === this.length) { + this.idx = this.length - 1; + } + + return this.idx > 0 ? this.sequence.getOffset(this.idx - 1) + 1 : 0; + } +} diff --git a/js/src/list.js b/js/src/list.js new file mode 100644 index 0000000000..17b5427b81 --- /dev/null +++ b/js/src/list.js @@ -0,0 +1,69 @@ + // @flow + +import type {ChunkStore} from './chunk_store.js'; +import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars +import {IndexedSequence} from './indexed_sequence.js'; +import {invariant} from './assert.js'; +import {Kind} from './noms_kind.js'; +import {MetaTuple, registerMetaValue} from './meta_sequence.js'; +import {Type} from './type.js'; + +export class NomsList extends IndexedSequence { + async get(idx: number): Promise { + invariant(idx < this.length, idx + ' >= ' + this.length); + let cursor = await this.newCursorAt(idx); + return cursor.getCurrent(); + } + + async forEach(cb: (v: K, i: number) => void): Promise { + let cursor = await this.newCursorAt(0); + return cursor.iter((v, i) => { + cb(v, i); + return false; + }); + } + + get length(): number { + return this.items.length; + } +} + +export class ListLeaf extends NomsList { + getOffset(idx: number): number { + return idx; + } +} + +export class CompoundList extends NomsList> { + offsets: Array; + + constructor(cs: ChunkStore, type: Type, items: Array>) { + super(cs, type, items); + this.isMeta = true; + this.offsets = []; + let cum = 0; + for (let i = 0; i < items.length; i++) { + let length = items[i].value; + this.offsets.push(cum + length - 1); + cum += length; + } + } + + getOffset(idx: number): number { + return this.offsets[idx]; + } + + async getChildSequence(idx: number): Promise { + let mt = this.items[idx]; + let ms = await mt.readValue(this.cs); + invariant(ms instanceof NomsList); + return ms; + } + + get length(): number { + return this.offsets[this.items.length - 1] + 1; + } +} + +registerMetaValue(Kind.List, (cs, type, tuples) => new CompoundList(cs, type, tuples)); + diff --git a/js/src/list_test.js b/js/src/list_test.js new file mode 100644 index 0000000000..a7ad4eb384 --- /dev/null +++ b/js/src/list_test.js @@ -0,0 +1,76 @@ +// @flow + +import {assert} from 'chai'; +import {suite} from 'mocha'; + +import MemoryStore from './memory_store.js'; +import test from './async_test.js'; +import {CompoundList, ListLeaf} from './list.js'; +import {Kind} from './noms_kind.js'; +import {makeCompoundType, makePrimitiveType} from './type.js'; +import {MetaTuple} from './meta_sequence.js'; +import {writeValue} from './encode.js'; + +suite('ListLeaf', () => { + test('get', async () => { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.List, makePrimitiveType(Kind.String)); + let l = new ListLeaf(ms, tr, ['z', 'x', 'a', 'b']); + assert.strictEqual('z', await l.get(0)); + assert.strictEqual('x', await l.get(1)); + assert.strictEqual('a', await l.get(2)); + assert.strictEqual('b', await l.get(3)); + }); + + test('forEach', async () => { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.List, makePrimitiveType(Kind.Int32)); + let l = new ListLeaf(ms, tr, [4, 2, 10, 16]); + + let values = []; + await l.forEach((v, i) => { values.push(v, i); }); + assert.deepEqual([4, 0, 2, 1, 10, 2, 16, 3], values); + }); +}); + +suite('CompoundList', () => { + function build(): CompoundList { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.List, makePrimitiveType(Kind.String)); + let l1 = new ListLeaf(ms, tr, ['a', 'b']); + let r1 = writeValue(l1, tr, ms); + let l2 = new ListLeaf(ms, tr, ['e', 'f']); + let r2 = writeValue(l2, tr, ms); + let l3 = new ListLeaf(ms, tr, ['h', 'i']); + let r3 = writeValue(l3, tr, ms); + let l4 = new ListLeaf(ms, tr, ['m', 'n']); + let r4 = writeValue(l4, tr, ms); + + let m1 = new CompoundList(ms, tr, [new MetaTuple(r1, 2), new MetaTuple(r2, 2)]); + let rm1 = writeValue(m1, tr, ms); + let m2 = new CompoundList(ms, tr, [new MetaTuple(r3, 2), new MetaTuple(r4, 2)]); + let rm2 = writeValue(m2, tr, ms); + + let l = new CompoundList(ms, tr, [new MetaTuple(rm1, 4), new MetaTuple(rm2, 4)]); + return l; + } + + test('get', async () => { + let l = build(); + assert.strictEqual('a', await l.get(0)); + assert.strictEqual('b', await l.get(1)); + assert.strictEqual('e', await l.get(2)); + assert.strictEqual('f', await l.get(3)); + assert.strictEqual('h', await l.get(4)); + assert.strictEqual('i', await l.get(5)); + assert.strictEqual('m', await l.get(6)); + assert.strictEqual('n', await l.get(7)); + }); + + test('forEach', async () => { + let l = build(); + let values = []; + await l.forEach((k, i) => { values.push(k, i); }); + assert.deepEqual(['a', 0, 'b', 1, 'e', 2, 'f', 3, 'h', 4, 'i', 5, 'm', 6, 'n', 7], values); + }); +}); diff --git a/js/src/map.js b/js/src/map.js new file mode 100644 index 0000000000..0c5e019406 --- /dev/null +++ b/js/src/map.js @@ -0,0 +1,80 @@ +// @flow + +import type {ChunkStore} from './chunk_store.js'; +import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars +import {equals} from './value.js'; +import {invariant} from './assert.js'; +import {Kind} from './noms_kind.js'; +import {MetaTuple, registerMetaValue} from './meta_sequence.js'; +import {OrderedSequence} from './ordered_sequence.js'; +import {Type} from './type.js'; + +type Entry = { + key: K, + value: V +}; + +export class NomsMap extends OrderedSequence { + async first(): Promise { + let cursor = await this.newCursorAt(null); + if (!cursor.valid) { + return undefined; + } + + let entry = cursor.getCurrent(); + return [entry.key, entry.value]; + } + + async get(key: K): Promise { + let cursor = await this.newCursorAt(key); + if (!cursor.valid) { + return undefined; + } + + let entry = cursor.getCurrent(); + return equals(entry.key, key) ? entry.value : undefined; + } + + async forEach(cb: (v: V, k: K) => void): Promise { + let cursor = await this.newCursorAt(null); + return cursor.iter(entry => { + cb(entry.value, entry.key); + return false; + }); + } + + get size(): number { + return this.items.length; + } +} + +export class MapLeaf extends NomsMap> { + getKey(idx: number): K { + return this.items[idx].key; + } +} + +export class CompoundMap extends NomsMap> { + constructor(cs: ChunkStore, type: Type, items: Array) { + super(cs, type, items); + this.isMeta = true; + } + + getKey(idx: number): K { + return this.items[idx].value; + } + + async getChildSequence(idx: number): Promise { + let mt = this.items[idx]; + let ms = await mt.readValue(this.cs); + invariant(ms instanceof NomsMap); + return ms; + } + + get size(): number { + throw new Error('not implemented'); + } +} + +registerMetaValue(Kind.Map, (cs, type, tuples) => new CompoundMap(cs, type, tuples)); + diff --git a/js/src/map_test.js b/js/src/map_test.js new file mode 100644 index 0000000000..ad8380955b --- /dev/null +++ b/js/src/map_test.js @@ -0,0 +1,121 @@ +// @flow + +import {assert} from 'chai'; +import {suite} from 'mocha'; + +import MemoryStore from './memory_store.js'; +import test from './async_test.js'; +import {CompoundMap, MapLeaf} from './map.js'; +import {Kind} from './noms_kind.js'; +import {makeCompoundType, makePrimitiveType} from './type.js'; +import {MetaTuple} from './meta_sequence.js'; +import {writeValue} from './encode.js'; + +suite('MapLeaf', () => { + test('has', async () => { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.Map, makePrimitiveType(Kind.String), makePrimitiveType(Kind.Bool)); + let m = new MapLeaf(ms, tr, [{key: 'a', value: false}, {key:'k', value:true}]); + assert.isTrue(await m.has('a')); + assert.isFalse(await m.has('b')); + assert.isTrue(await m.has('k')); + assert.isFalse(await m.has('z')); + }); + + test('first/get', async () => { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.Map, makePrimitiveType(Kind.String), makePrimitiveType(Kind.Int32)); + let m = new MapLeaf(ms, tr, [{key: 'a', value: 4}, {key:'k', value:8}]); + + assert.deepEqual(['a', 4], await m.first()); + + assert.strictEqual(4, await m.get('a')); + assert.strictEqual(undefined, await m.get('b')); + assert.strictEqual(8, await m.get('k')); + assert.strictEqual(undefined, await m.get('z')); + }); + + test('forEach', async () => { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.Map, makePrimitiveType(Kind.String), makePrimitiveType(Kind.Int32)); + let m = new MapLeaf(ms, tr, [{key: 'a', value: 4}, {key:'k', value:8}]); + + let kv = []; + await m.forEach((v, k) => { kv.push(k, v); }); + assert.deepEqual(['a', 4, 'k', 8], kv); + }); +}); + +suite('CompoundMap', () => { + function build(): Array { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.Map, makePrimitiveType(Kind.String), makePrimitiveType(Kind.Bool)); + let l1 = new MapLeaf(ms, tr, [{key: 'a', value: false}, {key:'b', value:false}]); + let r1 = writeValue(l1, tr, ms); + let l2 = new MapLeaf(ms, tr, [{key: 'e', value: true}, {key:'f', value:true}]); + let r2 = writeValue(l2, tr, ms); + let l3 = new MapLeaf(ms, tr, [{key: 'h', value: false}, {key:'i', value:true}]); + let r3 = writeValue(l3, tr, ms); + let l4 = new MapLeaf(ms, tr, [{key: 'm', value: true}, {key:'n', value:false}]); + let r4 = writeValue(l4, tr, ms); + + let m1 = new CompoundMap(ms, tr, [new MetaTuple(r1, 'b'), new MetaTuple(r2, 'f')]); + let rm1 = writeValue(m1, tr, ms); + let m2 = new CompoundMap(ms, tr, [new MetaTuple(r3, 'i'), new MetaTuple(r4, 'n')]); + let rm2 = writeValue(m2, tr, ms); + + let c = new CompoundMap(ms, tr, [new MetaTuple(rm1, 'f'), new MetaTuple(rm2, 'n')]); + return [c, m1, m2]; + } + + test('get', async () => { + let [c] = build(); + + assert.strictEqual(false, await c.get('a')); + assert.strictEqual(false, await c.get('b')); + assert.strictEqual(undefined, await c.get('c')); + assert.strictEqual(undefined, await c.get('d')); + assert.strictEqual(true, await c.get('e')); + assert.strictEqual(true, await c.get('f')); + assert.strictEqual(false, await c.get('h')); + assert.strictEqual(true, await c.get('i')); + assert.strictEqual(undefined, await c.get('j')); + assert.strictEqual(undefined, await c.get('k')); + assert.strictEqual(undefined, await c.get('l')); + assert.strictEqual(true, await c.get('m')); + assert.strictEqual(false, await c.get('n')); + assert.strictEqual(undefined, await c.get('o')); + }); + + test('first/has', async () => { + let [c, m1, m2] = build(); + + + assert.deepEqual(['a', false], await c.first()); + assert.deepEqual(['a', false], await m1.first()); + assert.deepEqual(['h', false], await m2.first()); + + assert.isTrue(await c.has('a')); + assert.isTrue(await c.has('b')); + assert.isFalse(await c.has('c')); + assert.isFalse(await c.has('d')); + assert.isTrue(await c.has('e')); + assert.isTrue(await c.has('f')); + assert.isTrue(await c.has('h')); + assert.isTrue(await c.has('i')); + assert.isFalse(await c.has('j')); + assert.isFalse(await c.has('k')); + assert.isFalse(await c.has('l')); + assert.isTrue(await c.has('m')); + assert.isTrue(await c.has('n')); + assert.isFalse(await c.has('o')); + }); + + test('forEach', async () => { + let [c] = build(); + + let kv = []; + await c.forEach((v, k) => { kv.push(k, v); }); + assert.deepEqual(['a', false, 'b', false, 'e', true, 'f', true, 'h', false, 'i', true, 'm', true, 'n', false], kv); + }); +}); diff --git a/js/src/meta_sequence.js b/js/src/meta_sequence.js index c330e77d95..92b861cb61 100644 --- a/js/src/meta_sequence.js +++ b/js/src/meta_sequence.js @@ -1,37 +1,62 @@ // @flow -'use strict'; - import Ref from './ref.js'; -import {ensureRef} from './get_ref.js'; -import {Type} from './type.js'; import type {ChunkStore} from './chunk_store.js'; +import type {NomsKind} from './noms_kind.js'; +import {CompoundDesc, makeCompoundType, makePrimitiveType, Type} from './type.js'; +import {invariant, notNull} from './assert.js'; +import {Kind} from './noms_kind.js'; +import {readValue} from './read_value.js'; +import {Sequence} from './sequence.js'; -export class MetaSequence { - tuples: Array; - type: Type; - _ref: ?Ref; - _cs: ChunkStore; +export type MetaSequence = Sequence; - constructor(cs: ChunkStore, type: Type, tuples: Array) { - this._cs = cs; - this.type = type; - this.tuples = tuples; - this._ref = null; - } - - get ref(): Ref { - return this._ref = ensureRef(this._ref, this, this.type); - } - -} - -export class MetaTuple { +export class MetaTuple { ref: Ref; - value: any; + value: K; - constructor(ref: Ref, value: any) { + constructor(ref: Ref, value: K) { this.ref = ref; this.value = value; } + + readValue(cs: ChunkStore): Promise { + return readValue(this.ref, cs); + } +} + +export type metaBuilderFn = (cs: ChunkStore, t: Type, tuples: Array) => MetaSequence; + +let metaFuncMap: Map = new Map(); + +export function newMetaSequenceFromData(cs: ChunkStore, t: Type, data: Array): MetaSequence { + let ctor = notNull(metaFuncMap.get(t.kind)); + return ctor(cs, t, data); +} + +export function registerMetaValue(k: NomsKind, bf: metaBuilderFn) { + metaFuncMap.set(k, bf); +} + +let indexedSequenceIndexType = makePrimitiveType(Kind.Uint64); + +export function indexTypeForMetaSequence(t: Type): Type { + switch (t.kind) { + case Kind.Map: + case Kind.Set: { + let desc = t.desc; + invariant(desc instanceof CompoundDesc); + let elemType = desc.elemTypes[0]; + if (elemType.ordered) { + return elemType; + } else { + return makeCompoundType(Kind.Ref, makePrimitiveType(Kind.Value)); + } + } + case Kind.Blob: + case Kind.List: + return indexedSequenceIndexType; + } + + throw new Error('Not reached'); } diff --git a/js/src/noms.js b/js/src/noms.js index ca960b4b9f..f6c20eaa26 100644 --- a/js/src/noms.js +++ b/js/src/noms.js @@ -1,14 +1,17 @@ // @flow -export {encodeNomsValue} from './encode.js'; -export {readValue} from './decode.js'; +export {decodeNomsValue} from './decode.js'; export {default as Chunk} from './chunk.js'; -export {default as CompoundList} from './compound_list.js'; export {default as HttpStore} from './http_store.js'; export {default as MemoryStore} from './memory_store.js'; export {default as Ref} from './ref.js'; export {default as Struct} from './struct.js'; +export {encodeNomsValue} from './encode.js'; export {lookupPackage, Package, readPackage, registerPackage} from './package.js'; +export {NomsList, ListLeaf, CompoundList} from './list.js'; +export {NomsMap, MapLeaf, CompoundMap} from './map.js'; +export {NomsSet, SetLeaf, CompoundSet} from './set.js'; +export {readValue} from './read_value.js'; export { CompoundDesc, EnumDesc, diff --git a/js/src/ordered_sequence.js b/js/src/ordered_sequence.js new file mode 100644 index 0000000000..f9fb2b3383 --- /dev/null +++ b/js/src/ordered_sequence.js @@ -0,0 +1,87 @@ +// @flow + +import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars +import {invariant, notNull} from './assert.js'; +import {less, equals} from './value.js'; +import {search, Sequence, SequenceCursor} from './sequence.js'; + +export class OrderedSequence extends Sequence { + // Returns: + // -null, if sequence is empty. + // -null, if all values in sequence are < key. + // -cursor positioned at + // -first value, if |key| is null + // -first value >= |key| + async newCursorAt(key: ?K): Promise { + let cursor: ?OrderedSequenceCursor = null; + let sequence: ?OrderedSequence = this; + + while (sequence) { + cursor = new OrderedSequenceCursor(cursor, sequence, 0); + if (key) { + if (!cursor._seekTo(key)) { + return cursor; // invalid + } + } + + sequence = await cursor.getChildSequence(); + } + + return notNull(cursor); + } + + getKey(idx: number): K { // eslint-disable-line no-unused-vars + throw new Error('override'); + } + + async has(key: K): Promise { + let cursor = await this.newCursorAt(key); + return cursor.valid && equals(cursor.getCurrentKey(), key); + } +} + +export class OrderedSequenceCursor extends SequenceCursor { + getCurrentKey(): K { + invariant(this.idx >= 0 && this.idx < this.length); + return this.sequence.getKey(this.idx); + } + + // Moves the cursor to the first value in sequence >= key and returns true. + // If none exists, returns false. + _seekTo(key: K): boolean { + this.idx = search(this.length, (i: number) => { + return !less(this.sequence.getKey(i), key); + }); + + return this.idx < this.length; + } + + async advanceTo(key: K): Promise { + if (!this.valid) { + throw new Error('Invalid Cursor'); + } + + if (this._seekTo(key)) { + return true; + } + + if (!this.parent) { + return false; + } + + let p = this.parent; + invariant(p instanceof OrderedSequenceCursor); + let old = p.getCurrent(); + if (!await p.advanceTo(key)) { + return false; + } + + this.idx = 0; + if (old !== p.getCurrent()) { + await this.sync(); + } + + invariant(this._seekTo(key)); + return true; + } +} diff --git a/js/src/package.js b/js/src/package.js index cdac43d3aa..1006c71f23 100644 --- a/js/src/package.js +++ b/js/src/package.js @@ -5,7 +5,7 @@ import type {ChunkStore} from './chunk_store.js'; import {ensureRef} from './get_ref.js'; import {invariant} from './assert.js'; import {packageType, Type} from './type.js'; -import {readValue} from './decode.js'; +import {readValue} from './read_value.js'; class Package { types: Array; diff --git a/js/src/primitives.js b/js/src/primitives.js new file mode 100644 index 0000000000..a9587a2ac0 --- /dev/null +++ b/js/src/primitives.js @@ -0,0 +1,26 @@ +// @flow + +export type uint8 = number; +export type uint16 = number; +export type uint32 = number; +export type uint64 = number; +export type int8 = number; +export type int16 = number; +export type int32 = number; +export type int64 = number; +export type float32 = number; +export type float64 = number; + +export type primitive = + uint8 | + uint16 | + uint32 | + uint64 | + int8 | + int16 | + int32 | + int64 | + float32 | + float64 | + string | + boolean; diff --git a/js/src/read_value.js b/js/src/read_value.js new file mode 100644 index 0000000000..39bc2a204c --- /dev/null +++ b/js/src/read_value.js @@ -0,0 +1,22 @@ +// @flow + +import Ref from './ref.js'; +import Chunk from './chunk.js'; +import type {ChunkStore} from './chunk_store.js'; +import {notNull} from './assert.js'; + +type decodeFn = (chunk: Chunk, cs: ChunkStore) => Promise +let decodeNomsValue: ?decodeFn = null; + +export async function readValue(r: Ref, cs: ChunkStore): Promise { + let chunk = await cs.get(r); + if (chunk.isEmpty()) { + return null; + } + + return notNull(decodeNomsValue)(chunk, cs); +} + +export function setDecodeNomsValue(decode: decodeFn) { + decodeNomsValue = decode; +} diff --git a/js/src/sequence.js b/js/src/sequence.js new file mode 100644 index 0000000000..43432f0a7c --- /dev/null +++ b/js/src/sequence.js @@ -0,0 +1,161 @@ +// @flow + +import type {ChunkStore} from './chunk_store.js'; +import {invariant, notNull} from './assert.js'; +import {Type} from './type.js'; +import {Value} from './value.js'; + +export class Sequence extends Value { + cs: ChunkStore; + items: Array; + isMeta: boolean; + + constructor(cs: ChunkStore, type: Type, items: Array) { + super(type); + + this.cs = cs; + this.items = items; + this.isMeta = false; + } + + getChildSequence(idx: number): Promise { // eslint-disable-line no-unused-vars + return Promise.resolve(null); + } +} + +export class SequenceCursor { + parent: ?SequenceCursor; + sequence: S; + idx: number; + + constructor(parent: ?SequenceCursor, sequence: S, idx: number) { + this.parent = parent; + this.sequence = sequence; + this.idx = idx; + } + + get length(): number { + return this.sequence.items.length; + } + + getItem(idx: number): T { + return this.sequence.items[idx]; + } + + async sync(): Promise { + invariant(this.parent); + this.sequence = notNull(await this.parent.getChildSequence()); + } + + getChildSequence(): Promise { + return this.sequence.getChildSequence(this.idx); + } + + getCurrent(): T { + invariant(this.valid); + return this.getItem(this.idx); + } + + get valid(): boolean { + return this.idx >= 0 && this.idx < this.length; + } + + get indexInChunk(): number { + return this.idx; + } + + advance(): Promise { + return this._advanceMaybeAllowPastEnd(true); + } + + advanceLocal(): boolean { + if (this.idx < this.length - 1) { + this.idx++; + return true; + } + + return false; + } + + async _advanceMaybeAllowPastEnd(allowPastEnd: boolean): Promise { + if (this.idx < this.length - 1) { + this.idx++; + return true; + } + + if (this.idx === this.length) { + return false; + } + + if (this.parent && (await this.parent._advanceMaybeAllowPastEnd(false))) { + await this.sync(); + this.idx = 0; + return true; + } + if (allowPastEnd) { + this.idx++; + } + + return false; + } + + retreat(): Promise { + return this._retreatMaybeAllowBeforeStart(true); + } + + async _retreatMaybeAllowBeforeStart(allowBeforeStart: boolean): Promise { + if (this.idx > 0) { + this.idx--; + return true; + } + if (this.idx === -1) { + return false; + } + invariant(this.idx === 0); + if (this.parent && this.parent._retreatMaybeAllowBeforeStart(false)) { + await this.sync(); + this.idx = this.length - 1; + return true; + } + + if (allowBeforeStart) { + this.idx--; + } + + return false; + } + + copy(): SequenceCursor { + return new SequenceCursor(this.parent ? this.parent.copy() : null, this.sequence, this.idx); + } + + async iter(cb: (v: T, i: number) => boolean): Promise { + let idx = 0; + while (this.valid) { + if (cb(this.getItem(this.idx), idx++)) { + return; + } + this.advanceLocal() || await this.advance(); + } + } +} + +// Translated from golang source (https://golang.org/src/sort/search.go?s=2249:2289#L49) +export function search(n: number, f: (i: number) => boolean): number { + // Define f(-1) == false and f(n) == true. + // Invariant: f(i-1) == false, f(j) == true. + let i = 0; + let j = n; + while (i < j) { + let h = i + (((j - i) / 2) | 0); // avoid overflow when computing h + // i ≤ h < j + if (!f(h)) { + i = h + 1; // preserves f(i-1) == false + } else { + j = h; // preserves f(j) == true + } + } + + // i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i. + return i; +} diff --git a/js/src/set.js b/js/src/set.js new file mode 100644 index 0000000000..cb76e78a1f --- /dev/null +++ b/js/src/set.js @@ -0,0 +1,58 @@ +// @flow + +import type {ChunkStore} from './chunk_store.js'; +import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars +import {invariant} from './assert.js'; +import {Kind} from './noms_kind.js'; +import {OrderedSequence} from './ordered_sequence.js'; +import {registerMetaValue, MetaTuple} from './meta_sequence.js'; +import {Type} from './type.js'; + +export class NomsSet extends OrderedSequence { + async first(): Promise { + let cursor = await this.newCursorAt(null); + return cursor.valid ? cursor.getCurrent() : null; + } + + async forEach(cb: (v: T) => void): Promise { + let cursor = await this.newCursorAt(null); + return cursor.iter(v => { + cb(v); + return false; + }); + } + + get size(): number { + return this.items.length; + } +} + +export class SetLeaf extends NomsSet { + getKey(idx: number): K { + return this.items[idx]; + } +} + +export class CompoundSet extends NomsSet> { + constructor(cs: ChunkStore, type: Type, items: Array) { + super(cs, type, items); + this.isMeta = true; + } + + getKey(idx: number): K { + return this.items[idx].value; + } + + async getChildSequence(idx: number): Promise { + let mt = this.items[idx]; + let ms = await mt.readValue(this.cs); + invariant(ms instanceof NomsSet); + return ms; + } + + get size(): number { + throw new Error('not implemented'); + } +} + +registerMetaValue(Kind.Set, (cs, type, tuples) => new CompoundSet(cs, type, tuples)); diff --git a/js/src/set_test.js b/js/src/set_test.js new file mode 100644 index 0000000000..32d6b9aafe --- /dev/null +++ b/js/src/set_test.js @@ -0,0 +1,140 @@ +// @flow + +import {assert} from 'chai'; +import {suite} from 'mocha'; + +import MemoryStore from './memory_store.js'; +import test from './async_test.js'; +import {CompoundSet, SetLeaf} from './set.js'; +import {notNull} from './assert.js'; +import {Kind} from './noms_kind.js'; +import {makeCompoundType, makePrimitiveType} from './type.js'; +import {MetaTuple} from './meta_sequence.js'; +import {writeValue} from './encode.js'; + +suite('SetLeaf', () => { + test('first/has', async () => { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.Set, makePrimitiveType(Kind.String)); + let s = new SetLeaf(ms, tr, ['a', 'k']); + + assert.strictEqual('a', await s.first()); + + assert.isTrue(await s.has('a')); + assert.isFalse(await s.has('b')); + assert.isTrue(await s.has('k')); + assert.isFalse(await s.has('z')); + }); + + test('forEach', async () => { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.Set, makePrimitiveType(Kind.String)); + let m = new SetLeaf(ms, tr, ['a', 'b']); + + let values = []; + await m.forEach((k) => { values.push(k); }); + assert.deepEqual(['a', 'b'], values); + }); +}); + +suite('CompoundSet', () => { + function build(): Array { + let ms = new MemoryStore(); + let tr = makeCompoundType(Kind.Set, makePrimitiveType(Kind.String)); + let l1 = new SetLeaf(ms, tr, ['a', 'b']); + let r1 = writeValue(l1, tr, ms); + let l2 = new SetLeaf(ms, tr, ['e', 'f']); + let r2 = writeValue(l2, tr, ms); + let l3 = new SetLeaf(ms, tr, ['h', 'i']); + let r3 = writeValue(l3, tr, ms); + let l4 = new SetLeaf(ms, tr, ['m', 'n']); + let r4 = writeValue(l4, tr, ms); + + let m1 = new CompoundSet(ms, tr, [new MetaTuple(r1, 'b'), new MetaTuple(r2, 'f')]); + let rm1 = writeValue(m1, tr, ms); + let m2 = new CompoundSet(ms, tr, [new MetaTuple(r3, 'i'), new MetaTuple(r4, 'n')]); + let rm2 = writeValue(m2, tr, ms); + + let c = new CompoundSet(ms, tr, [new MetaTuple(rm1, 'f'), new MetaTuple(rm2, 'n')]); + return [c, m1, m2]; + } + + test('first/has', async () => { + let [c, m1, m2] = build(); + assert.strictEqual('a', await m1.first()); + assert.strictEqual('h', await m2.first()); + assert.strictEqual('a', await c.first()); + + assert.isTrue(await c.has('a')); + assert.isTrue(await c.has('b')); + assert.isFalse(await c.has('c')); + assert.isFalse(await c.has('d')); + assert.isTrue(await c.has('e')); + assert.isTrue(await c.has('f')); + assert.isTrue(await c.has('h')); + assert.isTrue(await c.has('i')); + assert.isFalse(await c.has('j')); + assert.isFalse(await c.has('k')); + assert.isFalse(await c.has('l')); + assert.isTrue(await c.has('m')); + assert.isTrue(await c.has('n')); + assert.isFalse(await c.has('o')); + }); + + test('forEach', async () => { + let [c] = build(); + + let values = []; + await c.forEach((k) => { values.push(k); }); + assert.deepEqual(['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n'], values); + }); + + async function asyncAssertThrows(f: () => any):Promise { + let error: any = null; + try { + await f(); + } catch (er) { + error = er; + } + + return error !== null; + } + + test('advanceTo', async () => { + let [c] = build(); + + let cursor = await c.newCursorAt(null); + assert.ok(cursor); + assert.strictEqual('a', cursor.getCurrent()); + + assert.isTrue(await cursor.advanceTo('h')); + assert.strictEqual('h', cursor.getCurrent()); + + assert.isTrue(await cursor.advanceTo('k')); + assert.strictEqual('m', cursor.getCurrent()); + + assert.isFalse(await cursor.advanceTo('z')); // not found + assert.isFalse(cursor.valid); + + cursor = await c.newCursorAt('x'); // not found + assert.isFalse(cursor.valid); + + cursor = await c.newCursorAt('e'); + assert.ok(cursor); + assert.strictEqual('e', cursor.getCurrent()); + + assert.isTrue(await cursor.advanceTo('m')); + assert.strictEqual('m', cursor.getCurrent()); + + assert.isTrue(await cursor.advanceTo('n')); + assert.strictEqual('n', cursor.getCurrent()); + + assert.isFalse(await cursor.advanceTo('s')); + assert.isFalse(cursor.valid); + + asyncAssertThrows(async () => { + await notNull(cursor).advanceTo('x'); + }); + }); +}); + diff --git a/js/src/struct.js b/js/src/struct.js index a1ff31f5ac..346c61c5b6 100644 --- a/js/src/struct.js +++ b/js/src/struct.js @@ -1,23 +1,21 @@ // @flow -import Ref from './ref.js'; -import {ensureRef} from './get_ref.js'; import {Field, StructDesc, Type} from './type.js'; import {invariant, notNull} from './assert.js'; +import {Value} from './value.js'; type StructData = {[key: string]: any}; -export default class Struct { - type: Type; +export default class Struct extends Value { desc: StructDesc; unionField: ?Field; _data: StructData; typeDef: Type; - _ref: Ref; constructor(type: Type, typeDef: Type, data: StructData) { - this.type = type; + super(type); + this.typeDef = typeDef; let desc = typeDef.desc; @@ -28,14 +26,6 @@ export default class Struct { this.unionField = validate(this); } - get ref(): Ref { - return this._ref = ensureRef(this._ref, this, this.type); - } - - equals(other: Struct): boolean { - return this.ref.equals(other.ref); - } - get fields(): Array { return this.desc.fields; } diff --git a/js/src/type.js b/js/src/type.js index 820ed4a619..49c0d119f4 100644 --- a/js/src/type.js +++ b/js/src/type.js @@ -21,6 +21,25 @@ class PrimitiveDesc { equals(other: TypeDesc): boolean { return other instanceof PrimitiveDesc && other.kind === this.kind; } + + get ordered(): boolean { + switch (this.kind) { + case Kind.Float32: + case Kind.Float64: + case Kind.Int8: + case Kind.Int16: + case Kind.Int32: + case Kind.Int64: + case Kind.Uint8: + case Kind.Uint16: + case Kind.Uint32: + case Kind.Uint64: + case Kind.String: + return true; + default: + return false; + } + } } class UnresolvedDesc { @@ -189,6 +208,15 @@ class Type { return this._desc.kind; } + get ordered(): boolean { + let desc = this._desc; + if (desc instanceof PrimitiveDesc) { + return desc.ordered; + } + + return false; + } + get desc(): TypeDesc { return this._desc; } @@ -265,7 +293,7 @@ function makePrimitiveType(k: NomsKind): Type { function makeCompoundType(k: NomsKind, ...elemTypes: Array): Type { if (elemTypes.length === 1) { invariant(k !== Kind.Map, 'Map requires 2 element types'); - invariant(k === Kind.Ref || k === Kind.List || k === Kind.Set || k === Kind.MetaSequence); + invariant(k === Kind.Ref || k === Kind.List || k === Kind.Set); } else { invariant(k === Kind.Map, 'Only Map can have multiple element types'); invariant(elemTypes.length === 2, 'Map requires 2 element types'); diff --git a/js/src/type_test.js b/js/src/type_test.js index 2e3ce6398d..ab943b7ab0 100644 --- a/js/src/type_test.js +++ b/js/src/type_test.js @@ -6,7 +6,7 @@ import {assert} from 'chai'; import {Field, makeCompoundType, makePrimitiveType, makeStructType, makeType} from './type.js'; import {Kind} from './noms_kind.js'; import {Package, registerPackage} from './package.js'; -import {readValue} from './decode.js'; +import {readValue} from './read_value.js'; import {suite, test} from 'mocha'; import {writeValue} from './encode.js'; diff --git a/js/src/value.js b/js/src/value.js new file mode 100644 index 0000000000..daa23abf16 --- /dev/null +++ b/js/src/value.js @@ -0,0 +1,69 @@ +// @flow + +import Ref from './ref.js'; +import type {primitive} from './primitives.js'; +import {ensureRef} from './get_ref.js'; +import {invariant} from './assert.js'; +import {Type} from './type.js'; + +export class Value { + type: Type; + _ref: ?Ref; + + constructor(type: Type) { + this.type = type; + this._ref = null; + } + + get ref(): Ref { + return ensureRef(this._ref, this, this.type); + } + + equals(other: Value): boolean { + return this.ref.equals(other.ref); + } +} + +export type valueOrPrimitive = Value | primitive; + +export function less(v1: any, v2: any): boolean { + invariant(v1 !== null && v1 !== undefined && v2 !== null && v2 !== undefined); + + if (v1 instanceof Ref) { + invariant(v2 instanceof Ref); + return v1.compare(v2) < 0; + } + + if (typeof v1 === 'object') { + invariant(v1.ref instanceof Ref); + invariant(v2.ref instanceof Ref); + return v1.ref.compare(v2.ref) < 0; + } + + if (typeof v1 === 'string') { + invariant(typeof v2 === 'string'); + return v1 < v2; + } + + invariant(typeof v1 === 'number'); + invariant(typeof v2 === 'number'); + return v1 < v2; +} + +export function equals(v1: valueOrPrimitive, v2: valueOrPrimitive): boolean { + invariant(v1 !== null && v1 !== undefined && v2 !== null && v2 !== undefined); + + if (typeof v1 === 'object') { + invariant(typeof v2 === 'object'); + return (v1:Value).equals((v2:Value)); + } + + if (typeof v1 === 'string') { + invariant(typeof v2 === 'string'); + return v1 === v2; + } + + invariant(typeof v1 === 'number'); + invariant(typeof v2 === 'number'); + return v1 === v2; +}