From f26a94584cc88587fbcd8ef4266713f4b80fe1aa Mon Sep 17 00:00:00 2001 From: Chris Masone Date: Mon, 25 Apr 2016 15:00:18 -0700 Subject: [PATCH] JS: Implement new DataStore client protocol This replaces the HTTP ChunkStore implementation with an implementation of our new DataStore client protocol. It migrates much of the batching logic from RemoteStore into the new BatchStore, which is analogous to a class we have on the Go side, but continues to use a Delegate to handle all the HTTP work. This patch also introduces ValueStore, which handles validating Values as they're written. Instead of handling Value reading and writing itself, DataStore now extends ValueStore. Towards #1280 --- datas/datastore_test.go | 2 +- datas/http_batch_store.go | 2 +- js/src/batch-store-adaptor.js | 45 ++++ js/src/batch-store-test.js | 39 +++ js/src/{remote-store.js => batch-store.js} | 82 +++--- js/src/blob-test.js | 5 +- js/src/blob.js | 10 +- js/src/chunk-serializer-test.js | 56 ++-- js/src/chunk-serializer.js | 73 +++++- js/src/data-store-test.js | 91 ++----- js/src/data-store.js | 147 ++--------- js/src/dataset-test.js | 8 +- js/src/decode-test.js | 83 +++--- js/src/decode.js | 15 +- js/src/encode-test.js | 58 ++-- js/src/encode.js | 35 +-- js/src/http-batch-store-test.js | 24 ++ js/src/{http-store.js => http-batch-store.js} | 61 ++--- js/src/list-test.js | 26 +- js/src/list.js | 12 +- js/src/map-test.js | 58 ++-- js/src/map.js | 12 +- js/src/meta-sequence.js | 40 +-- js/src/noms.js | 2 +- js/src/sequence.js | 8 +- js/src/set-test.js | 70 ++--- js/src/set.js | 12 +- js/src/specs-test.js | 12 +- js/src/specs.js | 7 +- js/src/struct-test.js | 5 +- js/src/type-test.js | 5 +- js/src/value-store-test.js | 124 +++++++++ js/src/value-store.js | 248 ++++++++++++++++++ 33 files changed, 884 insertions(+), 593 deletions(-) create mode 100644 js/src/batch-store-adaptor.js create mode 100644 js/src/batch-store-test.js rename js/src/{remote-store.js => batch-store.js} (60%) create mode 100644 js/src/http-batch-store-test.js rename js/src/{http-store.js => http-batch-store.js} (63%) create mode 100644 js/src/value-store-test.js create mode 100644 js/src/value-store.js diff --git a/datas/datastore_test.go b/datas/datastore_test.go index e6e13a0aef..b61bec4c6a 100644 --- a/datas/datastore_test.go +++ b/datas/datastore_test.go @@ -138,7 +138,7 @@ func (suite *DataStoreSuite) TestDataStoreCommit() { // \----|c| // Should be disallowed. c := types.NewString("c") - cCommit := NewCommit().Set(ValueField, c) + cCommit := NewCommit().Set(ValueField, c).Set(ParentsField, NewSetOfRefOfCommit().Insert(types.NewTypedRefFromValue(aCommit))) suite.ds, err = suite.ds.Commit(datasetID, cCommit) suite.Error(err) suite.True(suite.ds.Head(datasetID).Get(ValueField).Equals(b)) diff --git a/datas/http_batch_store.go b/datas/http_batch_store.go index 688f6dd073..1af8207d31 100644 --- a/datas/http_batch_store.go +++ b/datas/http_batch_store.go @@ -139,7 +139,7 @@ func (bhcs *httpBatchStore) batchGetRequests() { func (bhcs *httpBatchStore) sendGetRequests(req chunks.ReadRequest) { batch := chunks.ReadBatch{} - refs := types.Hints{} + refs := map[ref.Ref]struct{}{} addReq := func(req chunks.ReadRequest) { r := req.Ref() diff --git a/js/src/batch-store-adaptor.js b/js/src/batch-store-adaptor.js new file mode 100644 index 0000000000..fa3cf11bda --- /dev/null +++ b/js/src/batch-store-adaptor.js @@ -0,0 +1,45 @@ +// @flow + +import Ref from './ref.js'; +import MemoryStore from './memory-store.js'; +import BatchStore from './batch-store.js'; +import type {ChunkStore} from './chunk-store.js'; +import type {UnsentReadMap, WriteRequest} from './batch-store.js'; + +export function makeTestingBatchStore(): BatchStore { + return new BatchStore(3, new BatchStoreAdaptorDelegate(new MemoryStore())); +} + +export default class BatchStoreAdaptor extends BatchStore { + constructor(cs: ChunkStore, maxReads: number = 3) { + super(maxReads, new BatchStoreAdaptorDelegate(cs)); + } +} + +export class BatchStoreAdaptorDelegate { + _cs: ChunkStore; + + constructor(cs: ChunkStore) { + this._cs = cs; + } + + async readBatch(reqs: UnsentReadMap): Promise { + Object.keys(reqs).forEach(refStr => { + this._cs.get(Ref.parse(refStr)).then(chunk => { reqs[refStr](chunk); }); + }); + } + + async writeBatch(reqs: Array): Promise { + reqs.forEach(req => { + this._cs.put(req.c); + }); + } + + async getRoot(): Promise { + return this._cs.getRoot(); + } + + async updateRoot(current: Ref, last: Ref): Promise { + return this._cs.updateRoot(current, last); + } +} diff --git a/js/src/batch-store-test.js b/js/src/batch-store-test.js new file mode 100644 index 0000000000..dc491ff07b --- /dev/null +++ b/js/src/batch-store-test.js @@ -0,0 +1,39 @@ +// @flow + +import {suite, test} from 'mocha'; +import {assert} from 'chai'; +import MemoryStore from './memory-store.js'; +import BatchStore from './batch-store.js'; +import {BatchStoreAdaptorDelegate} from './batch-store-adaptor.js'; +import {stringType} from './type.js'; +import {encodeNomsValue} from './encode.js'; + +suite('BatchStore', () => { + test('get after schedulePut works immediately', async () => { + const ms = new MemoryStore(); + const bs = new BatchStore(3, new BatchStoreAdaptorDelegate(ms)); + const input = 'abc'; + + const c = encodeNomsValue(input, stringType); + bs.schedulePut(c, new Set()); + + const chunk = await bs.get(c.ref); + assert.isTrue(c.ref.equals(chunk.ref)); + }); + + test('get after schedulePut works after flush', async () => { + const ms = new MemoryStore(); + const bs = new BatchStore(3, new BatchStoreAdaptorDelegate(ms)); + const input = 'abc'; + + const c = encodeNomsValue(input, stringType); + bs.schedulePut(c, new Set()); + + let chunk = await bs.get(c.ref); + assert.isTrue(c.ref.equals(chunk.ref)); + + await bs.flush(); + chunk = await bs.get(c.ref); + assert.isTrue(c.ref.equals(chunk.ref)); + }); +}); diff --git a/js/src/remote-store.js b/js/src/batch-store.js similarity index 60% rename from js/src/remote-store.js rename to js/src/batch-store.js index beb8361987..ddf5a318fd 100644 --- a/js/src/remote-store.js +++ b/js/src/batch-store.js @@ -7,15 +7,20 @@ import {notNull} from './assert.js'; type PendingReadMap = { [key: string]: Promise }; export type UnsentReadMap = { [key: string]: (c: Chunk) => void }; -export type WriteMap = { [key: string]: Chunk }; +type WriteMap = { [key: string]: Chunk }; +export type WriteRequest = { + c: Chunk; + hints: Set; +} interface Delegate { readBatch(reqs: UnsentReadMap): Promise; - writeBatch(reqs: WriteMap): Promise; + writeBatch(reqs: Array): Promise; + getRoot(): Promise; updateRoot(current: Ref, last: Ref): Promise; } -export class RemoteStore { +export default class BatchStore { _pendingReads: PendingReadMap; _unsentReads: ?UnsentReadMap; _readScheduled: boolean; @@ -23,15 +28,10 @@ export class RemoteStore { _maxReads: number; _pendingWrites: WriteMap; - _unsentWrites: ?WriteMap; - _writeScheduled: boolean; - _activeWrites: number; - _maxWrites: number; - _allWritesFinishedFn: ?() => void; - _canUpdateRoot: Promise; + _unsentWrites: ?Array; _delegate: Delegate; - constructor(maxReads: number, maxWrites: number, delegate: Delegate) { + constructor(maxReads: number, delegate: Delegate) { this._pendingReads = Object.create(null); this._unsentReads = null; this._readScheduled = false; @@ -40,20 +40,19 @@ export class RemoteStore { this._pendingWrites = Object.create(null); this._unsentWrites = null; - this._writeScheduled = false; - this._activeWrites = 0; - this._maxWrites = maxWrites; - this._allWritesFinishedFn = null; - this._canUpdateRoot = Promise.resolve(); this._delegate = delegate; } get(ref: Ref): Promise { const refStr = ref.toString(); - const p = this._pendingReads[refStr]; + let p = this._pendingReads[refStr]; if (p) { return p; } + p = this._pendingWrites[refStr]; + if (p) { + return Promise.resolve(p); + } return this._pendingReads[refStr] = new Promise(resolve => { if (!this._unsentReads) { @@ -92,7 +91,7 @@ export class RemoteStore { this._maybeStartRead(); } - put(c: Chunk): void { + schedulePut(c: Chunk, hints: Set): void { const refStr = c.ref.toString(); if (this._pendingWrites[refStr]) { return; // Already in flight. @@ -100,60 +99,41 @@ export class RemoteStore { this._pendingWrites[refStr] = c; if (!this._unsentWrites) { - this._unsentWrites = Object.create(null); + this._unsentWrites = []; } - this._unsentWrites[refStr] = c; - - if (!this._allWritesFinishedFn) { - this._canUpdateRoot = new Promise(resolve => { - this._allWritesFinishedFn = resolve; - }); - } - - this._maybeStartWrite(); + this._unsentWrites.push({c: c, hints: hints}); } - _maybeStartWrite() { - if (!this._writeScheduled && this._unsentWrites && this._activeWrites < this._maxWrites) { - this._writeScheduled = true; - setTimeout(() => { - this._write(); - }, 0); + async flush(): Promise { + if (!this._unsentWrites) { + return; } - } - - async _write(): Promise { - this._activeWrites++; const reqs = notNull(this._unsentWrites); this._unsentWrites = null; - this._writeScheduled = false; - await this._delegate.writeBatch(reqs); + await this._delegate.writeBatch(reqs); // TODO: Deal with backpressure const self = this; // TODO: Remove this when babel bug is fixed. - Object.keys(reqs).forEach(refStr => { - delete self._pendingWrites[refStr]; + reqs.forEach(req => { + delete self._pendingWrites[req.c.ref.toString()]; }); + } - this._activeWrites--; - - if (this._activeWrites === 0 && !this._unsentWrites) { - notNull(this._allWritesFinishedFn)(); - this._allWritesFinishedFn = null; - } - - this._maybeStartWrite(); + async getRoot(): Promise { + return this._delegate.getRoot(); } async updateRoot(current: Ref, last: Ref): Promise { - await this._canUpdateRoot; + await this.flush(); if (current.equals(last)) { - return Promise.resolve(true); + return true; } return this._delegate.updateRoot(current, last); } + // TODO: Should close() call flush() and block until it's done? Maybe closing with outstanding + // requests should be an error on both sides. TBD. close() {} } diff --git a/js/src/blob-test.js b/js/src/blob-test.js index 85009670ea..d26d7185d4 100644 --- a/js/src/blob-test.js +++ b/js/src/blob-test.js @@ -3,7 +3,7 @@ import {assert} from 'chai'; import {suite, test} from 'mocha'; import Random from './pseudo-random.js'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {newBlob, BlobWriter} from './blob.js'; import DataStore from './data-store.js'; import {blobType, makeRefType} from './type.js'; @@ -48,8 +48,7 @@ suite('Blob', () => { }); test('roundtrip', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const b1 = await newBlob(randomArray(15)); const r1 = await ds.writeValue(b1).targetRef; diff --git a/js/src/blob.js b/js/src/blob.js index 4ae403cd21..422771a830 100644 --- a/js/src/blob.js +++ b/js/src/blob.js @@ -4,7 +4,7 @@ import {Collection} from './collection.js'; import {IndexedSequence} from './indexed-sequence.js'; import {SequenceCursor} from './sequence.js'; import {invariant} from './assert.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import {blobType} from './type.js'; import { MetaTuple, @@ -59,9 +59,9 @@ export class BlobReader { } export class BlobLeafSequence extends IndexedSequence { - constructor(cs: ?DataStore, items: Uint8Array) { + constructor(vr: ?ValueReader, items: Uint8Array) { // $FlowIssue: The super class expects Array but we sidestep that. - super(cs, blobType, items); + super(vr, blobType, items); } getOffset(idx: number): number { @@ -72,9 +72,9 @@ export class BlobLeafSequence extends IndexedSequence { const blobWindowSize = 64; const blobPattern = ((1 << 13) | 0) - 1; -function newBlobLeafChunkFn(cs: ?DataStore = null): makeChunkFn { +function newBlobLeafChunkFn(vr: ?ValueReader = null): makeChunkFn { return (items: Array) => { - const blobLeaf = new BlobLeafSequence(cs, new Uint8Array(items)); + const blobLeaf = new BlobLeafSequence(vr, new Uint8Array(items)); const mt = new MetaTuple(blobLeaf, items.length, items.length); return [mt, blobLeaf]; }; diff --git a/js/src/chunk-serializer-test.js b/js/src/chunk-serializer-test.js index 6348c8b3ca..724306312c 100644 --- a/js/src/chunk-serializer-test.js +++ b/js/src/chunk-serializer-test.js @@ -3,42 +3,68 @@ import {suite, test} from 'mocha'; import {assert} from 'chai'; import Chunk from './chunk.js'; +import Ref from './ref.js'; import {deserialize, serialize} from './chunk-serializer.js'; suite('ChunkSerializer', () => { + function assertHints(expect: Array, actual: Array) { + assert.strictEqual(actual.length, expect.length); + for (let i = 0; i < expect.length; i++) { + assert.isTrue(expect[i].equals(actual[i])); + } + } + function assertChunks(expect: Array, actual: Array) { - assert.strictEqual(expect.length, actual.length); + assert.strictEqual(actual.length, expect.length); for (let i = 0; i < expect.length; i++) { assert.isTrue(expect[i].ref.equals(actual[i].ref)); } } test('simple', () => { - const chunks = [Chunk.fromString('abc'), Chunk.fromString('def'), Chunk.fromString('ghi'), - Chunk.fromString('wacka wack wack')]; + const expHints = []; + const expChunks = [Chunk.fromString('abc'), Chunk.fromString('def'), Chunk.fromString('ghi'), + Chunk.fromString('wacka wack wack')]; - const buffer = serialize(chunks); - const newChunks = deserialize(buffer); + const {hints, chunks} = deserialize(serialize(new Set(expHints), expChunks)); - assertChunks(chunks, newChunks); + assertHints(expHints, hints); + assertChunks(expChunks, chunks); }); test('leading & trailing empty', () => { - const chunks = [Chunk.fromString(''), Chunk.fromString('A'), Chunk.fromString('')]; + const expHints = []; + const expChunks = [Chunk.fromString(''), Chunk.fromString('A'), Chunk.fromString('')]; - const buffer = serialize(chunks); - const newChunks = deserialize(buffer); + const {hints, chunks} = deserialize(serialize(new Set(expHints), expChunks)); - assertChunks(chunks, newChunks); + assertHints(expHints, hints); + assertChunks(expChunks, chunks); }); - test('no chunks', () => { - const chunks = []; + test('all empty', () => { + const expHints = []; + const expChunks = []; - const buffer = serialize(chunks); - const newChunks = deserialize(buffer); + const {hints, chunks} = deserialize(serialize(new Set(expHints), expChunks)); - assertChunks(chunks, newChunks); + assertHints(expHints, hints); + assertChunks(expChunks, chunks); + }); + + test('with hints', () => { + const expHints = [ + Chunk.fromString('123').ref, + Chunk.fromString('456').ref, + Chunk.fromString('789').ref, + Chunk.fromString('wacka wack wack').ref, + ]; + const expChunks = [Chunk.fromString('abc'), Chunk.fromString('def'), Chunk.fromString('ghi')]; + + const {hints, chunks} = deserialize(serialize(new Set(expHints), expChunks)); + + assertHints(expHints, hints); + assertChunks(expChunks, chunks); }); }); diff --git a/js/src/chunk-serializer.js b/js/src/chunk-serializer.js index 602f1b1e10..e81df16cc4 100644 --- a/js/src/chunk-serializer.js +++ b/js/src/chunk-serializer.js @@ -4,21 +4,22 @@ import Chunk from './chunk.js'; import Ref from './ref.js'; import {invariant} from './assert.js'; +const headerSize = 4; // uint32 +const littleEndian = true; const sha1Size = 20; const chunkLengthSize = 4; // uint32 const chunkHeaderSize = sha1Size + chunkLengthSize; -export function serialize(chunks: Array): ArrayBuffer { - let totalSize = 0; - for (let i = 0; i < chunks.length; i++) { - totalSize += chunkHeaderSize + chunks[i].data.length; - } +export function serialize(hints: Set, chunks: Array): ArrayBuffer { + const buffer = new ArrayBuffer(serializedHintLength(hints) + serializedChunkLength(chunks)); - const buffer = new ArrayBuffer(totalSize); - let offset = 0; + let offset = serializeHints(hints, buffer); for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; + invariant(buffer.byteLength - offset >= chunkHeaderSize + chunk.data.length, + 'Invalid chunk buffer'); + const refArray = new Uint8Array(buffer, offset, sha1Size); refArray.set(chunk.ref.digest); offset += sha1Size; @@ -40,11 +41,65 @@ export function serialize(chunks: Array): ArrayBuffer { return buffer; } -export function deserialize(buffer: ArrayBuffer): Array { +function serializeHints(hints: Set, buffer: ArrayBuffer): number { + let offset = 0; + const view = new DataView(buffer, offset, headerSize); + view.setUint32(offset, hints.size | 0, littleEndian); // Coerce number to uint32 + offset += headerSize; + + hints.forEach(ref => { + const refArray = new Uint8Array(buffer, offset, sha1Size); + refArray.set(ref.digest); + offset += sha1Size; + }); + + return offset; +} + +function serializedHintLength(hints: Set): number { + return headerSize + sha1Size * hints.size; +} + +function serializedChunkLength(chunks: Array): number { + let totalSize = 0; + for (let i = 0; i < chunks.length; i++) { + totalSize += chunkHeaderSize + chunks[i].data.length; + } + return totalSize; +} + +export function deserialize(buffer: ArrayBuffer): {hints: Array, chunks: Array} { + const {hints, offset} = deserializeHints(buffer); + return {hints: hints, chunks: deserializeChunks(buffer, offset)}; +} + +function deserializeHints(buffer: ArrayBuffer): {hints: Array, offset: number} { + const hints:Array = []; + + let offset = 0; + const view = new DataView(buffer, 0, headerSize); + const numHints = view.getUint32(0, littleEndian); + offset += headerSize; + + const totalLength = headerSize + (numHints * sha1Size); + for (; offset < totalLength;) { + invariant(buffer.byteLength - offset >= sha1Size, 'Invalid hint buffer'); + + const refArray = new Uint8Array(buffer, offset, sha1Size); + const ref = Ref.fromDigest(new Uint8Array(refArray)); + offset += sha1Size; + + hints.push(ref); + } + + return {hints: hints, offset: offset}; +} + +export function deserializeChunks(buffer: ArrayBuffer, offset: number = 0): Array { const chunks:Array = []; const totalLenth = buffer.byteLength; - for (let offset = 0; offset < totalLenth;) { + for (; offset < totalLenth;) { invariant(buffer.byteLength - offset >= chunkHeaderSize, 'Invalid chunk buffer'); const refArray = new Uint8Array(buffer, offset, sha1Size); diff --git a/js/src/data-store-test.js b/js/src/data-store-test.js index 4ee590777e..8eec489497 100644 --- a/js/src/data-store-test.js +++ b/js/src/data-store-test.js @@ -1,34 +1,35 @@ // @flow import {suite, test} from 'mocha'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {emptyRef} from './ref.js'; import {assert} from 'chai'; import {default as DataStore, getDatasTypes, newCommit} from './data-store.js'; import {invariant, notNull} from './assert.js'; import {newMap} from './map.js'; import {stringType} from './type.js'; -import {getRef} from './get-ref.js'; import {encodeNomsValue} from './encode.js'; suite('DataStore', () => { test('access', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const bs = makeTestingBatchStore(); + const ds = new DataStore(bs); const input = 'abc'; const c = encodeNomsValue(input, stringType); const v1 = await ds.readValue(c.ref); assert.equal(null, v1); - ms.put(c); + bs.schedulePut(c, new Set()); + bs.flush(); + const v2 = await ds.readValue(c.ref); assert.equal('abc', v2); }); test('commit', async () => { - const ms = new MemoryStore(); - let ds = new DataStore(ms); + const bs = new makeTestingBatchStore(); + let ds = new DataStore(bs); const datasetID = 'ds1'; const datasets = await ds.datasets(); @@ -92,14 +93,14 @@ suite('DataStore', () => { assert.strictEqual('a', notNull(await ds.head('otherDs')).value); // Get a fresh datastore, and verify that both datasets are present - const newDs = new DataStore(ms); + const newDs = new DataStore(bs); assert.strictEqual('d', notNull(await newDs.head(datasetID)).value); assert.strictEqual('a', notNull(await newDs.head('otherDs')).value); }); test('concurrency', async () => { - const ms = new MemoryStore(); - let ds = new DataStore(ms); + const bs = new makeTestingBatchStore(); + let ds = new DataStore(bs); const datasetID = 'ds1'; // |a| @@ -112,7 +113,7 @@ suite('DataStore', () => { assert.strictEqual('b', notNull(await ds.head(datasetID)).value); // Important to create this here. - let ds2 = new DataStore(ms); + let ds2 = new DataStore(bs); // Change 1: // |a| <- |b| <- |c| @@ -137,15 +138,14 @@ suite('DataStore', () => { test('empty datasets', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const datasets = await ds.datasets(); assert.strictEqual(0, datasets.size); }); test('head', async () => { - const ms = new MemoryStore(); - let ds = new DataStore(ms); + const bs = new makeTestingBatchStore(); + let ds = new DataStore(bs); const types = getDatasTypes(); const commit = await newCommit('foo', []); @@ -153,8 +153,8 @@ suite('DataStore', () => { const commitRef = ds.writeValue(commit); const datasets = await newMap(['foo', commitRef], types.commitMapType); const rootRef = ds.writeValue(datasets).targetRef; - assert.isTrue(await ms.updateRoot(rootRef, emptyRef)); - ds = new DataStore(ms); // refresh the datasets + assert.isTrue(await bs.updateRoot(rootRef, emptyRef)); + ds = new DataStore(bs); // refresh the datasets assert.strictEqual(1, datasets.size); const fooHead = await ds.head('foo'); @@ -163,61 +163,4 @@ suite('DataStore', () => { const barHead = await ds.head('bar'); assert.isNull(barHead); }); - - test('writeValue primitives', async () => { - const ds = new DataStore(new MemoryStore()); - - const r1 = ds.writeValue('hello').targetRef; - const r2 = ds.writeValue(false).targetRef; - const r3 = ds.writeValue(2).targetRef; - - const v1 = await ds.readValue(r1); - assert.equal('hello', v1); - const v2 = await ds.readValue(r2); - assert.equal(false, v2); - const v3 = await ds.readValue(r3); - assert.equal(2, v3); - }); - - test('caching', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms, 1e6); - - const r1 = ds.writeValue('hello').targetRef; - (ms: any).get = (ms: any).put = () => { assert.fail('unreachable'); }; - const v1 = await ds.readValue(r1); - assert.equal(v1, 'hello'); - const r2 = ds.writeValue('hello').targetRef; - assert.isTrue(r1.equals(r2)); - }); - - test('caching eviction', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms, 15); - - const r1 = ds.writeValue('hello').targetRef; - const r2 = ds.writeValue('world').targetRef; - (ms: any).get = () => { throw new Error(); }; - const v2 = await ds.readValue(r2); - assert.equal(v2, 'world'); - let ex; - try { - await ds.readValue(r1); - } catch (e) { - ex = e; - } - assert.instanceOf(ex, Error); - }); - - test('caching has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms, 1e6); - - const r1 = getRef('hello', stringType); - const v1 = await ds.readValue(r1); - assert.equal(v1, null); - (ms: any).get = (ms: any).has = () => { assert.fail('unreachable'); }; - const v2 = await ds.readValue(r1); - assert.equal(v2, null); - }); }); diff --git a/js/src/data-store.js b/js/src/data-store.js index 00d0b5c3c9..8095f2d244 100644 --- a/js/src/data-store.js +++ b/js/src/data-store.js @@ -1,16 +1,15 @@ // @flow -import Chunk from './chunk.js'; import Ref from './ref.js'; -import {default as RefValue, refValueFromValue} from './ref-value.js'; +import {default as RefValue} from './ref-value.js'; import {newStruct} from './struct.js'; -import type {ChunkStore} from './chunk-store.js'; import type {NomsMap} from './map.js'; import type {NomsSet} from './set.js'; import type {valueOrPrimitive} from './value.js'; +import ValueStore from './value-store.js'; +import BatchStore from './batch-store.js'; import { Field, - getTypeOfValue, makeRefType, makeStructType, makeSetType, @@ -21,9 +20,6 @@ import { } from './type.js'; import {newMap} from './map.js'; import {newSet} from './set.js'; -import {decodeNomsValue} from './decode.js'; -import {invariant} from './assert.js'; -import {encodeNomsValue} from './encode.js'; import type {Commit} from './commit.js'; type DatasTypes = { @@ -66,21 +62,17 @@ export function getDatasTypes(): DatasTypes { return datasTypes; } -interface Cache { // eslint-disable-line no-undef - entry(ref: Ref): ?CacheEntry; // eslint-disable-line no-undef - get(ref: Ref): ?T; // eslint-disable-line no-undef - add(ref: Ref, size: number, value: T): void; // eslint-disable-line no-undef -} - -export default class DataStore { - _cs: ChunkStore; +export default class DataStore extends ValueStore { + _bs: BatchStore; + _cacheSize: number; _datasets: Promise>>; - _valueCache: Cache; - constructor(cs: ChunkStore, cacheSize: number = 0) { - this._cs = cs; - this._datasets = this._datasetsFromRootRef(cs.getRoot()); - this._valueCache = cacheSize > 0 ? new SizeCache(cacheSize) : new NoopCache(); + constructor(bs: BatchStore, cacheSize: number = 0) { + super(bs, cacheSize); + // bs and cacheSize should only be used when creating a new DataStore instance in commit() + this._bs = bs; + this._cacheSize = cacheSize; + this._datasets = this._datasetsFromRootRef(bs.getRoot()); } _datasetsFromRootRef(rootRef: Promise): Promise>> { @@ -116,40 +108,8 @@ export default class DataStore { return true; } - // TODO: This should return Promise - async readValue(ref: Ref): Promise { - const entry = this._valueCache.entry(ref); - if (entry) { - return entry.value; - } - const chunk: Chunk = await this._cs.get(ref); - if (chunk.isEmpty()) { - this._valueCache.add(ref, 0, null); - return null; - } - - const v = decodeNomsValue(chunk, this); - this._valueCache.add(ref, chunk.data.length, v); - return v; - } - - writeValue(v: T): RefValue { - const t = getTypeOfValue(v); - const chunk = encodeNomsValue(v, t, this); - invariant(!chunk.isEmpty()); - const {ref} = chunk; - const refValue = refValueFromValue(v); - const entry = this._valueCache.entry(ref); - if (entry && entry.present) { - return refValue; - } - this._cs.put(chunk); - this._valueCache.add(ref, chunk.data.length, v); - return refValue; - } - async commit(datasetId: string, commit: Commit): Promise { - const currentRootRefP = this._cs.getRoot(); + const currentRootRefP = this._bs.getRoot(); const datasetsP = this._datasetsFromRootRef(currentRootRefP); let currentDatasets = await (datasetsP:Promise); const currentRootRef = await currentRootRefP; @@ -169,8 +129,8 @@ export default class DataStore { currentDatasets = await currentDatasets.set(datasetId, commitRef); const newRootRef = this.writeValue(currentDatasets).targetRef; - if (await this._cs.updateRoot(newRootRef, currentRootRef)) { - return new DataStore(this._cs); + if (await this._bs.updateRoot(newRootRef, currentRootRef)) { + return new DataStore(this._bs, this._cacheSize); } throw new Error('Optimistic lock failed'); @@ -192,79 +152,4 @@ export function newCommit(value: valueOrPrimitive, const types = getDatasTypes(); return newSet(parentsArr, types.commitSetType).then(parents => newStruct(types.commitType, {value, parents})); -} - -class CacheEntry { - size: number; - value: ?T; - - constructor(size: number, value: ?T) { - this.size = size; - this.value = value; - } - - get present(): boolean { - return this.value !== null; - } -} - -/** - * This uses a Map as an LRU cache. It uses the behavior that iteration of keys in a Map is done in - * insertion order and any time a value is checked it is taken out and reinserted which puts it last - * in the iteration. - */ -class SizeCache { - _size: number; - _maxSize: number; - _cache: Map>; - - constructor(size: number) { - this._maxSize = size; - this._cache = new Map(); - this._size = 0; - } - - entry(ref: Ref): ?CacheEntry { - const key = ref.toString(); - const entry = this._cache.get(key); - if (!entry) { - return undefined; - } - this._cache.delete(key); - this._cache.set(key, entry); - return entry; - } - - get(ref: Ref): ?T { - const entry = this.entry(ref); - return entry ? entry.value : undefined; - } - - add(ref: Ref, size: number, value: ?T) { - const key = ref.toString(); - if (this._cache.has(key)) { - this._cache.delete(key); - } else { - this._size += size; - } - this._cache.set(key, new CacheEntry(size, value)); - - if (this._size > this._maxSize) { - for (const [key, {size}] of this._cache) { - if (this._size <= this._maxSize) { - break; - } - this._cache.delete(key); - this._size -= size; - } - } - } -} - -class NoopCache { - entry(ref: Ref): ?CacheEntry {} // eslint-disable-line no-unused-vars - - get(ref: Ref): ?T {} // eslint-disable-line no-unused-vars - - add(ref: Ref, size: number, value: T) {} // eslint-disable-line no-unused-vars -} +} \ No newline at end of file diff --git a/js/src/dataset-test.js b/js/src/dataset-test.js index b0fe12ca48..c11cf4ec10 100644 --- a/js/src/dataset-test.js +++ b/js/src/dataset-test.js @@ -1,7 +1,7 @@ // @flow import {suite, test} from 'mocha'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {assert} from 'chai'; import Dataset from './dataset.js'; import DataStore from './data-store.js'; @@ -9,8 +9,8 @@ import {invariant, notNull} from './assert.js'; suite('Dataset', () => { test('commit', async () => { - const ms = new MemoryStore(); - const store = new DataStore(ms); + const bs = makeTestingBatchStore(); + const store = new DataStore(bs); let ds = new Dataset(store, 'ds1'); // |a| @@ -54,7 +54,7 @@ suite('Dataset', () => { assert.strictEqual('a', notNull(await ds.head('otherDs')).value); // Get a fresh datastore, and verify that both datasets are present - const newStore = new DataStore(ms); + const newStore = new DataStore(bs); assert.strictEqual('d', notNull(await newStore.head('ds1')).value); assert.strictEqual('a', notNull(await newStore.head('otherDs')).value); }); diff --git a/js/src/decode-test.js b/js/src/decode-test.js index b0d5d358f0..61aabba16a 100644 --- a/js/src/decode-test.js +++ b/js/src/decode-test.js @@ -2,7 +2,7 @@ import Chunk from './chunk.js'; import DataStore from './data-store.js'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import Ref from './ref.js'; import RefValue from './ref-value.js'; import {default as Struct, StructMirror} from './struct.js'; @@ -44,8 +44,7 @@ suite('Decode', () => { } test('read', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [1, 'hi', true]; const r = new JsonArrayReader(a, ds); @@ -60,8 +59,7 @@ suite('Decode', () => { }); test('read type as tag', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); function doTest(expected: Type, a: Array) { const r = new JsonArrayReader(a, ds); const tr = r.readTypeAsTag([]); @@ -75,8 +73,7 @@ suite('Decode', () => { }); test('read primitives', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); async function doTest(expected: any, a: Array): Promise { const r = new JsonArrayReader(a, ds); @@ -96,8 +93,7 @@ suite('Decode', () => { }); test('read list of number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.List, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsList = await r.readTopLevelValue(); @@ -110,8 +106,7 @@ suite('Decode', () => { // TODO: Can't round-trip collections of value types. =-( test('read list of value', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.List, Kind.Value, false, [Kind.Number, '1', Kind.String, 'hi', Kind.Bool, true]]; const r = new JsonArrayReader(a, ds); @@ -126,8 +121,7 @@ suite('Decode', () => { }); test('read value list of number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Value, Kind.List, Kind.Number, false, ['0', '1', '2']]; const r = new JsonArrayReader(a, ds); const v = await r.readTopLevelValue(); @@ -139,8 +133,7 @@ suite('Decode', () => { }); test('read compound list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const ltr = makeListType(numberType); const r1 = ds.writeValue(new NomsList(ltr, new ListLeafSequence(ds, ltr, [0]))).targetRef; const r2 = ds.writeValue(new NomsList(ltr, new ListLeafSequence(ds, ltr, [1, 2]))).targetRef; @@ -161,8 +154,7 @@ suite('Decode', () => { }); test('read map of number to number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Map, Kind.Number, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsMap = await r.readTopLevelValue(); @@ -174,8 +166,7 @@ suite('Decode', () => { }); test('read map of ref to number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Map, Kind.Ref, Kind.Value, Kind.Number, false, ['sha1-0000000000000000000000000000000000000001', '2', 'sha1-0000000000000000000000000000000000000002', '4']]; @@ -195,8 +186,7 @@ suite('Decode', () => { }); test('read value map of number to number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Value, Kind.Map, Kind.Number, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsMap = await r.readTopLevelValue(); @@ -208,8 +198,7 @@ suite('Decode', () => { }); test('read set of number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Set, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsSet = await r.readTopLevelValue(); @@ -221,8 +210,7 @@ suite('Decode', () => { }); test('read compound set', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const ltr = makeSetType(numberType); const r1 = ds.writeValue(new NomsSet(ltr, new SetLeafSequence(ds, ltr, [0]))).targetRef; const r2 = ds.writeValue(new NomsSet(ltr, new SetLeafSequence(ds, ltr, [1, 2]))).targetRef; @@ -243,8 +231,7 @@ suite('Decode', () => { }); test('read value set of number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Value, Kind.Set, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsSet = await r.readTopLevelValue(); @@ -267,8 +254,7 @@ suite('Decode', () => { } test('test read struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeStructType('A1', [ new Field('x', numberType), new Field('s', stringType), @@ -291,8 +277,7 @@ suite('Decode', () => { }); test('test read struct with list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const ltr = makeListType(numberType); const tr = makeStructType('A4', [ new Field('b', boolType), @@ -316,8 +301,7 @@ suite('Decode', () => { }); test('test read struct with value', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeStructType('A5', [ new Field('b', boolType), new Field('v', valueType), @@ -337,8 +321,7 @@ suite('Decode', () => { }); test('test read value struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeStructType('A1', [ new Field('x', numberType), new Field('s', stringType), @@ -362,8 +345,7 @@ suite('Decode', () => { }); test('test read map of string to struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeStructType('s', [ new Field('b', boolType), new Field('i', numberType), @@ -384,11 +366,10 @@ suite('Decode', () => { }); test('decodeNomsValue', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const chunk = Chunk.fromString( `t [${Kind.Value}, ${Kind.Set}, ${Kind.Number}, false, ["0", "1", "2", "3"]]`); - const v = decodeNomsValue(chunk, new DataStore(new MemoryStore())); + const v = decodeNomsValue(chunk, new DataStore(makeTestingBatchStore())); invariant(v instanceof NomsSet); const t = makeSetType(numberType); @@ -397,8 +378,8 @@ suite('Decode', () => { }); test('decodeNomsValue: counter with one commit', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const bs = makeTestingBatchStore(); + const ds = new DataStore(bs); const makeChunk = a => Chunk.fromString(`t ${JSON.stringify(a)}`); @@ -413,15 +394,16 @@ suite('Decode', () => { ['value', Kind.Value, 'parents', Kind.Set, Kind.Ref, Kind.Parent, 0], Kind.Number, '1', false, []]); const commitRef = commitChunk.ref; - ms.put(commitChunk); + bs.schedulePut(commitChunk, new Set()); // Root const rootChunk = makeChunk([Kind.Map, Kind.String, Kind.Ref, Kind.Struct, 'Commit', ['value', Kind.Value, 'parents', Kind.Set, Kind.Ref, Kind.Parent, 0], false, ['counter', commitRef.toString()]]); const rootRef = rootChunk.ref; - ms.put(rootChunk); + bs.schedulePut(rootChunk, new Set()); + await bs.flush(); const rootMap = await ds.readValue(rootRef); const counterRef = await rootMap.get('counter'); const commit = await counterRef.targetValue(ds); @@ -430,7 +412,7 @@ suite('Decode', () => { test('out of line blob', async () => { const chunk = Chunk.fromString('b hi'); - const blob = decodeNomsValue(chunk, new DataStore(new MemoryStore())); + const blob = decodeNomsValue(chunk, new DataStore(makeTestingBatchStore())); invariant(blob instanceof NomsBlob); const r = await blob.getReader().read(); assert.isFalse(r.done); @@ -448,7 +430,7 @@ suite('Decode', () => { } const chunk2 = new Chunk(data); - const blob2 = decodeNomsValue(chunk2, new DataStore(new MemoryStore())); + const blob2 = decodeNomsValue(chunk2, new DataStore(makeTestingBatchStore())); invariant(blob2 instanceof NomsBlob); const r2 = await blob2.getReader().read(); assert.isFalse(r2.done); @@ -458,8 +440,7 @@ suite('Decode', () => { }); test('inline blob', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [ Kind.List, Kind.Blob, false, [false, encodeBase64(stringToUint8Array('hello')), @@ -478,8 +459,7 @@ suite('Decode', () => { }); test('compound blob', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const r1 = ds.writeValue(await newBlob(stringToUint8Array('hi'))).targetRef; const r2 = ds.writeValue(await newBlob(stringToUint8Array('world'))).targetRef; @@ -497,8 +477,7 @@ suite('Decode', () => { }); test('recursive struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); // struct A { // b: struct B { diff --git a/js/src/decode.js b/js/src/decode.js index 739e54807d..059b5f9f07 100644 --- a/js/src/decode.js +++ b/js/src/decode.js @@ -6,7 +6,6 @@ import Ref from './ref.js'; import RefValue from './ref-value.js'; import {newStruct} from './struct.js'; import type Struct from './struct.js'; -import type DataStore from './data-store.js'; import type {NomsKind} from './noms-kind.js'; import {decode as decodeBase64} from './base64.js'; import { @@ -34,9 +33,9 @@ const blobTag = 'b '; export class JsonArrayReader { _a: Array; _i: number; - _ds: DataStore; + _ds: ValueReader; - constructor(a: Array, ds: DataStore) { + constructor(a: Array, ds: ValueReader) { this._a = a; this._i = 0; this._ds = ds; @@ -325,17 +324,21 @@ export class JsonArrayReader { } } -export function decodeNomsValue(chunk: Chunk, ds: DataStore): valueOrPrimitive { +export interface ValueReader { + readValue(ref: Ref): Promise +} + +export function decodeNomsValue(chunk: Chunk, vr: ValueReader): valueOrPrimitive { const tag = new Chunk(new Uint8Array(chunk.data.buffer, 0, 2)).toString(); switch (tag) { case typedTag: { const payload = JSON.parse(new Chunk(new Uint8Array(chunk.data.buffer, 2)).toString()); - const reader = new JsonArrayReader(payload, ds); + const reader = new JsonArrayReader(payload, vr); return reader.readTopLevelValue(); } case blobTag: - return new NomsBlob(new BlobLeafSequence(ds, new Uint8Array(chunk.data.buffer, 2))); + return new NomsBlob(new BlobLeafSequence(vr, new Uint8Array(chunk.data.buffer, 2))); default: throw new Error('Not implemented'); } diff --git a/js/src/encode-test.js b/js/src/encode-test.js index 112b2031da..c37441db1e 100644 --- a/js/src/encode-test.js +++ b/js/src/encode-test.js @@ -3,7 +3,7 @@ import {assert} from 'chai'; import {suite, test} from 'mocha'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import Ref from './ref.js'; import RefValue from './ref-value.js'; import {newStruct} from './struct.js'; @@ -35,8 +35,7 @@ import type {valueOrPrimitive} from './value.js'; suite('Encode', () => { test('write primitives', () => { function f(k: NomsKind, t:Type, v: valueOrPrimitive, ex: valueOrPrimitive) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); w.writeTopLevel(t, v); assert.deepEqual([k, ex], w.array); @@ -55,8 +54,7 @@ suite('Encode', () => { }); test('write simple blob', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const blob = await newBlob(new Uint8Array([0x00, 0x01])); w.writeTopLevel(blobType, blob); @@ -64,8 +62,7 @@ suite('Encode', () => { }); test('write list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const tr = makeCompoundType(Kind.List, numberType); @@ -75,8 +72,7 @@ suite('Encode', () => { }); test('write list of value', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const tr = makeCompoundType(Kind.List, valueType); @@ -91,8 +87,7 @@ suite('Encode', () => { }); test('write list of list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const it = makeCompoundType(Kind.List, numberType); @@ -107,8 +102,7 @@ suite('Encode', () => { }); test('write leaf set', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const tr = makeCompoundType(Kind.Set, numberType); @@ -118,8 +112,7 @@ suite('Encode', () => { }); test('write compound set', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const ltr = makeCompoundType(Kind.Set, numberType); const r1 = ds.writeValue(new NomsSet(ltr, new SetLeafSequence(ds, ltr, [0]))).targetRef; @@ -138,8 +131,7 @@ suite('Encode', () => { }); test('write set of set', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const st = makeCompoundType(Kind.Set, numberType); @@ -155,8 +147,7 @@ suite('Encode', () => { }); test('write map', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const tr = makeCompoundType(Kind.Map, stringType, boolType); @@ -167,8 +158,7 @@ suite('Encode', () => { }); test('write map of map', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const kt = makeCompoundType(Kind.Map, stringType, numberType); @@ -184,8 +174,7 @@ suite('Encode', () => { }); test('write empty struct', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const type = makeStructType('S', []); @@ -196,8 +185,7 @@ suite('Encode', () => { }); test('write struct', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const type = makeStructType('S', [ @@ -212,8 +200,7 @@ suite('Encode', () => { }); test('write struct with list', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); let w = new JsonArrayWriter(ds); const ltr = makeCompoundType(Kind.List, stringType); @@ -232,8 +219,7 @@ suite('Encode', () => { }); test('write struct with struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const s2Type = makeStructType('S2', [ @@ -250,8 +236,7 @@ suite('Encode', () => { }); test('write compound list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const ltr = makeListType(numberType); const r1 = ds.writeValue(new NomsList(ltr, new ListLeafSequence(ds, ltr, [0]))).targetRef; @@ -270,8 +255,7 @@ suite('Encode', () => { }); test('write type value', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const test = (expected: Array, v: Type) => { const w = new JsonArrayWriter(ds); @@ -314,8 +298,7 @@ suite('Encode', () => { return bytes; } - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const blob = await newBlob(stringToUint8Array('hi')); const chunk = encodeNomsValue(blob, blobType, ds); @@ -338,8 +321,7 @@ suite('Encode', () => { }); test('write ref', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const ref = Ref.parse('sha1-0123456789abcdef0123456789abcdef01234567'); const t = makeCompoundType(Kind.Ref, blobType); @@ -350,7 +332,7 @@ suite('Encode', () => { }); test('type errors', async () => { - const ds = new DataStore(new MemoryStore()); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const test = (et, at, t, v) => { diff --git a/js/src/encode.js b/js/src/encode.js index 959e158dcf..9011362c21 100644 --- a/js/src/encode.js +++ b/js/src/encode.js @@ -4,7 +4,6 @@ import Chunk from './chunk.js'; import type Ref from './ref.js'; import RefValue from './ref-value.js'; import {default as Struct, StructMirror} from './struct.js'; -import type DataStore from './data-store.js'; import type {NomsKind} from './noms-kind.js'; import {encode as encodeBase64} from './base64.js'; import {StructDesc, Type, getTypeOfValue} from './type.js'; @@ -28,11 +27,11 @@ type primitiveOrArray = primitive | Array; export class JsonArrayWriter { array: Array; - _ds: ?DataStore; + _vw: ?ValueWriter; - constructor(ds: ?DataStore) { + constructor(ds: ?ValueWriter) { this.array = []; - this._ds = ds; + this._vw = ds; } write(v: primitiveOrArray) { @@ -97,14 +96,14 @@ export class JsonArrayWriter { } this.write(true); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); const indexType = indexTypeForMetaSequence(t); for (let i = 0; i < v.items.length; i++) { const tuple = v.items[i]; invariant(tuple instanceof MetaTuple); - if (tuple.sequence && this._ds) { + if (tuple.sequence && this._vw) { const child = tuple.sequence; - this._ds.writeValue(child); + this._vw.writeValue(child); } w2.writeRef(tuple.ref); w2.writeValue(tuple.value, indexType); @@ -154,7 +153,7 @@ export class JsonArrayWriter { } invariant(sequence instanceof ListLeafSequence); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); const elemType = t.elemTypes[0]; sequence.items.forEach(sv => w2.writeValue(sv, elemType)); this.write(w2.array); @@ -170,7 +169,7 @@ export class JsonArrayWriter { } invariant(sequence instanceof MapLeafSequence); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); const keyType = t.elemTypes[0]; const valueType = t.elemTypes[1]; sequence.items.forEach(entry => { @@ -196,7 +195,7 @@ export class JsonArrayWriter { } invariant(sequence instanceof SetLeafSequence); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); const elemType = t.elemTypes[0]; const elems = []; sequence.items.forEach(v => { @@ -236,7 +235,7 @@ export class JsonArrayWriter { case Kind.Ref: case Kind.Set: { this.writeKind(k); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); t.elemTypes.forEach(elem => w2.writeTypeAsValue(elem, parentStructTypes)); this.write(w2.array); break; @@ -262,7 +261,7 @@ export class JsonArrayWriter { const desc = t.desc; this.writeKind(t.kind); this.write(t.name); - const fieldWriter = new JsonArrayWriter(this._ds); + const fieldWriter = new JsonArrayWriter(this._vw); desc.fields.forEach(field => { fieldWriter.write(field.name); fieldWriter.writeTypeAsTag(field.type, parentStructTypes); @@ -290,8 +289,8 @@ export class JsonArrayWriter { } } -function encodeEmbeddedNomsValue(v: valueOrPrimitive, t: Type, ds: ?DataStore): Chunk { - const w = new JsonArrayWriter(ds); +function encodeEmbeddedNomsValue(v: valueOrPrimitive, t: Type, vw: ?ValueWriter): Chunk { + const w = new JsonArrayWriter(vw); w.writeTopLevel(t, v); return Chunk.fromString(typedTag + JSON.stringify(w.array)); } @@ -309,7 +308,11 @@ function encodeTopLevelBlob(sequence: BlobLeafSequence): Chunk { return new Chunk(data); } -export function encodeNomsValue(v: valueOrPrimitive, t: Type, ds: ?DataStore): Chunk { +interface ValueWriter { + writeValue(v: T, t: ?Type): RefValue +} + +export function encodeNomsValue(v: valueOrPrimitive, t: Type, vw: ?ValueWriter): Chunk { if (t.kind === Kind.Blob) { invariant(v instanceof NomsBlob || v instanceof IndexedSequence); const sequence = v instanceof NomsBlob ? v.sequence : v; @@ -318,7 +321,7 @@ export function encodeNomsValue(v: valueOrPrimitive, t: Type, ds: ?DataStore): C return encodeTopLevelBlob(sequence); } } - return encodeEmbeddedNomsValue(v, t, ds); + return encodeEmbeddedNomsValue(v, t, vw); } setEncodeNomsValue(encodeNomsValue); diff --git a/js/src/http-batch-store-test.js b/js/src/http-batch-store-test.js new file mode 100644 index 0000000000..1e32a7f543 --- /dev/null +++ b/js/src/http-batch-store-test.js @@ -0,0 +1,24 @@ +// @flow + +import {suite, test} from 'mocha'; +import Chunk from './chunk.js'; +import {assert} from 'chai'; +import {Delegate} from './http-batch-store.js'; +import {deserialize} from './chunk-serializer.js'; + +suite('HttpBatchStore', () => { + test('build write request', async () => { + const canned = new Set([Chunk.fromString('abc').ref, Chunk.fromString('def').ref]); + const reqs = [ + {c: Chunk.fromString('ghi'), hints: new Set()}, + {c: Chunk.fromString('wacka wack wack'), hints: canned}, + ]; + const d = new Delegate( + {getRefs: '', writeValue: '', root: ''}, + {method: 'POST', headers: {}}); + const body = d._buildWriteRequest(reqs); + const {hints, chunks} = deserialize(body); + assert.equal(2, chunks.length); + assert.equal(2, hints.length); + }); +}); \ No newline at end of file diff --git a/js/src/http-store.js b/js/src/http-batch-store.js similarity index 63% rename from js/src/http-store.js rename to js/src/http-batch-store.js index 89a0a53c08..d513c52526 100644 --- a/js/src/http-store.js +++ b/js/src/http-batch-store.js @@ -1,20 +1,19 @@ // @flow import Ref from './ref.js'; +import BatchStore from './batch-store.js'; +import type {UnsentReadMap, WriteRequest} from './batch-store.js'; import type {FetchOptions} from './fetch.js'; -import type {UnsentReadMap} from './remote-store.js'; -import type {WriteMap} from './remote-store.js'; -import {deserialize, serialize} from './chunk-serializer.js'; +import {deserializeChunks, serialize} from './chunk-serializer.js'; import {emptyChunk} from './chunk.js'; import {fetchArrayBuffer, fetchText} from './fetch.js'; -import {RemoteStore} from './remote-store.js'; const HTTP_STATUS_CONFLICT = 409; type RpcStrings = { getRefs: string, - postRefs: string, root: string, + writeValue: string, }; const readBatchOptions = { @@ -24,31 +23,18 @@ const readBatchOptions = { }, }; -export default class HttpStore extends RemoteStore { +export default class HttpBatchStore extends BatchStore { _rpc: RpcStrings; - _rootOptions: FetchOptions; - constructor(url: string, maxReads: number = 3, maxWrites: number = 3, - fetchOptions: FetchOptions = {}) { + constructor(url: string, maxReads: number = 5, fetchOptions: FetchOptions = {}) { const rpc = { getRefs: url + '/getRefs/', - postRefs: url + '/postRefs/', root: url + '/root/', + writeValue: url + '/writeValue/', }; - const mergedOptions = mergeOptions(readBatchOptions, fetchOptions); - super(maxReads, maxWrites, new Delegate(rpc, mergedOptions)); + super(maxReads, new Delegate(rpc, fetchOptions)); this._rpc = rpc; - this._rootOptions = fetchOptions; - } - - async getRoot(): Promise { - const refStr = await fetchText(this._rpc.root, this._rootOptions); - return Ref.parse(refStr); - } - - has(ref: Ref): Promise { // eslint-disable-line no-unused-vars - throw new Error('not implemented'); } } @@ -57,13 +43,15 @@ function mergeOptions(baseOpts: FetchOptions, opts: FetchOptions): FetchOptions return Object.assign({}, opts, baseOpts, {headers: hdrs}); } -class Delegate { +export class Delegate { _rpc: RpcStrings; _readBatchOptions: FetchOptions; + _rootOptions: FetchOptions; - constructor(rpc: RpcStrings, readBatchOptions: FetchOptions) { + constructor(rpc: RpcStrings, fetchOptions: FetchOptions) { this._rpc = rpc; - this._readBatchOptions = readBatchOptions; + this._rootOptions = fetchOptions; + this._readBatchOptions = mergeOptions(readBatchOptions, fetchOptions); } async readBatch(reqs: UnsentReadMap): Promise { @@ -72,7 +60,7 @@ class Delegate { const opts = Object.assign(this._readBatchOptions, {body: body}); const buf = await fetchArrayBuffer(this._rpc.getRefs, opts); - const chunks = deserialize(buf); + const chunks = deserializeChunks(buf); // Return success chunks.forEach(chunk => { @@ -85,10 +73,23 @@ class Delegate { Object.keys(reqs).forEach(refStr => reqs[refStr](emptyChunk)); } - async writeBatch(reqs: WriteMap): Promise { - const chunks = Object.keys(reqs).map(refStr => reqs[refStr]); - const body = serialize(chunks); - await fetchText(this._rpc.postRefs, {method: 'POST', body}); + async writeBatch(reqs: Array): Promise { + const body = this._buildWriteRequest(reqs); + await fetchText(this._rpc.writeValue, {method: 'POST', body}); + } + + _buildWriteRequest(reqs: Array): ArrayBuffer { + let hints = new Set(); + const chunks = reqs.map(writeReq => { + writeReq.hints.forEach(hint => { hints = hints.add(hint); }); + return writeReq.c; + }); + return serialize(hints, chunks); + } + + async getRoot(): Promise { + const refStr = await fetchText(this._rpc.root, this._rootOptions); + return Ref.parse(refStr); } async updateRoot(current: Ref, last: Ref): Promise { diff --git a/js/src/list-test.js b/js/src/list-test.js index 675b850c1f..1ea5faae14 100644 --- a/js/src/list-test.js +++ b/js/src/list-test.js @@ -4,7 +4,7 @@ import {assert} from 'chai'; import {suite, test} from 'mocha'; import DataStore from './data-store.js'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import RefValue from './ref-value.js'; import {newStruct} from './struct.js'; import {calcSplices} from './edit-distance.js'; @@ -152,8 +152,7 @@ suite('BuildList', () => { }); test('LONG: write, read, modify, read', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const nums = firstNNumbers(testListSize); const tr = makeListType(numberType); @@ -173,8 +172,7 @@ suite('BuildList', () => { suite('ListLeafSequence', () => { test('isEmpty', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(stringType); const newList = items => new NomsList(tr, new ListLeafSequence(ds, tr, items)); assert.isTrue(newList([]).isEmpty()); @@ -182,8 +180,7 @@ suite('ListLeafSequence', () => { }); test('get', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(stringType); const l = new NomsList(tr, new ListLeafSequence(ds, tr, ['z', 'x', 'a', 'b'])); assert.strictEqual('z', await l.get(0)); @@ -193,8 +190,7 @@ suite('ListLeafSequence', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(numberType); const l = new NomsList(tr, new ListLeafSequence(ds, tr, [4, 2, 10, 16])); @@ -204,8 +200,7 @@ suite('ListLeafSequence', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(numberType); const test = async items => { @@ -220,8 +215,7 @@ suite('ListLeafSequence', () => { }); test('iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(numberType); const test = async items => { @@ -239,8 +233,7 @@ suite('ListLeafSequence', () => { }); function testChunks(elemType: Type) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(elemType); const r1 = ds.writeValue('x'); const r2 = ds.writeValue('a'); @@ -263,8 +256,7 @@ suite('ListLeafSequence', () => { suite('CompoundList', () => { function build(): NomsList { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(stringType); const l1 = new NomsList(tr, new ListLeafSequence(ds, tr, ['a', 'b'])); const r1 = ds.writeValue(l1).targetRef; diff --git a/js/src/list.js b/js/src/list.js index e0334064ab..e0a054f9c5 100644 --- a/js/src/list.js +++ b/js/src/list.js @@ -2,7 +2,7 @@ import BuzHashBoundaryChecker from './buzhash-boundary-checker.js'; import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import type {Splice} from './edit-distance.js'; import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars import type {AsyncIterator} from './async-iterator.js'; @@ -22,9 +22,9 @@ import {listOfValueType, Type} from './type.js'; const listWindowSize = 64; const listPattern = ((1 << 6) | 0) - 1; -function newListLeafChunkFn(t: Type, ds: ?DataStore = null): makeChunkFn { +function newListLeafChunkFn(t: Type, vr: ?ValueReader = null): makeChunkFn { return (items: Array) => { - const listLeaf = new ListLeafSequence(ds, t, items); + const listLeaf = new ListLeafSequence(vr, t, items); const mt = new MetaTuple(listLeaf, items.length, items.length); return [mt, listLeaf]; }; @@ -54,10 +54,10 @@ export class NomsList extends Collection { async splice(idx: number, deleteCount: number, ...insert: Array): Promise> { const cursor = await this.sequence.newCursorAt(idx); - const ds = this.sequence.ds; + const vr = this.sequence.vr; const type = this.type; - const seq = await chunkSequence(cursor, insert, deleteCount, newListLeafChunkFn(type, ds), - newIndexedMetaSequenceChunkFn(type, ds), + const seq = await chunkSequence(cursor, insert, deleteCount, newListLeafChunkFn(type, vr), + newIndexedMetaSequenceChunkFn(type, vr), newListLeafBoundaryChecker(type), newIndexedMetaSequenceBoundaryChecker); invariant(seq instanceof IndexedSequence); diff --git a/js/src/map-test.js b/js/src/map-test.js index 9c8625b165..a8bfd01705 100644 --- a/js/src/map-test.js +++ b/js/src/map-test.js @@ -6,6 +6,8 @@ import {suite, test} from 'mocha'; import DataStore from './data-store'; import MemoryStore from './memory-store.js'; import RefValue from './ref-value.js'; +import BatchStore from './batch-store.js'; +import {BatchStoreAdaptorDelegate, makeTestingBatchStore} from './batch-store-adaptor.js'; import {newStruct} from './struct.js'; import { boolType, @@ -164,8 +166,7 @@ suite('BuildMap', () => { }); test('LONG: write, read, modify, read', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const kvs = []; for (let i = 0; i < testMapSize; i++) { @@ -194,8 +195,7 @@ suite('BuildMap', () => { suite('MapLeaf', () => { test('isEmpty/size', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, boolType); const newMap = entries => new NomsMap(tr, new MapLeafSequence(ds, tr, entries)); let m = newMap([]); @@ -207,8 +207,7 @@ suite('MapLeaf', () => { }); test('has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, boolType); const m = new NomsMap(tr, new MapLeafSequence(ds, tr, [{key: 'a', value: false}, {key:'k', value:true}])); @@ -219,8 +218,7 @@ suite('MapLeaf', () => { }); test('first/last/get', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, numberType); const m = new NomsMap(tr, new MapLeafSequence(ds, tr, [{key: 'a', value: 4}, {key:'k', value:8}])); @@ -235,8 +233,7 @@ suite('MapLeaf', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, numberType); const m = new NomsMap(tr, new MapLeafSequence(ds, tr, [{key: 'a', value: 4}, {key:'k', value:8}])); @@ -247,8 +244,7 @@ suite('MapLeaf', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, numberType); const test = async entries => { @@ -263,8 +259,7 @@ suite('MapLeaf', () => { }); test('LONG: iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, numberType); const build = entries => new NomsMap(tr, new MapLeafSequence(ds, tr, entries)); @@ -288,8 +283,7 @@ suite('MapLeaf', () => { }); function testChunks(keyType: Type, valueType: Type) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(keyType, valueType); const r1 = ds.writeValue('x'); const r2 = ds.writeValue(true); @@ -343,16 +337,14 @@ suite('CompoundMap', () => { } test('isEmpty/size', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); assert.isFalse(c.isEmpty()); assert.strictEqual(8, c.size); }); test('get', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); assert.strictEqual(false, await c.get('a')); @@ -372,8 +364,7 @@ suite('CompoundMap', () => { }); test('first/last/has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c, m1, m2] = build(ds); assert.deepEqual(['a', false], await c.first()); @@ -400,8 +391,7 @@ suite('CompoundMap', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const kv = []; @@ -411,8 +401,7 @@ suite('CompoundMap', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const expected = [{key: 'a', value: false}, {key: 'b', value: false}, {key: 'e', value: true}, {key: 'f', value: true}, {key: 'h', value: false}, {key: 'i', value: true}, @@ -422,8 +411,7 @@ suite('CompoundMap', () => { }); test('LONG: iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const entries = [{key: 'a', value: false}, {key: 'b', value: false}, {key: 'e', value: true}, {key: 'f', value: true}, {key: 'h', value: false}, {key: 'i', value: true}, @@ -447,8 +435,7 @@ suite('CompoundMap', () => { }); test('iterator return', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const iter = c.iterator(); const values = []; @@ -464,8 +451,7 @@ suite('CompoundMap', () => { }); test('iterator return parallel', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const iter = c.iterator(); const values = await Promise.all([iter.next(), iter.next(), iter.return(), iter.next()]); @@ -476,8 +462,7 @@ suite('CompoundMap', () => { }); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); assert.strictEqual(2, c.chunks.length); }); @@ -515,7 +500,7 @@ suite('CompoundMap', () => { } const ms = new CountingMemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(new BatchStore(3, new BatchStoreAdaptorDelegate(ms))); [m1, m2] = await Promise.all([m1, m2].map(s => ds.readValue(ds.writeValue(s).targetRef))); assert.deepEqual([[], [], []], await m1.diff(m1)); @@ -565,8 +550,7 @@ suite('CompoundMap', () => { test('LONG: random map diff 0.1/0.9/0', () => testRandomDiff(randomMapSize, 0.1, 0.9, 0)); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const m = build(ds)[1]; const chunks = m.chunks; const sequence = m.sequence; diff --git a/js/src/map.js b/js/src/map.js index 75004b486d..b0cfd3ccba 100644 --- a/js/src/map.js +++ b/js/src/map.js @@ -2,7 +2,7 @@ import BuzHashBoundaryChecker from './buzhash-boundary-checker.js'; import RefValue from './ref-value.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js'; import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars import type {AsyncIterator} from './async-iterator.js'; @@ -28,9 +28,9 @@ export type MapEntry = { const mapWindowSize = 1; const mapPattern = ((1 << 6) | 0) - 1; -function newMapLeafChunkFn(t: Type, ds: ?DataStore = null): makeChunkFn { +function newMapLeafChunkFn(t: Type, vr: ?ValueReader = null): makeChunkFn { return (items: Array) => { - const mapLeaf = new MapLeafSequence(ds, t, items); + const mapLeaf = new MapLeafSequence(vr, t, items); let indexValue: ?valueOrPrimitive = null; if (items.length > 0) { @@ -150,9 +150,9 @@ export class NomsMap extends Collectio async _splice(cursor: OrderedSequenceCursor, insert: Array, remove: number): Promise> { const type = this.type; - const ds = this.sequence.ds; - const seq = await chunkSequence(cursor, insert, remove, newMapLeafChunkFn(type, ds), - newOrderedMetaSequenceChunkFn(type, ds), + const vr = this.sequence.vr; + const seq = await chunkSequence(cursor, insert, remove, newMapLeafChunkFn(type, vr), + newOrderedMetaSequenceChunkFn(type, vr), newMapLeafBoundaryChecker(type), newOrderedMetaSequenceBoundaryChecker); invariant(seq instanceof OrderedSequence); diff --git a/js/src/meta-sequence.js b/js/src/meta-sequence.js index f86b44b1c1..0951f86db1 100644 --- a/js/src/meta-sequence.js +++ b/js/src/meta-sequence.js @@ -3,7 +3,7 @@ import BuzHashBoundaryChecker from './buzhash-boundary-checker.js'; import {default as Ref, sha1Size} from './ref.js'; import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars import type {Collection} from './collection.js'; import {CompoundDesc, makeCompoundType, makeRefType, numberType, valueType} from './type.js'; @@ -44,13 +44,13 @@ export class MetaTuple { return this._sequenceOrRef instanceof Sequence ? this._sequenceOrRef : null; } - getSequence(ds: ?DataStore): Promise { + getSequence(vr: ?ValueReader): Promise { if (this._sequenceOrRef instanceof Sequence) { return Promise.resolve(this._sequenceOrRef); } else { const ref = this._sequenceOrRef; - invariant(ds && ref instanceof Ref); - return ds.readValue(ref).then((c: Collection) => c.sequence); + invariant(vr && ref instanceof Ref); + return vr.readValue(ref).then((c: Collection) => c.sequence); } } } @@ -58,8 +58,8 @@ export class MetaTuple { export class IndexedMetaSequence extends IndexedSequence> { _offsets: Array; - constructor(ds: ?DataStore, type: Type, items: Array>) { - super(ds, type, items); + constructor(vr: ?ValueReader, type: Type, items: Array>) { + super(vr, type, items); let cum = 0; this._offsets = this.items.map(i => { cum += i.value; @@ -111,7 +111,7 @@ export class IndexedMetaSequence extends IndexedSequence> { } const mt = this.items[idx]; - return mt.getSequence(this.ds); + return mt.getSequence(this.vr); } // Returns the sequences pointed to by all items[i], s.t. start <= i < end, and returns the @@ -120,14 +120,14 @@ export class IndexedMetaSequence extends IndexedSequence> { Promise { const childrenP = []; for (let i = start; i < start + length; i++) { - childrenP.push(this.items[i].getSequence(this.ds)); + childrenP.push(this.items[i].getSequence(this.vr)); } return Promise.all(childrenP).then(children => { const items = []; children.forEach(child => items.push(...child.items)); - return children[0].isMeta ? new IndexedMetaSequence(this.ds, this.type, items) - : new IndexedSequence(this.ds, this.type, items); + return children[0].isMeta ? new IndexedMetaSequence(this.vr, this.type, items) + : new IndexedSequence(this.vr, this.type, items); }); } @@ -139,8 +139,8 @@ export class IndexedMetaSequence extends IndexedSequence> { export class OrderedMetaSequence extends OrderedSequence> { _numLeaves: number; - constructor(ds: ?DataStore, type: Type, items: Array>) { - super(ds, type, items); + constructor(vr: ?ValueReader, type: Type, items: Array>) { + super(vr, type, items); this._numLeaves = items.reduce((l, mt) => l + mt.numLeaves, 0); } @@ -162,7 +162,7 @@ export class OrderedMetaSequence extends OrderedSequence extends OrderedSequence): +export function newMetaSequenceFromData(vr: ValueReader, type: Type, tuples: Array): MetaSequence { switch (type.kind) { case Kind.Map: case Kind.Set: - return new OrderedMetaSequence(ds, type, tuples); + return new OrderedMetaSequence(vr, type, tuples); case Kind.Blob: case Kind.List: - return new IndexedMetaSequence(ds, type, tuples); + return new IndexedMetaSequence(vr, type, tuples); default: throw new Error('Not reached'); } @@ -211,10 +211,10 @@ export function indexTypeForMetaSequence(t: Type): Type { throw new Error('Not reached'); } -export function newOrderedMetaSequenceChunkFn(t: Type, ds: ?DataStore = null): makeChunkFn { +export function newOrderedMetaSequenceChunkFn(t: Type, vr: ?ValueReader = null): makeChunkFn { return (tuples: Array) => { const numLeaves = tuples.reduce((l, mt) => l + mt.numLeaves, 0); - const meta = new OrderedMetaSequence(ds, t, tuples); + const meta = new OrderedMetaSequence(vr, t, tuples); const lastValue = tuples[tuples.length - 1].value; return [new MetaTuple(meta, lastValue, numLeaves), meta]; }; @@ -230,13 +230,13 @@ export function newOrderedMetaSequenceBoundaryChecker(): BoundaryChecker) => { const sum = tuples.reduce((l, mt) => { invariant(mt.value === mt.numLeaves); return l + mt.value; }, 0); - const meta = new IndexedMetaSequence(ds, t, tuples); + const meta = new IndexedMetaSequence(vr, t, tuples); return [new MetaTuple(meta, sum, sum), meta]; }; } diff --git a/js/src/noms.js b/js/src/noms.js index 7d22c2b860..99b1ae7df7 100644 --- a/js/src/noms.js +++ b/js/src/noms.js @@ -6,7 +6,7 @@ export {default as Dataset} from './dataset.js'; export {newBlob, NomsBlob, BlobReader, BlobWriter} from './blob.js'; export {decodeNomsValue} from './decode.js'; export {default as Chunk} from './chunk.js'; -export {default as HttpStore} from './http-store.js'; +export {default as HttpBatchStore} from './http-batch-store.js'; export {default as MemoryStore} from './memory-store.js'; export {default as Ref, emptyRef} from './ref.js'; export {default as RefValue} from './ref-value.js'; diff --git a/js/src/sequence.js b/js/src/sequence.js index 4fdcb54ff5..3d20c9d568 100644 --- a/js/src/sequence.js +++ b/js/src/sequence.js @@ -1,6 +1,6 @@ // @flow -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import {invariant, notNull} from './assert.js'; import {AsyncIterator} from './async-iterator.js'; import type {AsyncIteratorResult} from './async-iterator.js'; @@ -9,13 +9,13 @@ import type {Type} from './type.js'; import {Value} from './value.js'; export class Sequence extends Value { - ds: ?DataStore; + vr: ?ValueReader; _type: Type; _items: Array; - constructor(ds: ?DataStore, type: Type, items: Array) { + constructor(vr: ?ValueReader, type: Type, items: Array) { super(); - this.ds = ds; + this.vr = vr; this._type = type; this._items = items; } diff --git a/js/src/set-test.js b/js/src/set-test.js index 0b1d0ad0ab..70d40f80d8 100644 --- a/js/src/set-test.js +++ b/js/src/set-test.js @@ -7,6 +7,8 @@ import Chunk from './chunk.js'; import DataStore from './data-store.js'; import MemoryStore from './memory-store.js'; import RefValue from './ref-value.js'; +import BatchStore from './batch-store.js'; +import {BatchStoreAdaptorDelegate, makeTestingBatchStore} from './batch-store-adaptor.js'; import {newStruct} from './struct.js'; import { boolType, @@ -134,8 +136,7 @@ suite('BuildSet', () => { }); test('LONG: write, read, modify, read', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const nums = firstNNumbers(testSetSize); const tr = makeSetType(numberType); @@ -159,8 +160,7 @@ suite('BuildSet', () => { suite('SetLeaf', () => { test('isEmpty/size', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const newSet = items => new NomsSet(tr, new SetLeafSequence(ds, tr, items)); let s = newSet([]); @@ -172,8 +172,7 @@ suite('SetLeaf', () => { }); test('first/last/has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const s = new NomsSet(tr, new SetLeafSequence(ds, tr, ['a', 'k'])); @@ -187,8 +186,7 @@ suite('SetLeaf', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const m = new NomsSet(tr, new SetLeafSequence(ds, tr, ['a', 'b'])); @@ -198,8 +196,7 @@ suite('SetLeaf', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const test = async items => { @@ -214,8 +211,7 @@ suite('SetLeaf', () => { }); test('LONG: iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const build = items => new NomsSet(tr, new SetLeafSequence(ds, tr, items)); @@ -233,8 +229,7 @@ suite('SetLeaf', () => { }); function testChunks(elemType: Type) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(elemType); const r1 = ds.writeValue('x'); const r2 = ds.writeValue('a'); @@ -284,16 +279,14 @@ suite('CompoundSet', () => { } test('isEmpty/size', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); assert.isFalse(c.isEmpty()); assert.strictEqual(8, c.size); }); test('first/last/has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); assert.strictEqual('a', await c.first()); assert.strictEqual('n', await c.last()); @@ -314,8 +307,7 @@ suite('CompoundSet', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); const values = []; await c.forEach((k) => { values.push(k); }); @@ -323,8 +315,7 @@ suite('CompoundSet', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const values = ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']; const c = build(ds, values); assert.deepEqual(values, await flatten(c.iterator())); @@ -332,8 +323,7 @@ suite('CompoundSet', () => { }); test('LONG: iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const values = ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']; const c = build(ds, values); const offsets = { @@ -355,8 +345,7 @@ suite('CompoundSet', () => { }); test('iterator return', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const values = ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']; const c = build(ds, values); const iter = c.iterator(); @@ -371,8 +360,7 @@ suite('CompoundSet', () => { }); test('iterator return parallel', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); const iter = c.iterator(); const values = await Promise.all([iter.next(), iter.next(), iter.return(), iter.next()]); @@ -382,23 +370,20 @@ suite('CompoundSet', () => { }); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); assert.strictEqual(2, c.chunks.length); }); test('map', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); const values = await c.map((k) => k + '*'); assert.deepEqual(['a*', 'b*', 'e*', 'f*', 'h*', 'i*', 'm*', 'n*'], values); }); test('map async', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); const values = await c.map((k) => Promise.resolve(k + '*')); assert.deepEqual(['a*', 'b*', 'e*', 'f*', 'h*', 'i*', 'm*', 'n*'], values); @@ -416,8 +401,7 @@ suite('CompoundSet', () => { } test('advanceTo', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); @@ -459,8 +443,7 @@ suite('CompoundSet', () => { }); async function testIntersect(expect: Array, seqs: Array>) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const first = build(ds, seqs[0]); const sets:Array = []; @@ -489,8 +472,7 @@ suite('CompoundSet', () => { }); test('iterator at 0', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(numberType); const test = async (expected, items) => { @@ -513,8 +495,7 @@ suite('CompoundSet', () => { }); test('LONG: canned set diff', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(numberType); const s1 = await newSet( firstNNumbers(testSetSize), tr).then(s => ds.readValue(ds.writeValue(s).targetRef)); @@ -568,7 +549,7 @@ suite('CompoundSet', () => { } const ms = new CountingMemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(new BatchStore(3, new BatchStoreAdaptorDelegate(ms))); [s1, s2] = await Promise.all([s1, s2].map(s => ds.readValue(ds.writeValue(s).targetRef))); assert.deepEqual([[], []], await s1.diff(s1)); @@ -607,8 +588,7 @@ suite('CompoundSet', () => { test('LONG: random set diff 0.1/0.9', () => testRandomDiff(randomSetSize, 0.1, 0.9)); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const s = build(ds, ['a', 'b', 'c', 'd']); const chunks = s.chunks; const sequence = s.sequence; diff --git a/js/src/set.js b/js/src/set.js index 47d3284b1c..ec9aebc73c 100644 --- a/js/src/set.js +++ b/js/src/set.js @@ -2,7 +2,7 @@ import BuzHashBoundaryChecker from './buzhash-boundary-checker.js'; import RefValue from './ref-value.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js'; import type {valueOrPrimitive, Value} from './value.js'; // eslint-disable-line no-unused-vars import {AsyncIterator} from './async-iterator.js'; @@ -24,9 +24,9 @@ import {getValueChunks} from './sequence.js'; const setWindowSize = 1; const setPattern = ((1 << 6) | 0) - 1; -function newSetLeafChunkFn(t: Type, ds: ?DataStore = null): makeChunkFn { +function newSetLeafChunkFn(t: Type, vr: ?ValueReader = null): makeChunkFn { return (items: Array) => { - const setLeaf = new SetLeafSequence(ds, t, items); + const setLeaf = new SetLeafSequence(vr, t, items); let indexValue: ?(T | RefValue) = null; if (items.length > 0) { @@ -105,9 +105,9 @@ export class NomsSet extends Collection { async _splice(cursor: OrderedSequenceCursor, insert: Array, remove: number): Promise> { const type = this.type; - const ds = this.sequence.ds; - const seq = await chunkSequence(cursor, insert, remove, newSetLeafChunkFn(type, ds), - newOrderedMetaSequenceChunkFn(type, ds), + const vr = this.sequence.vr; + const seq = await chunkSequence(cursor, insert, remove, newSetLeafChunkFn(type, vr), + newOrderedMetaSequenceChunkFn(type, vr), newSetLeafBoundaryChecker(type), newOrderedMetaSequenceBoundaryChecker); invariant(seq instanceof OrderedSequence); diff --git a/js/src/specs-test.js b/js/src/specs-test.js index 95dffef8ce..7bf3b6daea 100644 --- a/js/src/specs-test.js +++ b/js/src/specs-test.js @@ -1,10 +1,10 @@ // @flow import {invariant} from './assert.js'; +import BatchStoreAdaptor from './batch-store-adaptor.js'; import Dataset from './dataset.js'; import DataStore from './data-store.js'; -import HttpStore from './http-store.js'; -import MemoryStore from './memory-store.js'; +import HttpBatchStore from './http-batch-store.js'; import Ref from './ref.js'; import {DataStoreSpec, DatasetSpec, RefSpec, parseObjectSpec} from './specs.js'; import {assert} from 'chai'; @@ -20,7 +20,7 @@ suite('Specs', () => { assert.equal(spec.scheme, 'mem'); assert.equal(spec.path, ''); assert.instanceOf(spec.store(), DataStore); - assert.instanceOf(spec.store()._cs, MemoryStore); + assert.instanceOf(spec.store()._bs, BatchStoreAdaptor); spec = DataStoreSpec.parse('http://foo'); invariant(spec); @@ -28,7 +28,7 @@ suite('Specs', () => { assert.equal(spec.scheme, 'http'); assert.equal(spec.path, '//foo'); assert.instanceOf(spec.store(), DataStore); - assert.instanceOf(spec.store()._cs, HttpStore); + assert.instanceOf(spec.store()._bs, HttpBatchStore); spec = DataStoreSpec.parse('https://foo'); invariant(spec); @@ -48,7 +48,7 @@ suite('Specs', () => { assert.equal(spec.store.path, ''); let ds = spec.set(); assert.instanceOf(ds, Dataset); - assert.instanceOf(ds.store._cs, MemoryStore); + assert.instanceOf(ds.store._bs, BatchStoreAdaptor); spec = DatasetSpec.parse('http://localhost:8000/foo:ds'); invariant(spec); @@ -57,7 +57,7 @@ suite('Specs', () => { assert.equal(spec.store.path, '//localhost:8000/foo'); ds = spec.set(); assert.instanceOf(ds, Dataset); - assert.instanceOf(ds.store._cs, HttpStore); + assert.instanceOf(ds.store._bs, HttpBatchStore); }); test('RefSpec', () => { diff --git a/js/src/specs.js b/js/src/specs.js index 9a1526a917..405c23daeb 100644 --- a/js/src/specs.js +++ b/js/src/specs.js @@ -1,8 +1,9 @@ // @flow +import BatchStoreAdaptor from './batch-store-adaptor.js'; import Dataset from './dataset.js'; import DataStore from './data-store.js'; -import HttpStore from './http-store.js'; +import HttpBatchStore from './http-batch-store.js'; import MemoryStore from './memory-store.js'; import Ref from './ref.js'; @@ -48,10 +49,10 @@ export class DataStoreSpec { // Constructs a new DataStore based on the parsed spec. store(): DataStore { if (this.scheme === 'mem') { - return new DataStore(new MemoryStore()); + return new DataStore(new BatchStoreAdaptor(new MemoryStore())); } if (this.scheme === 'http') { - return new DataStore(new HttpStore(`${this.scheme}:${this.path}`)); + return new DataStore(new HttpBatchStore(`${this.scheme}:${this.path}`)); } throw new Error('Unreached'); } diff --git a/js/src/struct-test.js b/js/src/struct-test.js index c578109096..55005f379b 100644 --- a/js/src/struct-test.js +++ b/js/src/struct-test.js @@ -1,6 +1,6 @@ // @flow -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {default as Struct, newStruct, StructMirror, createStructClass} from './struct.js'; import {assert} from 'chai'; import { @@ -32,8 +32,7 @@ suite('Struct', () => { }); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const bt = boolType; const refOfBoolType = makeRefType(bt); diff --git a/js/src/type-test.js b/js/src/type-test.js index 4a6230d332..bad45b4cf5 100644 --- a/js/src/type-test.js +++ b/js/src/type-test.js @@ -1,6 +1,6 @@ // @flow -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {assert} from 'chai'; import { boolType, @@ -18,8 +18,7 @@ import DataStore from './data-store.js'; suite('Type', () => { test('types', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const mapType = makeMapType(stringType, numberType); const setType = makeSetType(stringType); diff --git a/js/src/value-store-test.js b/js/src/value-store-test.js new file mode 100644 index 0000000000..3570dbacb4 --- /dev/null +++ b/js/src/value-store-test.js @@ -0,0 +1,124 @@ +// @flow + +import {suite, test} from 'mocha'; +import {assert} from 'chai'; +import MemoryStore from './memory-store.js'; +import type {ChunkStore} from './chunk-store.js'; +import BatchStore from './batch-store.js'; +import {BatchStoreAdaptorDelegate} from './batch-store-adaptor.js'; +import ValueStore from './value-store.js'; +import {newList} from './list.js'; +import {stringType} from './type.js'; +import {encodeNomsValue} from './encode.js'; + +export class FakeBatchStore extends BatchStore { + constructor(cs: ChunkStore) { + super(3, new BatchStoreAdaptorDelegate(cs)); + } +} + +suite('ValueStore', () => { + test('readValue', async () => { + const ms = new MemoryStore(); + const vs = new ValueStore(new FakeBatchStore(ms)); + const input = 'abc'; + + const c = encodeNomsValue(input, stringType); + const v1 = await vs.readValue(c.ref); + assert.equal(null, v1); + + ms.put(c); + const v2 = await vs.readValue(c.ref); + assert.equal('abc', v2); + }); + + test('writeValue primitives', async () => { + const vs = new ValueStore(new FakeBatchStore(new MemoryStore())); + + const r1 = vs.writeValue('hello').targetRef; + const r2 = vs.writeValue(false).targetRef; + const r3 = vs.writeValue(2).targetRef; + + const v1 = await vs.readValue(r1); + assert.equal('hello', v1); + const v2 = await vs.readValue(r2); + assert.equal(false, v2); + const v3 = await vs.readValue(r3); + assert.equal(2, v3); + }); + + test('writeValue rejects invalid', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + let vs = new ValueStore(bs); + const r = vs.writeValue('hello'); + vs.flush().then(() => { vs.close(); }); + + vs = new ValueStore(bs); + const l = await newList([r]); + let ex; + try { + await vs.writeValue(l); + } catch (e) { + ex = e; + } + assert.instanceOf(ex, Error); + }); + + test('write coalescing', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + const vs = new ValueStore(bs, 1e6); + + const r1 = vs.writeValue('hello').targetRef; + (bs: any).schedulePut = () => { assert.fail('unreachable'); }; + const r2 = vs.writeValue('hello').targetRef; + assert.isTrue(r1.equals(r2)); + }); + + test('read caching', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + const vs = new ValueStore(bs, 1e6); + + const r1 = vs.writeValue('hello').targetRef; + const v1 = await vs.readValue(r1); + assert.equal(v1, 'hello'); + (bs: any).get = () => { throw new Error(); }; + const v2 = await vs.readValue(r1); + assert.equal(v1, v2); + }); + + test('caching eviction', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + const vs = new ValueStore(bs, 15); + + const r1 = vs.writeValue('hello').targetRef; + const r2 = vs.writeValue('world').targetRef; + + // Prime the cache + const v1 = await vs.readValue(r1); + assert.equal(v1, 'hello'); + // Evict v1 from the cache + const v2 = await vs.readValue(r2); + assert.equal(v2, 'world'); + + (bs: any).get = () => { throw new Error(); }; + let ex; + try { + await vs.readValue(r1); + } catch (e) { + ex = e; + } + assert.instanceOf(ex, Error); + }); + + test('hints on cache', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + const vs = new ValueStore(bs, 15); + + const l = await newList([vs.writeValue(1), vs.writeValue(2)]); + const r = vs.writeValue(l); + // await vs.flush(); + + const v = await vs.readValue(r.targetRef); + assert.isTrue(l.equals(v)); + }); +}); diff --git a/js/src/value-store.js b/js/src/value-store.js new file mode 100644 index 0000000000..87e1824157 --- /dev/null +++ b/js/src/value-store.js @@ -0,0 +1,248 @@ +// @flow + +import Chunk from './chunk.js'; +import {default as Ref, emptyRef} from './ref.js'; +import {default as RefValue, refValueFromValue} from './ref-value.js'; +import type BatchStore from './batch-store.js'; +import type {valueOrPrimitive} from './value.js'; +import { + getTypeOfValue, + Type, + valueType, +} from './type.js'; +import {Kind} from './noms-kind.js'; +import {Value} from './value.js'; +import {decodeNomsValue} from './decode.js'; +import {invariant, notNull} from './assert.js'; +import {encodeNomsValue} from './encode.js'; +import {describeType, describeTypeOfValue} from './encode-human-readable.js'; + +export interface Cache { // eslint-disable-line no-undef + entry(ref: Ref): ?CacheEntry; // eslint-disable-line no-undef + get(ref: Ref): ?T; // eslint-disable-line no-undef + add(ref: Ref, size: number, value: T): void; // eslint-disable-line no-undef +} + +export default class ValueStore { + _bs: BatchStore; + _knownRefs: RefCache; + _valueCache: Cache; + + constructor(bs: BatchStore, cacheSize: number = 0) { + this._bs = bs; + this._knownRefs = new RefCache(); + this._valueCache = cacheSize > 0 ? new SizeCache(cacheSize) : new NoopCache(); + } + + // TODO: This should return Promise + async readValue(ref: Ref): Promise { + const entry = this._valueCache.entry(ref); + if (entry) { + return entry.value; + } + const chunk: Chunk = await this._bs.get(ref); + if (chunk.isEmpty()) { + this._valueCache.add(ref, 0, null); + this._knownRefs.addIfNotPresent(ref, new RefCacheEntry(false)); + return null; + } + + const v = decodeNomsValue(chunk, this); + this._valueCache.add(ref, chunk.data.length, v); + this._knownRefs.cacheChunks(v, ref); + // ref is trivially a hint for v, so consider putting that in the cache. + // If we got to v by reading some higher-level chunk, this entry gets dropped on + // the floor because r already has a hint in the cache. If we later read some other + // chunk that references v, cacheChunks will overwrite this with a hint pointing to that chunk. + // If we don't do this, top-level Values that get read but not written -- such as the + // existing Head of a DataStore upon a Commit -- can be erroneously left out during a pull. + this._knownRefs.addIfNotPresent(ref, new RefCacheEntry(true, getTypeOfValue(v), ref)); + return v; + } + + writeValue(v: T): RefValue { + const t = getTypeOfValue(v); + const chunk = encodeNomsValue(v, t, this); + invariant(!chunk.isEmpty()); + const {ref} = chunk; + const refValue = refValueFromValue(v); + const entry = this._knownRefs.get(ref); + if (entry && entry.present) { + return refValue; + } + const hints = this._knownRefs.checkChunksInCache(v); + this._bs.schedulePut(chunk, hints); + this._knownRefs.add(ref, new RefCacheEntry(true, t)); + return refValue; + } + + async flush(): Promise { + return this._bs.flush(); + } + + close() { + this._bs.close(); + } +} + +export class CacheEntry { + size: number; + value: ?T; + + constructor(size: number, value: ?T) { + this.size = size; + this.value = value; + } + + get present(): boolean { + return this.value !== null; + } +} + +/** + * This uses a Map as an LRU cache. It uses the behavior that iteration of keys in a Map is done in + * insertion order and any time a value is checked it is taken out and reinserted which puts it last + * in the iteration. + */ +export class SizeCache { + _size: number; + _maxSize: number; + _cache: Map>; + + constructor(size: number) { + this._maxSize = size; + this._cache = new Map(); + this._size = 0; + } + + entry(ref: Ref): ?CacheEntry { + const key = ref.toString(); + const entry = this._cache.get(key); + if (!entry) { + return undefined; + } + this._cache.delete(key); + this._cache.set(key, entry); + return entry; + } + + get(ref: Ref): ?T { + const entry = this.entry(ref); + return entry ? entry.value : undefined; + } + + add(ref: Ref, size: number, value: ?T) { + const key = ref.toString(); + if (this._cache.has(key)) { + this._cache.delete(key); + } else { + this._size += size; + } + this._cache.set(key, new CacheEntry(size, value)); + + if (this._size > this._maxSize) { + for (const [key, {size}] of this._cache) { + if (this._size <= this._maxSize) { + break; + } + this._cache.delete(key); + this._size -= size; + } + } + } +} + +export class NoopCache { + entry(ref: Ref): ?CacheEntry {} // eslint-disable-line no-unused-vars + + get(ref: Ref): ?T {} // eslint-disable-line no-unused-vars + + add(ref: Ref, size: number, value: T) {} // eslint-disable-line no-unused-vars +} + + +class RefCacheEntry { + present: boolean; + type: ?Type; + provenance: Ref; + + constructor(present: boolean = false, type: ?Type = null, provenance: Ref = emptyRef) { + invariant((!present && !type) || (present && type), `present = ${present}, type = ${type}`); + this.present = present; + this.type = type; + this.provenance = provenance; + } +} + +class RefCache { + _cache: Map; + + constructor() { + this._cache = new Map(); + } + + get(ref: Ref): ?RefCacheEntry { + return this._cache.get(ref.toString()); + } + + add(ref: Ref, entry: RefCacheEntry) { + this._cache.set(ref.toString(), entry); + } + + addIfNotPresent(ref: Ref, entry: RefCacheEntry) { + const refStr = ref.toString(); + const cur = this._cache.get(refStr); + if (!cur || cur.provenance.isEmpty()) { + this._cache.set(refStr, entry); + } + } + + cacheChunks(v: valueOrPrimitive, ref: Ref) { + if (v instanceof Value) { + v.chunks.forEach(reachable => { + const hash = reachable.targetRef; + const cur = this.get(hash); + if (!cur || cur.provenance.isEmpty() || cur.provenance.equals(hash)) { + this.add(hash, new RefCacheEntry(true, getTargetType(reachable), ref)); + } + }); + } + } + + checkChunksInCache(v: valueOrPrimitive): Set { + const hints = new Set(); + if (v instanceof Value) { + const chunks = v.chunks; + for (let i = 0; i < chunks.length; i++) { + const reachable = chunks[i]; + const entry = this.get(reachable.targetRef); + invariant(entry && entry.present, () => + `Value to write -- Type ${describeTypeOfValue(v)} -- contains ref ` + + `${reachable.targetRef.toString()}, which points to a non-existent Value.`); + if (!entry.provenance.isEmpty()) { + hints.add(entry.provenance); + } + + // BUG 1121 + // It's possible that entry.type will be simply 'Value', but that 'reachable' is actually a + // properly-typed object -- that is, a Ref to some specific Type. The Exp below would fail, + // though it's possible that the Type is actually correct. We wouldn't be able to verify + // without reading it, though, so we'll dig into this later. + const targetType = getTargetType(reachable); + if (targetType.equals(valueType)) { + continue; + } + const entryType = notNull(entry.type); + invariant(entryType.equals(targetType), () => + `Value to write contains ref ${reachable.targetRef.toString()}, which points to a ` + + `value of a different type: ${describeType(entryType)} != ${describeType(targetType)}`); + } + } + return hints; + } +} + +function getTargetType(refVal: RefValue): Type { + invariant(refVal.type.kind === Kind.Ref, refVal.type.kind); + return refVal.type.elemTypes[0]; +}