diff --git a/datas/datastore_test.go b/datas/datastore_test.go index e6e13a0aef..b61bec4c6a 100644 --- a/datas/datastore_test.go +++ b/datas/datastore_test.go @@ -138,7 +138,7 @@ func (suite *DataStoreSuite) TestDataStoreCommit() { // \----|c| // Should be disallowed. c := types.NewString("c") - cCommit := NewCommit().Set(ValueField, c) + cCommit := NewCommit().Set(ValueField, c).Set(ParentsField, NewSetOfRefOfCommit().Insert(types.NewTypedRefFromValue(aCommit))) suite.ds, err = suite.ds.Commit(datasetID, cCommit) suite.Error(err) suite.True(suite.ds.Head(datasetID).Get(ValueField).Equals(b)) diff --git a/datas/http_batch_store.go b/datas/http_batch_store.go index 688f6dd073..1af8207d31 100644 --- a/datas/http_batch_store.go +++ b/datas/http_batch_store.go @@ -139,7 +139,7 @@ func (bhcs *httpBatchStore) batchGetRequests() { func (bhcs *httpBatchStore) sendGetRequests(req chunks.ReadRequest) { batch := chunks.ReadBatch{} - refs := types.Hints{} + refs := map[ref.Ref]struct{}{} addReq := func(req chunks.ReadRequest) { r := req.Ref() diff --git a/js/src/batch-store-adaptor.js b/js/src/batch-store-adaptor.js new file mode 100644 index 0000000000..fa3cf11bda --- /dev/null +++ b/js/src/batch-store-adaptor.js @@ -0,0 +1,45 @@ +// @flow + +import Ref from './ref.js'; +import MemoryStore from './memory-store.js'; +import BatchStore from './batch-store.js'; +import type {ChunkStore} from './chunk-store.js'; +import type {UnsentReadMap, WriteRequest} from './batch-store.js'; + +export function makeTestingBatchStore(): BatchStore { + return new BatchStore(3, new BatchStoreAdaptorDelegate(new MemoryStore())); +} + +export default class BatchStoreAdaptor extends BatchStore { + constructor(cs: ChunkStore, maxReads: number = 3) { + super(maxReads, new BatchStoreAdaptorDelegate(cs)); + } +} + +export class BatchStoreAdaptorDelegate { + _cs: ChunkStore; + + constructor(cs: ChunkStore) { + this._cs = cs; + } + + async readBatch(reqs: UnsentReadMap): Promise { + Object.keys(reqs).forEach(refStr => { + this._cs.get(Ref.parse(refStr)).then(chunk => { reqs[refStr](chunk); }); + }); + } + + async writeBatch(reqs: Array): Promise { + reqs.forEach(req => { + this._cs.put(req.c); + }); + } + + async getRoot(): Promise { + return this._cs.getRoot(); + } + + async updateRoot(current: Ref, last: Ref): Promise { + return this._cs.updateRoot(current, last); + } +} diff --git a/js/src/batch-store-test.js b/js/src/batch-store-test.js new file mode 100644 index 0000000000..dc491ff07b --- /dev/null +++ b/js/src/batch-store-test.js @@ -0,0 +1,39 @@ +// @flow + +import {suite, test} from 'mocha'; +import {assert} from 'chai'; +import MemoryStore from './memory-store.js'; +import BatchStore from './batch-store.js'; +import {BatchStoreAdaptorDelegate} from './batch-store-adaptor.js'; +import {stringType} from './type.js'; +import {encodeNomsValue} from './encode.js'; + +suite('BatchStore', () => { + test('get after schedulePut works immediately', async () => { + const ms = new MemoryStore(); + const bs = new BatchStore(3, new BatchStoreAdaptorDelegate(ms)); + const input = 'abc'; + + const c = encodeNomsValue(input, stringType); + bs.schedulePut(c, new Set()); + + const chunk = await bs.get(c.ref); + assert.isTrue(c.ref.equals(chunk.ref)); + }); + + test('get after schedulePut works after flush', async () => { + const ms = new MemoryStore(); + const bs = new BatchStore(3, new BatchStoreAdaptorDelegate(ms)); + const input = 'abc'; + + const c = encodeNomsValue(input, stringType); + bs.schedulePut(c, new Set()); + + let chunk = await bs.get(c.ref); + assert.isTrue(c.ref.equals(chunk.ref)); + + await bs.flush(); + chunk = await bs.get(c.ref); + assert.isTrue(c.ref.equals(chunk.ref)); + }); +}); diff --git a/js/src/remote-store.js b/js/src/batch-store.js similarity index 60% rename from js/src/remote-store.js rename to js/src/batch-store.js index beb8361987..ddf5a318fd 100644 --- a/js/src/remote-store.js +++ b/js/src/batch-store.js @@ -7,15 +7,20 @@ import {notNull} from './assert.js'; type PendingReadMap = { [key: string]: Promise }; export type UnsentReadMap = { [key: string]: (c: Chunk) => void }; -export type WriteMap = { [key: string]: Chunk }; +type WriteMap = { [key: string]: Chunk }; +export type WriteRequest = { + c: Chunk; + hints: Set; +} interface Delegate { readBatch(reqs: UnsentReadMap): Promise; - writeBatch(reqs: WriteMap): Promise; + writeBatch(reqs: Array): Promise; + getRoot(): Promise; updateRoot(current: Ref, last: Ref): Promise; } -export class RemoteStore { +export default class BatchStore { _pendingReads: PendingReadMap; _unsentReads: ?UnsentReadMap; _readScheduled: boolean; @@ -23,15 +28,10 @@ export class RemoteStore { _maxReads: number; _pendingWrites: WriteMap; - _unsentWrites: ?WriteMap; - _writeScheduled: boolean; - _activeWrites: number; - _maxWrites: number; - _allWritesFinishedFn: ?() => void; - _canUpdateRoot: Promise; + _unsentWrites: ?Array; _delegate: Delegate; - constructor(maxReads: number, maxWrites: number, delegate: Delegate) { + constructor(maxReads: number, delegate: Delegate) { this._pendingReads = Object.create(null); this._unsentReads = null; this._readScheduled = false; @@ -40,20 +40,19 @@ export class RemoteStore { this._pendingWrites = Object.create(null); this._unsentWrites = null; - this._writeScheduled = false; - this._activeWrites = 0; - this._maxWrites = maxWrites; - this._allWritesFinishedFn = null; - this._canUpdateRoot = Promise.resolve(); this._delegate = delegate; } get(ref: Ref): Promise { const refStr = ref.toString(); - const p = this._pendingReads[refStr]; + let p = this._pendingReads[refStr]; if (p) { return p; } + p = this._pendingWrites[refStr]; + if (p) { + return Promise.resolve(p); + } return this._pendingReads[refStr] = new Promise(resolve => { if (!this._unsentReads) { @@ -92,7 +91,7 @@ export class RemoteStore { this._maybeStartRead(); } - put(c: Chunk): void { + schedulePut(c: Chunk, hints: Set): void { const refStr = c.ref.toString(); if (this._pendingWrites[refStr]) { return; // Already in flight. @@ -100,60 +99,41 @@ export class RemoteStore { this._pendingWrites[refStr] = c; if (!this._unsentWrites) { - this._unsentWrites = Object.create(null); + this._unsentWrites = []; } - this._unsentWrites[refStr] = c; - - if (!this._allWritesFinishedFn) { - this._canUpdateRoot = new Promise(resolve => { - this._allWritesFinishedFn = resolve; - }); - } - - this._maybeStartWrite(); + this._unsentWrites.push({c: c, hints: hints}); } - _maybeStartWrite() { - if (!this._writeScheduled && this._unsentWrites && this._activeWrites < this._maxWrites) { - this._writeScheduled = true; - setTimeout(() => { - this._write(); - }, 0); + async flush(): Promise { + if (!this._unsentWrites) { + return; } - } - - async _write(): Promise { - this._activeWrites++; const reqs = notNull(this._unsentWrites); this._unsentWrites = null; - this._writeScheduled = false; - await this._delegate.writeBatch(reqs); + await this._delegate.writeBatch(reqs); // TODO: Deal with backpressure const self = this; // TODO: Remove this when babel bug is fixed. - Object.keys(reqs).forEach(refStr => { - delete self._pendingWrites[refStr]; + reqs.forEach(req => { + delete self._pendingWrites[req.c.ref.toString()]; }); + } - this._activeWrites--; - - if (this._activeWrites === 0 && !this._unsentWrites) { - notNull(this._allWritesFinishedFn)(); - this._allWritesFinishedFn = null; - } - - this._maybeStartWrite(); + async getRoot(): Promise { + return this._delegate.getRoot(); } async updateRoot(current: Ref, last: Ref): Promise { - await this._canUpdateRoot; + await this.flush(); if (current.equals(last)) { - return Promise.resolve(true); + return true; } return this._delegate.updateRoot(current, last); } + // TODO: Should close() call flush() and block until it's done? Maybe closing with outstanding + // requests should be an error on both sides. TBD. close() {} } diff --git a/js/src/blob-test.js b/js/src/blob-test.js index 85009670ea..d26d7185d4 100644 --- a/js/src/blob-test.js +++ b/js/src/blob-test.js @@ -3,7 +3,7 @@ import {assert} from 'chai'; import {suite, test} from 'mocha'; import Random from './pseudo-random.js'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {newBlob, BlobWriter} from './blob.js'; import DataStore from './data-store.js'; import {blobType, makeRefType} from './type.js'; @@ -48,8 +48,7 @@ suite('Blob', () => { }); test('roundtrip', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const b1 = await newBlob(randomArray(15)); const r1 = await ds.writeValue(b1).targetRef; diff --git a/js/src/blob.js b/js/src/blob.js index 4ae403cd21..422771a830 100644 --- a/js/src/blob.js +++ b/js/src/blob.js @@ -4,7 +4,7 @@ import {Collection} from './collection.js'; import {IndexedSequence} from './indexed-sequence.js'; import {SequenceCursor} from './sequence.js'; import {invariant} from './assert.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import {blobType} from './type.js'; import { MetaTuple, @@ -59,9 +59,9 @@ export class BlobReader { } export class BlobLeafSequence extends IndexedSequence { - constructor(cs: ?DataStore, items: Uint8Array) { + constructor(vr: ?ValueReader, items: Uint8Array) { // $FlowIssue: The super class expects Array but we sidestep that. - super(cs, blobType, items); + super(vr, blobType, items); } getOffset(idx: number): number { @@ -72,9 +72,9 @@ export class BlobLeafSequence extends IndexedSequence { const blobWindowSize = 64; const blobPattern = ((1 << 13) | 0) - 1; -function newBlobLeafChunkFn(cs: ?DataStore = null): makeChunkFn { +function newBlobLeafChunkFn(vr: ?ValueReader = null): makeChunkFn { return (items: Array) => { - const blobLeaf = new BlobLeafSequence(cs, new Uint8Array(items)); + const blobLeaf = new BlobLeafSequence(vr, new Uint8Array(items)); const mt = new MetaTuple(blobLeaf, items.length, items.length); return [mt, blobLeaf]; }; diff --git a/js/src/chunk-serializer-test.js b/js/src/chunk-serializer-test.js index 6348c8b3ca..724306312c 100644 --- a/js/src/chunk-serializer-test.js +++ b/js/src/chunk-serializer-test.js @@ -3,42 +3,68 @@ import {suite, test} from 'mocha'; import {assert} from 'chai'; import Chunk from './chunk.js'; +import Ref from './ref.js'; import {deserialize, serialize} from './chunk-serializer.js'; suite('ChunkSerializer', () => { + function assertHints(expect: Array, actual: Array) { + assert.strictEqual(actual.length, expect.length); + for (let i = 0; i < expect.length; i++) { + assert.isTrue(expect[i].equals(actual[i])); + } + } + function assertChunks(expect: Array, actual: Array) { - assert.strictEqual(expect.length, actual.length); + assert.strictEqual(actual.length, expect.length); for (let i = 0; i < expect.length; i++) { assert.isTrue(expect[i].ref.equals(actual[i].ref)); } } test('simple', () => { - const chunks = [Chunk.fromString('abc'), Chunk.fromString('def'), Chunk.fromString('ghi'), - Chunk.fromString('wacka wack wack')]; + const expHints = []; + const expChunks = [Chunk.fromString('abc'), Chunk.fromString('def'), Chunk.fromString('ghi'), + Chunk.fromString('wacka wack wack')]; - const buffer = serialize(chunks); - const newChunks = deserialize(buffer); + const {hints, chunks} = deserialize(serialize(new Set(expHints), expChunks)); - assertChunks(chunks, newChunks); + assertHints(expHints, hints); + assertChunks(expChunks, chunks); }); test('leading & trailing empty', () => { - const chunks = [Chunk.fromString(''), Chunk.fromString('A'), Chunk.fromString('')]; + const expHints = []; + const expChunks = [Chunk.fromString(''), Chunk.fromString('A'), Chunk.fromString('')]; - const buffer = serialize(chunks); - const newChunks = deserialize(buffer); + const {hints, chunks} = deserialize(serialize(new Set(expHints), expChunks)); - assertChunks(chunks, newChunks); + assertHints(expHints, hints); + assertChunks(expChunks, chunks); }); - test('no chunks', () => { - const chunks = []; + test('all empty', () => { + const expHints = []; + const expChunks = []; - const buffer = serialize(chunks); - const newChunks = deserialize(buffer); + const {hints, chunks} = deserialize(serialize(new Set(expHints), expChunks)); - assertChunks(chunks, newChunks); + assertHints(expHints, hints); + assertChunks(expChunks, chunks); + }); + + test('with hints', () => { + const expHints = [ + Chunk.fromString('123').ref, + Chunk.fromString('456').ref, + Chunk.fromString('789').ref, + Chunk.fromString('wacka wack wack').ref, + ]; + const expChunks = [Chunk.fromString('abc'), Chunk.fromString('def'), Chunk.fromString('ghi')]; + + const {hints, chunks} = deserialize(serialize(new Set(expHints), expChunks)); + + assertHints(expHints, hints); + assertChunks(expChunks, chunks); }); }); diff --git a/js/src/chunk-serializer.js b/js/src/chunk-serializer.js index 602f1b1e10..e81df16cc4 100644 --- a/js/src/chunk-serializer.js +++ b/js/src/chunk-serializer.js @@ -4,21 +4,22 @@ import Chunk from './chunk.js'; import Ref from './ref.js'; import {invariant} from './assert.js'; +const headerSize = 4; // uint32 +const littleEndian = true; const sha1Size = 20; const chunkLengthSize = 4; // uint32 const chunkHeaderSize = sha1Size + chunkLengthSize; -export function serialize(chunks: Array): ArrayBuffer { - let totalSize = 0; - for (let i = 0; i < chunks.length; i++) { - totalSize += chunkHeaderSize + chunks[i].data.length; - } +export function serialize(hints: Set, chunks: Array): ArrayBuffer { + const buffer = new ArrayBuffer(serializedHintLength(hints) + serializedChunkLength(chunks)); - const buffer = new ArrayBuffer(totalSize); - let offset = 0; + let offset = serializeHints(hints, buffer); for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; + invariant(buffer.byteLength - offset >= chunkHeaderSize + chunk.data.length, + 'Invalid chunk buffer'); + const refArray = new Uint8Array(buffer, offset, sha1Size); refArray.set(chunk.ref.digest); offset += sha1Size; @@ -40,11 +41,65 @@ export function serialize(chunks: Array): ArrayBuffer { return buffer; } -export function deserialize(buffer: ArrayBuffer): Array { +function serializeHints(hints: Set, buffer: ArrayBuffer): number { + let offset = 0; + const view = new DataView(buffer, offset, headerSize); + view.setUint32(offset, hints.size | 0, littleEndian); // Coerce number to uint32 + offset += headerSize; + + hints.forEach(ref => { + const refArray = new Uint8Array(buffer, offset, sha1Size); + refArray.set(ref.digest); + offset += sha1Size; + }); + + return offset; +} + +function serializedHintLength(hints: Set): number { + return headerSize + sha1Size * hints.size; +} + +function serializedChunkLength(chunks: Array): number { + let totalSize = 0; + for (let i = 0; i < chunks.length; i++) { + totalSize += chunkHeaderSize + chunks[i].data.length; + } + return totalSize; +} + +export function deserialize(buffer: ArrayBuffer): {hints: Array, chunks: Array} { + const {hints, offset} = deserializeHints(buffer); + return {hints: hints, chunks: deserializeChunks(buffer, offset)}; +} + +function deserializeHints(buffer: ArrayBuffer): {hints: Array, offset: number} { + const hints:Array = []; + + let offset = 0; + const view = new DataView(buffer, 0, headerSize); + const numHints = view.getUint32(0, littleEndian); + offset += headerSize; + + const totalLength = headerSize + (numHints * sha1Size); + for (; offset < totalLength;) { + invariant(buffer.byteLength - offset >= sha1Size, 'Invalid hint buffer'); + + const refArray = new Uint8Array(buffer, offset, sha1Size); + const ref = Ref.fromDigest(new Uint8Array(refArray)); + offset += sha1Size; + + hints.push(ref); + } + + return {hints: hints, offset: offset}; +} + +export function deserializeChunks(buffer: ArrayBuffer, offset: number = 0): Array { const chunks:Array = []; const totalLenth = buffer.byteLength; - for (let offset = 0; offset < totalLenth;) { + for (; offset < totalLenth;) { invariant(buffer.byteLength - offset >= chunkHeaderSize, 'Invalid chunk buffer'); const refArray = new Uint8Array(buffer, offset, sha1Size); diff --git a/js/src/data-store-test.js b/js/src/data-store-test.js index 4ee590777e..8eec489497 100644 --- a/js/src/data-store-test.js +++ b/js/src/data-store-test.js @@ -1,34 +1,35 @@ // @flow import {suite, test} from 'mocha'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {emptyRef} from './ref.js'; import {assert} from 'chai'; import {default as DataStore, getDatasTypes, newCommit} from './data-store.js'; import {invariant, notNull} from './assert.js'; import {newMap} from './map.js'; import {stringType} from './type.js'; -import {getRef} from './get-ref.js'; import {encodeNomsValue} from './encode.js'; suite('DataStore', () => { test('access', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const bs = makeTestingBatchStore(); + const ds = new DataStore(bs); const input = 'abc'; const c = encodeNomsValue(input, stringType); const v1 = await ds.readValue(c.ref); assert.equal(null, v1); - ms.put(c); + bs.schedulePut(c, new Set()); + bs.flush(); + const v2 = await ds.readValue(c.ref); assert.equal('abc', v2); }); test('commit', async () => { - const ms = new MemoryStore(); - let ds = new DataStore(ms); + const bs = new makeTestingBatchStore(); + let ds = new DataStore(bs); const datasetID = 'ds1'; const datasets = await ds.datasets(); @@ -92,14 +93,14 @@ suite('DataStore', () => { assert.strictEqual('a', notNull(await ds.head('otherDs')).value); // Get a fresh datastore, and verify that both datasets are present - const newDs = new DataStore(ms); + const newDs = new DataStore(bs); assert.strictEqual('d', notNull(await newDs.head(datasetID)).value); assert.strictEqual('a', notNull(await newDs.head('otherDs')).value); }); test('concurrency', async () => { - const ms = new MemoryStore(); - let ds = new DataStore(ms); + const bs = new makeTestingBatchStore(); + let ds = new DataStore(bs); const datasetID = 'ds1'; // |a| @@ -112,7 +113,7 @@ suite('DataStore', () => { assert.strictEqual('b', notNull(await ds.head(datasetID)).value); // Important to create this here. - let ds2 = new DataStore(ms); + let ds2 = new DataStore(bs); // Change 1: // |a| <- |b| <- |c| @@ -137,15 +138,14 @@ suite('DataStore', () => { test('empty datasets', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const datasets = await ds.datasets(); assert.strictEqual(0, datasets.size); }); test('head', async () => { - const ms = new MemoryStore(); - let ds = new DataStore(ms); + const bs = new makeTestingBatchStore(); + let ds = new DataStore(bs); const types = getDatasTypes(); const commit = await newCommit('foo', []); @@ -153,8 +153,8 @@ suite('DataStore', () => { const commitRef = ds.writeValue(commit); const datasets = await newMap(['foo', commitRef], types.commitMapType); const rootRef = ds.writeValue(datasets).targetRef; - assert.isTrue(await ms.updateRoot(rootRef, emptyRef)); - ds = new DataStore(ms); // refresh the datasets + assert.isTrue(await bs.updateRoot(rootRef, emptyRef)); + ds = new DataStore(bs); // refresh the datasets assert.strictEqual(1, datasets.size); const fooHead = await ds.head('foo'); @@ -163,61 +163,4 @@ suite('DataStore', () => { const barHead = await ds.head('bar'); assert.isNull(barHead); }); - - test('writeValue primitives', async () => { - const ds = new DataStore(new MemoryStore()); - - const r1 = ds.writeValue('hello').targetRef; - const r2 = ds.writeValue(false).targetRef; - const r3 = ds.writeValue(2).targetRef; - - const v1 = await ds.readValue(r1); - assert.equal('hello', v1); - const v2 = await ds.readValue(r2); - assert.equal(false, v2); - const v3 = await ds.readValue(r3); - assert.equal(2, v3); - }); - - test('caching', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms, 1e6); - - const r1 = ds.writeValue('hello').targetRef; - (ms: any).get = (ms: any).put = () => { assert.fail('unreachable'); }; - const v1 = await ds.readValue(r1); - assert.equal(v1, 'hello'); - const r2 = ds.writeValue('hello').targetRef; - assert.isTrue(r1.equals(r2)); - }); - - test('caching eviction', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms, 15); - - const r1 = ds.writeValue('hello').targetRef; - const r2 = ds.writeValue('world').targetRef; - (ms: any).get = () => { throw new Error(); }; - const v2 = await ds.readValue(r2); - assert.equal(v2, 'world'); - let ex; - try { - await ds.readValue(r1); - } catch (e) { - ex = e; - } - assert.instanceOf(ex, Error); - }); - - test('caching has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms, 1e6); - - const r1 = getRef('hello', stringType); - const v1 = await ds.readValue(r1); - assert.equal(v1, null); - (ms: any).get = (ms: any).has = () => { assert.fail('unreachable'); }; - const v2 = await ds.readValue(r1); - assert.equal(v2, null); - }); }); diff --git a/js/src/data-store.js b/js/src/data-store.js index 00d0b5c3c9..8095f2d244 100644 --- a/js/src/data-store.js +++ b/js/src/data-store.js @@ -1,16 +1,15 @@ // @flow -import Chunk from './chunk.js'; import Ref from './ref.js'; -import {default as RefValue, refValueFromValue} from './ref-value.js'; +import {default as RefValue} from './ref-value.js'; import {newStruct} from './struct.js'; -import type {ChunkStore} from './chunk-store.js'; import type {NomsMap} from './map.js'; import type {NomsSet} from './set.js'; import type {valueOrPrimitive} from './value.js'; +import ValueStore from './value-store.js'; +import BatchStore from './batch-store.js'; import { Field, - getTypeOfValue, makeRefType, makeStructType, makeSetType, @@ -21,9 +20,6 @@ import { } from './type.js'; import {newMap} from './map.js'; import {newSet} from './set.js'; -import {decodeNomsValue} from './decode.js'; -import {invariant} from './assert.js'; -import {encodeNomsValue} from './encode.js'; import type {Commit} from './commit.js'; type DatasTypes = { @@ -66,21 +62,17 @@ export function getDatasTypes(): DatasTypes { return datasTypes; } -interface Cache { // eslint-disable-line no-undef - entry(ref: Ref): ?CacheEntry; // eslint-disable-line no-undef - get(ref: Ref): ?T; // eslint-disable-line no-undef - add(ref: Ref, size: number, value: T): void; // eslint-disable-line no-undef -} - -export default class DataStore { - _cs: ChunkStore; +export default class DataStore extends ValueStore { + _bs: BatchStore; + _cacheSize: number; _datasets: Promise>>; - _valueCache: Cache; - constructor(cs: ChunkStore, cacheSize: number = 0) { - this._cs = cs; - this._datasets = this._datasetsFromRootRef(cs.getRoot()); - this._valueCache = cacheSize > 0 ? new SizeCache(cacheSize) : new NoopCache(); + constructor(bs: BatchStore, cacheSize: number = 0) { + super(bs, cacheSize); + // bs and cacheSize should only be used when creating a new DataStore instance in commit() + this._bs = bs; + this._cacheSize = cacheSize; + this._datasets = this._datasetsFromRootRef(bs.getRoot()); } _datasetsFromRootRef(rootRef: Promise): Promise>> { @@ -116,40 +108,8 @@ export default class DataStore { return true; } - // TODO: This should return Promise - async readValue(ref: Ref): Promise { - const entry = this._valueCache.entry(ref); - if (entry) { - return entry.value; - } - const chunk: Chunk = await this._cs.get(ref); - if (chunk.isEmpty()) { - this._valueCache.add(ref, 0, null); - return null; - } - - const v = decodeNomsValue(chunk, this); - this._valueCache.add(ref, chunk.data.length, v); - return v; - } - - writeValue(v: T): RefValue { - const t = getTypeOfValue(v); - const chunk = encodeNomsValue(v, t, this); - invariant(!chunk.isEmpty()); - const {ref} = chunk; - const refValue = refValueFromValue(v); - const entry = this._valueCache.entry(ref); - if (entry && entry.present) { - return refValue; - } - this._cs.put(chunk); - this._valueCache.add(ref, chunk.data.length, v); - return refValue; - } - async commit(datasetId: string, commit: Commit): Promise { - const currentRootRefP = this._cs.getRoot(); + const currentRootRefP = this._bs.getRoot(); const datasetsP = this._datasetsFromRootRef(currentRootRefP); let currentDatasets = await (datasetsP:Promise); const currentRootRef = await currentRootRefP; @@ -169,8 +129,8 @@ export default class DataStore { currentDatasets = await currentDatasets.set(datasetId, commitRef); const newRootRef = this.writeValue(currentDatasets).targetRef; - if (await this._cs.updateRoot(newRootRef, currentRootRef)) { - return new DataStore(this._cs); + if (await this._bs.updateRoot(newRootRef, currentRootRef)) { + return new DataStore(this._bs, this._cacheSize); } throw new Error('Optimistic lock failed'); @@ -192,79 +152,4 @@ export function newCommit(value: valueOrPrimitive, const types = getDatasTypes(); return newSet(parentsArr, types.commitSetType).then(parents => newStruct(types.commitType, {value, parents})); -} - -class CacheEntry { - size: number; - value: ?T; - - constructor(size: number, value: ?T) { - this.size = size; - this.value = value; - } - - get present(): boolean { - return this.value !== null; - } -} - -/** - * This uses a Map as an LRU cache. It uses the behavior that iteration of keys in a Map is done in - * insertion order and any time a value is checked it is taken out and reinserted which puts it last - * in the iteration. - */ -class SizeCache { - _size: number; - _maxSize: number; - _cache: Map>; - - constructor(size: number) { - this._maxSize = size; - this._cache = new Map(); - this._size = 0; - } - - entry(ref: Ref): ?CacheEntry { - const key = ref.toString(); - const entry = this._cache.get(key); - if (!entry) { - return undefined; - } - this._cache.delete(key); - this._cache.set(key, entry); - return entry; - } - - get(ref: Ref): ?T { - const entry = this.entry(ref); - return entry ? entry.value : undefined; - } - - add(ref: Ref, size: number, value: ?T) { - const key = ref.toString(); - if (this._cache.has(key)) { - this._cache.delete(key); - } else { - this._size += size; - } - this._cache.set(key, new CacheEntry(size, value)); - - if (this._size > this._maxSize) { - for (const [key, {size}] of this._cache) { - if (this._size <= this._maxSize) { - break; - } - this._cache.delete(key); - this._size -= size; - } - } - } -} - -class NoopCache { - entry(ref: Ref): ?CacheEntry {} // eslint-disable-line no-unused-vars - - get(ref: Ref): ?T {} // eslint-disable-line no-unused-vars - - add(ref: Ref, size: number, value: T) {} // eslint-disable-line no-unused-vars -} +} \ No newline at end of file diff --git a/js/src/dataset-test.js b/js/src/dataset-test.js index b0fe12ca48..c11cf4ec10 100644 --- a/js/src/dataset-test.js +++ b/js/src/dataset-test.js @@ -1,7 +1,7 @@ // @flow import {suite, test} from 'mocha'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {assert} from 'chai'; import Dataset from './dataset.js'; import DataStore from './data-store.js'; @@ -9,8 +9,8 @@ import {invariant, notNull} from './assert.js'; suite('Dataset', () => { test('commit', async () => { - const ms = new MemoryStore(); - const store = new DataStore(ms); + const bs = makeTestingBatchStore(); + const store = new DataStore(bs); let ds = new Dataset(store, 'ds1'); // |a| @@ -54,7 +54,7 @@ suite('Dataset', () => { assert.strictEqual('a', notNull(await ds.head('otherDs')).value); // Get a fresh datastore, and verify that both datasets are present - const newStore = new DataStore(ms); + const newStore = new DataStore(bs); assert.strictEqual('d', notNull(await newStore.head('ds1')).value); assert.strictEqual('a', notNull(await newStore.head('otherDs')).value); }); diff --git a/js/src/decode-test.js b/js/src/decode-test.js index b0d5d358f0..61aabba16a 100644 --- a/js/src/decode-test.js +++ b/js/src/decode-test.js @@ -2,7 +2,7 @@ import Chunk from './chunk.js'; import DataStore from './data-store.js'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import Ref from './ref.js'; import RefValue from './ref-value.js'; import {default as Struct, StructMirror} from './struct.js'; @@ -44,8 +44,7 @@ suite('Decode', () => { } test('read', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [1, 'hi', true]; const r = new JsonArrayReader(a, ds); @@ -60,8 +59,7 @@ suite('Decode', () => { }); test('read type as tag', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); function doTest(expected: Type, a: Array) { const r = new JsonArrayReader(a, ds); const tr = r.readTypeAsTag([]); @@ -75,8 +73,7 @@ suite('Decode', () => { }); test('read primitives', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); async function doTest(expected: any, a: Array): Promise { const r = new JsonArrayReader(a, ds); @@ -96,8 +93,7 @@ suite('Decode', () => { }); test('read list of number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.List, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsList = await r.readTopLevelValue(); @@ -110,8 +106,7 @@ suite('Decode', () => { // TODO: Can't round-trip collections of value types. =-( test('read list of value', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.List, Kind.Value, false, [Kind.Number, '1', Kind.String, 'hi', Kind.Bool, true]]; const r = new JsonArrayReader(a, ds); @@ -126,8 +121,7 @@ suite('Decode', () => { }); test('read value list of number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Value, Kind.List, Kind.Number, false, ['0', '1', '2']]; const r = new JsonArrayReader(a, ds); const v = await r.readTopLevelValue(); @@ -139,8 +133,7 @@ suite('Decode', () => { }); test('read compound list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const ltr = makeListType(numberType); const r1 = ds.writeValue(new NomsList(ltr, new ListLeafSequence(ds, ltr, [0]))).targetRef; const r2 = ds.writeValue(new NomsList(ltr, new ListLeafSequence(ds, ltr, [1, 2]))).targetRef; @@ -161,8 +154,7 @@ suite('Decode', () => { }); test('read map of number to number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Map, Kind.Number, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsMap = await r.readTopLevelValue(); @@ -174,8 +166,7 @@ suite('Decode', () => { }); test('read map of ref to number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Map, Kind.Ref, Kind.Value, Kind.Number, false, ['sha1-0000000000000000000000000000000000000001', '2', 'sha1-0000000000000000000000000000000000000002', '4']]; @@ -195,8 +186,7 @@ suite('Decode', () => { }); test('read value map of number to number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Value, Kind.Map, Kind.Number, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsMap = await r.readTopLevelValue(); @@ -208,8 +198,7 @@ suite('Decode', () => { }); test('read set of number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Set, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsSet = await r.readTopLevelValue(); @@ -221,8 +210,7 @@ suite('Decode', () => { }); test('read compound set', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const ltr = makeSetType(numberType); const r1 = ds.writeValue(new NomsSet(ltr, new SetLeafSequence(ds, ltr, [0]))).targetRef; const r2 = ds.writeValue(new NomsSet(ltr, new SetLeafSequence(ds, ltr, [1, 2]))).targetRef; @@ -243,8 +231,7 @@ suite('Decode', () => { }); test('read value set of number', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [Kind.Value, Kind.Set, Kind.Number, false, ['0', '1', '2', '3']]; const r = new JsonArrayReader(a, ds); const v: NomsSet = await r.readTopLevelValue(); @@ -267,8 +254,7 @@ suite('Decode', () => { } test('test read struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeStructType('A1', [ new Field('x', numberType), new Field('s', stringType), @@ -291,8 +277,7 @@ suite('Decode', () => { }); test('test read struct with list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const ltr = makeListType(numberType); const tr = makeStructType('A4', [ new Field('b', boolType), @@ -316,8 +301,7 @@ suite('Decode', () => { }); test('test read struct with value', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeStructType('A5', [ new Field('b', boolType), new Field('v', valueType), @@ -337,8 +321,7 @@ suite('Decode', () => { }); test('test read value struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeStructType('A1', [ new Field('x', numberType), new Field('s', stringType), @@ -362,8 +345,7 @@ suite('Decode', () => { }); test('test read map of string to struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeStructType('s', [ new Field('b', boolType), new Field('i', numberType), @@ -384,11 +366,10 @@ suite('Decode', () => { }); test('decodeNomsValue', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const chunk = Chunk.fromString( `t [${Kind.Value}, ${Kind.Set}, ${Kind.Number}, false, ["0", "1", "2", "3"]]`); - const v = decodeNomsValue(chunk, new DataStore(new MemoryStore())); + const v = decodeNomsValue(chunk, new DataStore(makeTestingBatchStore())); invariant(v instanceof NomsSet); const t = makeSetType(numberType); @@ -397,8 +378,8 @@ suite('Decode', () => { }); test('decodeNomsValue: counter with one commit', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const bs = makeTestingBatchStore(); + const ds = new DataStore(bs); const makeChunk = a => Chunk.fromString(`t ${JSON.stringify(a)}`); @@ -413,15 +394,16 @@ suite('Decode', () => { ['value', Kind.Value, 'parents', Kind.Set, Kind.Ref, Kind.Parent, 0], Kind.Number, '1', false, []]); const commitRef = commitChunk.ref; - ms.put(commitChunk); + bs.schedulePut(commitChunk, new Set()); // Root const rootChunk = makeChunk([Kind.Map, Kind.String, Kind.Ref, Kind.Struct, 'Commit', ['value', Kind.Value, 'parents', Kind.Set, Kind.Ref, Kind.Parent, 0], false, ['counter', commitRef.toString()]]); const rootRef = rootChunk.ref; - ms.put(rootChunk); + bs.schedulePut(rootChunk, new Set()); + await bs.flush(); const rootMap = await ds.readValue(rootRef); const counterRef = await rootMap.get('counter'); const commit = await counterRef.targetValue(ds); @@ -430,7 +412,7 @@ suite('Decode', () => { test('out of line blob', async () => { const chunk = Chunk.fromString('b hi'); - const blob = decodeNomsValue(chunk, new DataStore(new MemoryStore())); + const blob = decodeNomsValue(chunk, new DataStore(makeTestingBatchStore())); invariant(blob instanceof NomsBlob); const r = await blob.getReader().read(); assert.isFalse(r.done); @@ -448,7 +430,7 @@ suite('Decode', () => { } const chunk2 = new Chunk(data); - const blob2 = decodeNomsValue(chunk2, new DataStore(new MemoryStore())); + const blob2 = decodeNomsValue(chunk2, new DataStore(makeTestingBatchStore())); invariant(blob2 instanceof NomsBlob); const r2 = await blob2.getReader().read(); assert.isFalse(r2.done); @@ -458,8 +440,7 @@ suite('Decode', () => { }); test('inline blob', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const a = [ Kind.List, Kind.Blob, false, [false, encodeBase64(stringToUint8Array('hello')), @@ -478,8 +459,7 @@ suite('Decode', () => { }); test('compound blob', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const r1 = ds.writeValue(await newBlob(stringToUint8Array('hi'))).targetRef; const r2 = ds.writeValue(await newBlob(stringToUint8Array('world'))).targetRef; @@ -497,8 +477,7 @@ suite('Decode', () => { }); test('recursive struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); // struct A { // b: struct B { diff --git a/js/src/decode.js b/js/src/decode.js index 739e54807d..059b5f9f07 100644 --- a/js/src/decode.js +++ b/js/src/decode.js @@ -6,7 +6,6 @@ import Ref from './ref.js'; import RefValue from './ref-value.js'; import {newStruct} from './struct.js'; import type Struct from './struct.js'; -import type DataStore from './data-store.js'; import type {NomsKind} from './noms-kind.js'; import {decode as decodeBase64} from './base64.js'; import { @@ -34,9 +33,9 @@ const blobTag = 'b '; export class JsonArrayReader { _a: Array; _i: number; - _ds: DataStore; + _ds: ValueReader; - constructor(a: Array, ds: DataStore) { + constructor(a: Array, ds: ValueReader) { this._a = a; this._i = 0; this._ds = ds; @@ -325,17 +324,21 @@ export class JsonArrayReader { } } -export function decodeNomsValue(chunk: Chunk, ds: DataStore): valueOrPrimitive { +export interface ValueReader { + readValue(ref: Ref): Promise +} + +export function decodeNomsValue(chunk: Chunk, vr: ValueReader): valueOrPrimitive { const tag = new Chunk(new Uint8Array(chunk.data.buffer, 0, 2)).toString(); switch (tag) { case typedTag: { const payload = JSON.parse(new Chunk(new Uint8Array(chunk.data.buffer, 2)).toString()); - const reader = new JsonArrayReader(payload, ds); + const reader = new JsonArrayReader(payload, vr); return reader.readTopLevelValue(); } case blobTag: - return new NomsBlob(new BlobLeafSequence(ds, new Uint8Array(chunk.data.buffer, 2))); + return new NomsBlob(new BlobLeafSequence(vr, new Uint8Array(chunk.data.buffer, 2))); default: throw new Error('Not implemented'); } diff --git a/js/src/encode-test.js b/js/src/encode-test.js index 112b2031da..c37441db1e 100644 --- a/js/src/encode-test.js +++ b/js/src/encode-test.js @@ -3,7 +3,7 @@ import {assert} from 'chai'; import {suite, test} from 'mocha'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import Ref from './ref.js'; import RefValue from './ref-value.js'; import {newStruct} from './struct.js'; @@ -35,8 +35,7 @@ import type {valueOrPrimitive} from './value.js'; suite('Encode', () => { test('write primitives', () => { function f(k: NomsKind, t:Type, v: valueOrPrimitive, ex: valueOrPrimitive) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); w.writeTopLevel(t, v); assert.deepEqual([k, ex], w.array); @@ -55,8 +54,7 @@ suite('Encode', () => { }); test('write simple blob', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const blob = await newBlob(new Uint8Array([0x00, 0x01])); w.writeTopLevel(blobType, blob); @@ -64,8 +62,7 @@ suite('Encode', () => { }); test('write list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const tr = makeCompoundType(Kind.List, numberType); @@ -75,8 +72,7 @@ suite('Encode', () => { }); test('write list of value', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const tr = makeCompoundType(Kind.List, valueType); @@ -91,8 +87,7 @@ suite('Encode', () => { }); test('write list of list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const it = makeCompoundType(Kind.List, numberType); @@ -107,8 +102,7 @@ suite('Encode', () => { }); test('write leaf set', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const tr = makeCompoundType(Kind.Set, numberType); @@ -118,8 +112,7 @@ suite('Encode', () => { }); test('write compound set', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const ltr = makeCompoundType(Kind.Set, numberType); const r1 = ds.writeValue(new NomsSet(ltr, new SetLeafSequence(ds, ltr, [0]))).targetRef; @@ -138,8 +131,7 @@ suite('Encode', () => { }); test('write set of set', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const st = makeCompoundType(Kind.Set, numberType); @@ -155,8 +147,7 @@ suite('Encode', () => { }); test('write map', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const tr = makeCompoundType(Kind.Map, stringType, boolType); @@ -167,8 +158,7 @@ suite('Encode', () => { }); test('write map of map', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const kt = makeCompoundType(Kind.Map, stringType, numberType); @@ -184,8 +174,7 @@ suite('Encode', () => { }); test('write empty struct', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const type = makeStructType('S', []); @@ -196,8 +185,7 @@ suite('Encode', () => { }); test('write struct', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const type = makeStructType('S', [ @@ -212,8 +200,7 @@ suite('Encode', () => { }); test('write struct with list', async() => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); let w = new JsonArrayWriter(ds); const ltr = makeCompoundType(Kind.List, stringType); @@ -232,8 +219,7 @@ suite('Encode', () => { }); test('write struct with struct', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const s2Type = makeStructType('S2', [ @@ -250,8 +236,7 @@ suite('Encode', () => { }); test('write compound list', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const ltr = makeListType(numberType); const r1 = ds.writeValue(new NomsList(ltr, new ListLeafSequence(ds, ltr, [0]))).targetRef; @@ -270,8 +255,7 @@ suite('Encode', () => { }); test('write type value', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const test = (expected: Array, v: Type) => { const w = new JsonArrayWriter(ds); @@ -314,8 +298,7 @@ suite('Encode', () => { return bytes; } - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const blob = await newBlob(stringToUint8Array('hi')); const chunk = encodeNomsValue(blob, blobType, ds); @@ -338,8 +321,7 @@ suite('Encode', () => { }); test('write ref', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const ref = Ref.parse('sha1-0123456789abcdef0123456789abcdef01234567'); const t = makeCompoundType(Kind.Ref, blobType); @@ -350,7 +332,7 @@ suite('Encode', () => { }); test('type errors', async () => { - const ds = new DataStore(new MemoryStore()); + const ds = new DataStore(makeTestingBatchStore()); const w = new JsonArrayWriter(ds); const test = (et, at, t, v) => { diff --git a/js/src/encode.js b/js/src/encode.js index 959e158dcf..9011362c21 100644 --- a/js/src/encode.js +++ b/js/src/encode.js @@ -4,7 +4,6 @@ import Chunk from './chunk.js'; import type Ref from './ref.js'; import RefValue from './ref-value.js'; import {default as Struct, StructMirror} from './struct.js'; -import type DataStore from './data-store.js'; import type {NomsKind} from './noms-kind.js'; import {encode as encodeBase64} from './base64.js'; import {StructDesc, Type, getTypeOfValue} from './type.js'; @@ -28,11 +27,11 @@ type primitiveOrArray = primitive | Array; export class JsonArrayWriter { array: Array; - _ds: ?DataStore; + _vw: ?ValueWriter; - constructor(ds: ?DataStore) { + constructor(ds: ?ValueWriter) { this.array = []; - this._ds = ds; + this._vw = ds; } write(v: primitiveOrArray) { @@ -97,14 +96,14 @@ export class JsonArrayWriter { } this.write(true); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); const indexType = indexTypeForMetaSequence(t); for (let i = 0; i < v.items.length; i++) { const tuple = v.items[i]; invariant(tuple instanceof MetaTuple); - if (tuple.sequence && this._ds) { + if (tuple.sequence && this._vw) { const child = tuple.sequence; - this._ds.writeValue(child); + this._vw.writeValue(child); } w2.writeRef(tuple.ref); w2.writeValue(tuple.value, indexType); @@ -154,7 +153,7 @@ export class JsonArrayWriter { } invariant(sequence instanceof ListLeafSequence); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); const elemType = t.elemTypes[0]; sequence.items.forEach(sv => w2.writeValue(sv, elemType)); this.write(w2.array); @@ -170,7 +169,7 @@ export class JsonArrayWriter { } invariant(sequence instanceof MapLeafSequence); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); const keyType = t.elemTypes[0]; const valueType = t.elemTypes[1]; sequence.items.forEach(entry => { @@ -196,7 +195,7 @@ export class JsonArrayWriter { } invariant(sequence instanceof SetLeafSequence); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); const elemType = t.elemTypes[0]; const elems = []; sequence.items.forEach(v => { @@ -236,7 +235,7 @@ export class JsonArrayWriter { case Kind.Ref: case Kind.Set: { this.writeKind(k); - const w2 = new JsonArrayWriter(this._ds); + const w2 = new JsonArrayWriter(this._vw); t.elemTypes.forEach(elem => w2.writeTypeAsValue(elem, parentStructTypes)); this.write(w2.array); break; @@ -262,7 +261,7 @@ export class JsonArrayWriter { const desc = t.desc; this.writeKind(t.kind); this.write(t.name); - const fieldWriter = new JsonArrayWriter(this._ds); + const fieldWriter = new JsonArrayWriter(this._vw); desc.fields.forEach(field => { fieldWriter.write(field.name); fieldWriter.writeTypeAsTag(field.type, parentStructTypes); @@ -290,8 +289,8 @@ export class JsonArrayWriter { } } -function encodeEmbeddedNomsValue(v: valueOrPrimitive, t: Type, ds: ?DataStore): Chunk { - const w = new JsonArrayWriter(ds); +function encodeEmbeddedNomsValue(v: valueOrPrimitive, t: Type, vw: ?ValueWriter): Chunk { + const w = new JsonArrayWriter(vw); w.writeTopLevel(t, v); return Chunk.fromString(typedTag + JSON.stringify(w.array)); } @@ -309,7 +308,11 @@ function encodeTopLevelBlob(sequence: BlobLeafSequence): Chunk { return new Chunk(data); } -export function encodeNomsValue(v: valueOrPrimitive, t: Type, ds: ?DataStore): Chunk { +interface ValueWriter { + writeValue(v: T, t: ?Type): RefValue +} + +export function encodeNomsValue(v: valueOrPrimitive, t: Type, vw: ?ValueWriter): Chunk { if (t.kind === Kind.Blob) { invariant(v instanceof NomsBlob || v instanceof IndexedSequence); const sequence = v instanceof NomsBlob ? v.sequence : v; @@ -318,7 +321,7 @@ export function encodeNomsValue(v: valueOrPrimitive, t: Type, ds: ?DataStore): C return encodeTopLevelBlob(sequence); } } - return encodeEmbeddedNomsValue(v, t, ds); + return encodeEmbeddedNomsValue(v, t, vw); } setEncodeNomsValue(encodeNomsValue); diff --git a/js/src/http-batch-store-test.js b/js/src/http-batch-store-test.js new file mode 100644 index 0000000000..1e32a7f543 --- /dev/null +++ b/js/src/http-batch-store-test.js @@ -0,0 +1,24 @@ +// @flow + +import {suite, test} from 'mocha'; +import Chunk from './chunk.js'; +import {assert} from 'chai'; +import {Delegate} from './http-batch-store.js'; +import {deserialize} from './chunk-serializer.js'; + +suite('HttpBatchStore', () => { + test('build write request', async () => { + const canned = new Set([Chunk.fromString('abc').ref, Chunk.fromString('def').ref]); + const reqs = [ + {c: Chunk.fromString('ghi'), hints: new Set()}, + {c: Chunk.fromString('wacka wack wack'), hints: canned}, + ]; + const d = new Delegate( + {getRefs: '', writeValue: '', root: ''}, + {method: 'POST', headers: {}}); + const body = d._buildWriteRequest(reqs); + const {hints, chunks} = deserialize(body); + assert.equal(2, chunks.length); + assert.equal(2, hints.length); + }); +}); \ No newline at end of file diff --git a/js/src/http-store.js b/js/src/http-batch-store.js similarity index 63% rename from js/src/http-store.js rename to js/src/http-batch-store.js index 89a0a53c08..d513c52526 100644 --- a/js/src/http-store.js +++ b/js/src/http-batch-store.js @@ -1,20 +1,19 @@ // @flow import Ref from './ref.js'; +import BatchStore from './batch-store.js'; +import type {UnsentReadMap, WriteRequest} from './batch-store.js'; import type {FetchOptions} from './fetch.js'; -import type {UnsentReadMap} from './remote-store.js'; -import type {WriteMap} from './remote-store.js'; -import {deserialize, serialize} from './chunk-serializer.js'; +import {deserializeChunks, serialize} from './chunk-serializer.js'; import {emptyChunk} from './chunk.js'; import {fetchArrayBuffer, fetchText} from './fetch.js'; -import {RemoteStore} from './remote-store.js'; const HTTP_STATUS_CONFLICT = 409; type RpcStrings = { getRefs: string, - postRefs: string, root: string, + writeValue: string, }; const readBatchOptions = { @@ -24,31 +23,18 @@ const readBatchOptions = { }, }; -export default class HttpStore extends RemoteStore { +export default class HttpBatchStore extends BatchStore { _rpc: RpcStrings; - _rootOptions: FetchOptions; - constructor(url: string, maxReads: number = 3, maxWrites: number = 3, - fetchOptions: FetchOptions = {}) { + constructor(url: string, maxReads: number = 5, fetchOptions: FetchOptions = {}) { const rpc = { getRefs: url + '/getRefs/', - postRefs: url + '/postRefs/', root: url + '/root/', + writeValue: url + '/writeValue/', }; - const mergedOptions = mergeOptions(readBatchOptions, fetchOptions); - super(maxReads, maxWrites, new Delegate(rpc, mergedOptions)); + super(maxReads, new Delegate(rpc, fetchOptions)); this._rpc = rpc; - this._rootOptions = fetchOptions; - } - - async getRoot(): Promise { - const refStr = await fetchText(this._rpc.root, this._rootOptions); - return Ref.parse(refStr); - } - - has(ref: Ref): Promise { // eslint-disable-line no-unused-vars - throw new Error('not implemented'); } } @@ -57,13 +43,15 @@ function mergeOptions(baseOpts: FetchOptions, opts: FetchOptions): FetchOptions return Object.assign({}, opts, baseOpts, {headers: hdrs}); } -class Delegate { +export class Delegate { _rpc: RpcStrings; _readBatchOptions: FetchOptions; + _rootOptions: FetchOptions; - constructor(rpc: RpcStrings, readBatchOptions: FetchOptions) { + constructor(rpc: RpcStrings, fetchOptions: FetchOptions) { this._rpc = rpc; - this._readBatchOptions = readBatchOptions; + this._rootOptions = fetchOptions; + this._readBatchOptions = mergeOptions(readBatchOptions, fetchOptions); } async readBatch(reqs: UnsentReadMap): Promise { @@ -72,7 +60,7 @@ class Delegate { const opts = Object.assign(this._readBatchOptions, {body: body}); const buf = await fetchArrayBuffer(this._rpc.getRefs, opts); - const chunks = deserialize(buf); + const chunks = deserializeChunks(buf); // Return success chunks.forEach(chunk => { @@ -85,10 +73,23 @@ class Delegate { Object.keys(reqs).forEach(refStr => reqs[refStr](emptyChunk)); } - async writeBatch(reqs: WriteMap): Promise { - const chunks = Object.keys(reqs).map(refStr => reqs[refStr]); - const body = serialize(chunks); - await fetchText(this._rpc.postRefs, {method: 'POST', body}); + async writeBatch(reqs: Array): Promise { + const body = this._buildWriteRequest(reqs); + await fetchText(this._rpc.writeValue, {method: 'POST', body}); + } + + _buildWriteRequest(reqs: Array): ArrayBuffer { + let hints = new Set(); + const chunks = reqs.map(writeReq => { + writeReq.hints.forEach(hint => { hints = hints.add(hint); }); + return writeReq.c; + }); + return serialize(hints, chunks); + } + + async getRoot(): Promise { + const refStr = await fetchText(this._rpc.root, this._rootOptions); + return Ref.parse(refStr); } async updateRoot(current: Ref, last: Ref): Promise { diff --git a/js/src/list-test.js b/js/src/list-test.js index 675b850c1f..1ea5faae14 100644 --- a/js/src/list-test.js +++ b/js/src/list-test.js @@ -4,7 +4,7 @@ import {assert} from 'chai'; import {suite, test} from 'mocha'; import DataStore from './data-store.js'; -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import RefValue from './ref-value.js'; import {newStruct} from './struct.js'; import {calcSplices} from './edit-distance.js'; @@ -152,8 +152,7 @@ suite('BuildList', () => { }); test('LONG: write, read, modify, read', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const nums = firstNNumbers(testListSize); const tr = makeListType(numberType); @@ -173,8 +172,7 @@ suite('BuildList', () => { suite('ListLeafSequence', () => { test('isEmpty', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(stringType); const newList = items => new NomsList(tr, new ListLeafSequence(ds, tr, items)); assert.isTrue(newList([]).isEmpty()); @@ -182,8 +180,7 @@ suite('ListLeafSequence', () => { }); test('get', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(stringType); const l = new NomsList(tr, new ListLeafSequence(ds, tr, ['z', 'x', 'a', 'b'])); assert.strictEqual('z', await l.get(0)); @@ -193,8 +190,7 @@ suite('ListLeafSequence', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(numberType); const l = new NomsList(tr, new ListLeafSequence(ds, tr, [4, 2, 10, 16])); @@ -204,8 +200,7 @@ suite('ListLeafSequence', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(numberType); const test = async items => { @@ -220,8 +215,7 @@ suite('ListLeafSequence', () => { }); test('iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(numberType); const test = async items => { @@ -239,8 +233,7 @@ suite('ListLeafSequence', () => { }); function testChunks(elemType: Type) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(elemType); const r1 = ds.writeValue('x'); const r2 = ds.writeValue('a'); @@ -263,8 +256,7 @@ suite('ListLeafSequence', () => { suite('CompoundList', () => { function build(): NomsList { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeListType(stringType); const l1 = new NomsList(tr, new ListLeafSequence(ds, tr, ['a', 'b'])); const r1 = ds.writeValue(l1).targetRef; diff --git a/js/src/list.js b/js/src/list.js index e0334064ab..e0a054f9c5 100644 --- a/js/src/list.js +++ b/js/src/list.js @@ -2,7 +2,7 @@ import BuzHashBoundaryChecker from './buzhash-boundary-checker.js'; import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import type {Splice} from './edit-distance.js'; import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars import type {AsyncIterator} from './async-iterator.js'; @@ -22,9 +22,9 @@ import {listOfValueType, Type} from './type.js'; const listWindowSize = 64; const listPattern = ((1 << 6) | 0) - 1; -function newListLeafChunkFn(t: Type, ds: ?DataStore = null): makeChunkFn { +function newListLeafChunkFn(t: Type, vr: ?ValueReader = null): makeChunkFn { return (items: Array) => { - const listLeaf = new ListLeafSequence(ds, t, items); + const listLeaf = new ListLeafSequence(vr, t, items); const mt = new MetaTuple(listLeaf, items.length, items.length); return [mt, listLeaf]; }; @@ -54,10 +54,10 @@ export class NomsList extends Collection { async splice(idx: number, deleteCount: number, ...insert: Array): Promise> { const cursor = await this.sequence.newCursorAt(idx); - const ds = this.sequence.ds; + const vr = this.sequence.vr; const type = this.type; - const seq = await chunkSequence(cursor, insert, deleteCount, newListLeafChunkFn(type, ds), - newIndexedMetaSequenceChunkFn(type, ds), + const seq = await chunkSequence(cursor, insert, deleteCount, newListLeafChunkFn(type, vr), + newIndexedMetaSequenceChunkFn(type, vr), newListLeafBoundaryChecker(type), newIndexedMetaSequenceBoundaryChecker); invariant(seq instanceof IndexedSequence); diff --git a/js/src/map-test.js b/js/src/map-test.js index 9c8625b165..a8bfd01705 100644 --- a/js/src/map-test.js +++ b/js/src/map-test.js @@ -6,6 +6,8 @@ import {suite, test} from 'mocha'; import DataStore from './data-store'; import MemoryStore from './memory-store.js'; import RefValue from './ref-value.js'; +import BatchStore from './batch-store.js'; +import {BatchStoreAdaptorDelegate, makeTestingBatchStore} from './batch-store-adaptor.js'; import {newStruct} from './struct.js'; import { boolType, @@ -164,8 +166,7 @@ suite('BuildMap', () => { }); test('LONG: write, read, modify, read', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const kvs = []; for (let i = 0; i < testMapSize; i++) { @@ -194,8 +195,7 @@ suite('BuildMap', () => { suite('MapLeaf', () => { test('isEmpty/size', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, boolType); const newMap = entries => new NomsMap(tr, new MapLeafSequence(ds, tr, entries)); let m = newMap([]); @@ -207,8 +207,7 @@ suite('MapLeaf', () => { }); test('has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, boolType); const m = new NomsMap(tr, new MapLeafSequence(ds, tr, [{key: 'a', value: false}, {key:'k', value:true}])); @@ -219,8 +218,7 @@ suite('MapLeaf', () => { }); test('first/last/get', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, numberType); const m = new NomsMap(tr, new MapLeafSequence(ds, tr, [{key: 'a', value: 4}, {key:'k', value:8}])); @@ -235,8 +233,7 @@ suite('MapLeaf', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, numberType); const m = new NomsMap(tr, new MapLeafSequence(ds, tr, [{key: 'a', value: 4}, {key:'k', value:8}])); @@ -247,8 +244,7 @@ suite('MapLeaf', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, numberType); const test = async entries => { @@ -263,8 +259,7 @@ suite('MapLeaf', () => { }); test('LONG: iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(stringType, numberType); const build = entries => new NomsMap(tr, new MapLeafSequence(ds, tr, entries)); @@ -288,8 +283,7 @@ suite('MapLeaf', () => { }); function testChunks(keyType: Type, valueType: Type) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeMapType(keyType, valueType); const r1 = ds.writeValue('x'); const r2 = ds.writeValue(true); @@ -343,16 +337,14 @@ suite('CompoundMap', () => { } test('isEmpty/size', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); assert.isFalse(c.isEmpty()); assert.strictEqual(8, c.size); }); test('get', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); assert.strictEqual(false, await c.get('a')); @@ -372,8 +364,7 @@ suite('CompoundMap', () => { }); test('first/last/has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c, m1, m2] = build(ds); assert.deepEqual(['a', false], await c.first()); @@ -400,8 +391,7 @@ suite('CompoundMap', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const kv = []; @@ -411,8 +401,7 @@ suite('CompoundMap', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const expected = [{key: 'a', value: false}, {key: 'b', value: false}, {key: 'e', value: true}, {key: 'f', value: true}, {key: 'h', value: false}, {key: 'i', value: true}, @@ -422,8 +411,7 @@ suite('CompoundMap', () => { }); test('LONG: iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const entries = [{key: 'a', value: false}, {key: 'b', value: false}, {key: 'e', value: true}, {key: 'f', value: true}, {key: 'h', value: false}, {key: 'i', value: true}, @@ -447,8 +435,7 @@ suite('CompoundMap', () => { }); test('iterator return', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const iter = c.iterator(); const values = []; @@ -464,8 +451,7 @@ suite('CompoundMap', () => { }); test('iterator return parallel', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); const iter = c.iterator(); const values = await Promise.all([iter.next(), iter.next(), iter.return(), iter.next()]); @@ -476,8 +462,7 @@ suite('CompoundMap', () => { }); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const [c] = build(ds); assert.strictEqual(2, c.chunks.length); }); @@ -515,7 +500,7 @@ suite('CompoundMap', () => { } const ms = new CountingMemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(new BatchStore(3, new BatchStoreAdaptorDelegate(ms))); [m1, m2] = await Promise.all([m1, m2].map(s => ds.readValue(ds.writeValue(s).targetRef))); assert.deepEqual([[], [], []], await m1.diff(m1)); @@ -565,8 +550,7 @@ suite('CompoundMap', () => { test('LONG: random map diff 0.1/0.9/0', () => testRandomDiff(randomMapSize, 0.1, 0.9, 0)); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const m = build(ds)[1]; const chunks = m.chunks; const sequence = m.sequence; diff --git a/js/src/map.js b/js/src/map.js index 75004b486d..b0cfd3ccba 100644 --- a/js/src/map.js +++ b/js/src/map.js @@ -2,7 +2,7 @@ import BuzHashBoundaryChecker from './buzhash-boundary-checker.js'; import RefValue from './ref-value.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js'; import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars import type {AsyncIterator} from './async-iterator.js'; @@ -28,9 +28,9 @@ export type MapEntry = { const mapWindowSize = 1; const mapPattern = ((1 << 6) | 0) - 1; -function newMapLeafChunkFn(t: Type, ds: ?DataStore = null): makeChunkFn { +function newMapLeafChunkFn(t: Type, vr: ?ValueReader = null): makeChunkFn { return (items: Array) => { - const mapLeaf = new MapLeafSequence(ds, t, items); + const mapLeaf = new MapLeafSequence(vr, t, items); let indexValue: ?valueOrPrimitive = null; if (items.length > 0) { @@ -150,9 +150,9 @@ export class NomsMap extends Collectio async _splice(cursor: OrderedSequenceCursor, insert: Array, remove: number): Promise> { const type = this.type; - const ds = this.sequence.ds; - const seq = await chunkSequence(cursor, insert, remove, newMapLeafChunkFn(type, ds), - newOrderedMetaSequenceChunkFn(type, ds), + const vr = this.sequence.vr; + const seq = await chunkSequence(cursor, insert, remove, newMapLeafChunkFn(type, vr), + newOrderedMetaSequenceChunkFn(type, vr), newMapLeafBoundaryChecker(type), newOrderedMetaSequenceBoundaryChecker); invariant(seq instanceof OrderedSequence); diff --git a/js/src/meta-sequence.js b/js/src/meta-sequence.js index f86b44b1c1..0951f86db1 100644 --- a/js/src/meta-sequence.js +++ b/js/src/meta-sequence.js @@ -3,7 +3,7 @@ import BuzHashBoundaryChecker from './buzhash-boundary-checker.js'; import {default as Ref, sha1Size} from './ref.js'; import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import type {valueOrPrimitive} from './value.js'; // eslint-disable-line no-unused-vars import type {Collection} from './collection.js'; import {CompoundDesc, makeCompoundType, makeRefType, numberType, valueType} from './type.js'; @@ -44,13 +44,13 @@ export class MetaTuple { return this._sequenceOrRef instanceof Sequence ? this._sequenceOrRef : null; } - getSequence(ds: ?DataStore): Promise { + getSequence(vr: ?ValueReader): Promise { if (this._sequenceOrRef instanceof Sequence) { return Promise.resolve(this._sequenceOrRef); } else { const ref = this._sequenceOrRef; - invariant(ds && ref instanceof Ref); - return ds.readValue(ref).then((c: Collection) => c.sequence); + invariant(vr && ref instanceof Ref); + return vr.readValue(ref).then((c: Collection) => c.sequence); } } } @@ -58,8 +58,8 @@ export class MetaTuple { export class IndexedMetaSequence extends IndexedSequence> { _offsets: Array; - constructor(ds: ?DataStore, type: Type, items: Array>) { - super(ds, type, items); + constructor(vr: ?ValueReader, type: Type, items: Array>) { + super(vr, type, items); let cum = 0; this._offsets = this.items.map(i => { cum += i.value; @@ -111,7 +111,7 @@ export class IndexedMetaSequence extends IndexedSequence> { } const mt = this.items[idx]; - return mt.getSequence(this.ds); + return mt.getSequence(this.vr); } // Returns the sequences pointed to by all items[i], s.t. start <= i < end, and returns the @@ -120,14 +120,14 @@ export class IndexedMetaSequence extends IndexedSequence> { Promise { const childrenP = []; for (let i = start; i < start + length; i++) { - childrenP.push(this.items[i].getSequence(this.ds)); + childrenP.push(this.items[i].getSequence(this.vr)); } return Promise.all(childrenP).then(children => { const items = []; children.forEach(child => items.push(...child.items)); - return children[0].isMeta ? new IndexedMetaSequence(this.ds, this.type, items) - : new IndexedSequence(this.ds, this.type, items); + return children[0].isMeta ? new IndexedMetaSequence(this.vr, this.type, items) + : new IndexedSequence(this.vr, this.type, items); }); } @@ -139,8 +139,8 @@ export class IndexedMetaSequence extends IndexedSequence> { export class OrderedMetaSequence extends OrderedSequence> { _numLeaves: number; - constructor(ds: ?DataStore, type: Type, items: Array>) { - super(ds, type, items); + constructor(vr: ?ValueReader, type: Type, items: Array>) { + super(vr, type, items); this._numLeaves = items.reduce((l, mt) => l + mt.numLeaves, 0); } @@ -162,7 +162,7 @@ export class OrderedMetaSequence extends OrderedSequence extends OrderedSequence): +export function newMetaSequenceFromData(vr: ValueReader, type: Type, tuples: Array): MetaSequence { switch (type.kind) { case Kind.Map: case Kind.Set: - return new OrderedMetaSequence(ds, type, tuples); + return new OrderedMetaSequence(vr, type, tuples); case Kind.Blob: case Kind.List: - return new IndexedMetaSequence(ds, type, tuples); + return new IndexedMetaSequence(vr, type, tuples); default: throw new Error('Not reached'); } @@ -211,10 +211,10 @@ export function indexTypeForMetaSequence(t: Type): Type { throw new Error('Not reached'); } -export function newOrderedMetaSequenceChunkFn(t: Type, ds: ?DataStore = null): makeChunkFn { +export function newOrderedMetaSequenceChunkFn(t: Type, vr: ?ValueReader = null): makeChunkFn { return (tuples: Array) => { const numLeaves = tuples.reduce((l, mt) => l + mt.numLeaves, 0); - const meta = new OrderedMetaSequence(ds, t, tuples); + const meta = new OrderedMetaSequence(vr, t, tuples); const lastValue = tuples[tuples.length - 1].value; return [new MetaTuple(meta, lastValue, numLeaves), meta]; }; @@ -230,13 +230,13 @@ export function newOrderedMetaSequenceBoundaryChecker(): BoundaryChecker) => { const sum = tuples.reduce((l, mt) => { invariant(mt.value === mt.numLeaves); return l + mt.value; }, 0); - const meta = new IndexedMetaSequence(ds, t, tuples); + const meta = new IndexedMetaSequence(vr, t, tuples); return [new MetaTuple(meta, sum, sum), meta]; }; } diff --git a/js/src/noms.js b/js/src/noms.js index 7d22c2b860..99b1ae7df7 100644 --- a/js/src/noms.js +++ b/js/src/noms.js @@ -6,7 +6,7 @@ export {default as Dataset} from './dataset.js'; export {newBlob, NomsBlob, BlobReader, BlobWriter} from './blob.js'; export {decodeNomsValue} from './decode.js'; export {default as Chunk} from './chunk.js'; -export {default as HttpStore} from './http-store.js'; +export {default as HttpBatchStore} from './http-batch-store.js'; export {default as MemoryStore} from './memory-store.js'; export {default as Ref, emptyRef} from './ref.js'; export {default as RefValue} from './ref-value.js'; diff --git a/js/src/sequence.js b/js/src/sequence.js index 4fdcb54ff5..3d20c9d568 100644 --- a/js/src/sequence.js +++ b/js/src/sequence.js @@ -1,6 +1,6 @@ // @flow -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import {invariant, notNull} from './assert.js'; import {AsyncIterator} from './async-iterator.js'; import type {AsyncIteratorResult} from './async-iterator.js'; @@ -9,13 +9,13 @@ import type {Type} from './type.js'; import {Value} from './value.js'; export class Sequence extends Value { - ds: ?DataStore; + vr: ?ValueReader; _type: Type; _items: Array; - constructor(ds: ?DataStore, type: Type, items: Array) { + constructor(vr: ?ValueReader, type: Type, items: Array) { super(); - this.ds = ds; + this.vr = vr; this._type = type; this._items = items; } diff --git a/js/src/set-test.js b/js/src/set-test.js index 0b1d0ad0ab..70d40f80d8 100644 --- a/js/src/set-test.js +++ b/js/src/set-test.js @@ -7,6 +7,8 @@ import Chunk from './chunk.js'; import DataStore from './data-store.js'; import MemoryStore from './memory-store.js'; import RefValue from './ref-value.js'; +import BatchStore from './batch-store.js'; +import {BatchStoreAdaptorDelegate, makeTestingBatchStore} from './batch-store-adaptor.js'; import {newStruct} from './struct.js'; import { boolType, @@ -134,8 +136,7 @@ suite('BuildSet', () => { }); test('LONG: write, read, modify, read', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const nums = firstNNumbers(testSetSize); const tr = makeSetType(numberType); @@ -159,8 +160,7 @@ suite('BuildSet', () => { suite('SetLeaf', () => { test('isEmpty/size', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const newSet = items => new NomsSet(tr, new SetLeafSequence(ds, tr, items)); let s = newSet([]); @@ -172,8 +172,7 @@ suite('SetLeaf', () => { }); test('first/last/has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const s = new NomsSet(tr, new SetLeafSequence(ds, tr, ['a', 'k'])); @@ -187,8 +186,7 @@ suite('SetLeaf', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const m = new NomsSet(tr, new SetLeafSequence(ds, tr, ['a', 'b'])); @@ -198,8 +196,7 @@ suite('SetLeaf', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const test = async items => { @@ -214,8 +211,7 @@ suite('SetLeaf', () => { }); test('LONG: iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(stringType); const build = items => new NomsSet(tr, new SetLeafSequence(ds, tr, items)); @@ -233,8 +229,7 @@ suite('SetLeaf', () => { }); function testChunks(elemType: Type) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(elemType); const r1 = ds.writeValue('x'); const r2 = ds.writeValue('a'); @@ -284,16 +279,14 @@ suite('CompoundSet', () => { } test('isEmpty/size', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); assert.isFalse(c.isEmpty()); assert.strictEqual(8, c.size); }); test('first/last/has', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); assert.strictEqual('a', await c.first()); assert.strictEqual('n', await c.last()); @@ -314,8 +307,7 @@ suite('CompoundSet', () => { }); test('forEach', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); const values = []; await c.forEach((k) => { values.push(k); }); @@ -323,8 +315,7 @@ suite('CompoundSet', () => { }); test('iterator', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const values = ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']; const c = build(ds, values); assert.deepEqual(values, await flatten(c.iterator())); @@ -332,8 +323,7 @@ suite('CompoundSet', () => { }); test('LONG: iteratorAt', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const values = ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']; const c = build(ds, values); const offsets = { @@ -355,8 +345,7 @@ suite('CompoundSet', () => { }); test('iterator return', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const values = ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']; const c = build(ds, values); const iter = c.iterator(); @@ -371,8 +360,7 @@ suite('CompoundSet', () => { }); test('iterator return parallel', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); const iter = c.iterator(); const values = await Promise.all([iter.next(), iter.next(), iter.return(), iter.next()]); @@ -382,23 +370,20 @@ suite('CompoundSet', () => { }); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); assert.strictEqual(2, c.chunks.length); }); test('map', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); const values = await c.map((k) => k + '*'); assert.deepEqual(['a*', 'b*', 'e*', 'f*', 'h*', 'i*', 'm*', 'n*'], values); }); test('map async', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); const values = await c.map((k) => Promise.resolve(k + '*')); assert.deepEqual(['a*', 'b*', 'e*', 'f*', 'h*', 'i*', 'm*', 'n*'], values); @@ -416,8 +401,7 @@ suite('CompoundSet', () => { } test('advanceTo', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const c = build(ds, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n']); @@ -459,8 +443,7 @@ suite('CompoundSet', () => { }); async function testIntersect(expect: Array, seqs: Array>) { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const first = build(ds, seqs[0]); const sets:Array = []; @@ -489,8 +472,7 @@ suite('CompoundSet', () => { }); test('iterator at 0', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(numberType); const test = async (expected, items) => { @@ -513,8 +495,7 @@ suite('CompoundSet', () => { }); test('LONG: canned set diff', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const tr = makeSetType(numberType); const s1 = await newSet( firstNNumbers(testSetSize), tr).then(s => ds.readValue(ds.writeValue(s).targetRef)); @@ -568,7 +549,7 @@ suite('CompoundSet', () => { } const ms = new CountingMemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(new BatchStore(3, new BatchStoreAdaptorDelegate(ms))); [s1, s2] = await Promise.all([s1, s2].map(s => ds.readValue(ds.writeValue(s).targetRef))); assert.deepEqual([[], []], await s1.diff(s1)); @@ -607,8 +588,7 @@ suite('CompoundSet', () => { test('LONG: random set diff 0.1/0.9', () => testRandomDiff(randomSetSize, 0.1, 0.9)); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const s = build(ds, ['a', 'b', 'c', 'd']); const chunks = s.chunks; const sequence = s.sequence; diff --git a/js/src/set.js b/js/src/set.js index 47d3284b1c..ec9aebc73c 100644 --- a/js/src/set.js +++ b/js/src/set.js @@ -2,7 +2,7 @@ import BuzHashBoundaryChecker from './buzhash-boundary-checker.js'; import RefValue from './ref-value.js'; -import type DataStore from './data-store.js'; +import type {ValueReader} from './decode.js'; import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js'; import type {valueOrPrimitive, Value} from './value.js'; // eslint-disable-line no-unused-vars import {AsyncIterator} from './async-iterator.js'; @@ -24,9 +24,9 @@ import {getValueChunks} from './sequence.js'; const setWindowSize = 1; const setPattern = ((1 << 6) | 0) - 1; -function newSetLeafChunkFn(t: Type, ds: ?DataStore = null): makeChunkFn { +function newSetLeafChunkFn(t: Type, vr: ?ValueReader = null): makeChunkFn { return (items: Array) => { - const setLeaf = new SetLeafSequence(ds, t, items); + const setLeaf = new SetLeafSequence(vr, t, items); let indexValue: ?(T | RefValue) = null; if (items.length > 0) { @@ -105,9 +105,9 @@ export class NomsSet extends Collection { async _splice(cursor: OrderedSequenceCursor, insert: Array, remove: number): Promise> { const type = this.type; - const ds = this.sequence.ds; - const seq = await chunkSequence(cursor, insert, remove, newSetLeafChunkFn(type, ds), - newOrderedMetaSequenceChunkFn(type, ds), + const vr = this.sequence.vr; + const seq = await chunkSequence(cursor, insert, remove, newSetLeafChunkFn(type, vr), + newOrderedMetaSequenceChunkFn(type, vr), newSetLeafBoundaryChecker(type), newOrderedMetaSequenceBoundaryChecker); invariant(seq instanceof OrderedSequence); diff --git a/js/src/specs-test.js b/js/src/specs-test.js index 95dffef8ce..7bf3b6daea 100644 --- a/js/src/specs-test.js +++ b/js/src/specs-test.js @@ -1,10 +1,10 @@ // @flow import {invariant} from './assert.js'; +import BatchStoreAdaptor from './batch-store-adaptor.js'; import Dataset from './dataset.js'; import DataStore from './data-store.js'; -import HttpStore from './http-store.js'; -import MemoryStore from './memory-store.js'; +import HttpBatchStore from './http-batch-store.js'; import Ref from './ref.js'; import {DataStoreSpec, DatasetSpec, RefSpec, parseObjectSpec} from './specs.js'; import {assert} from 'chai'; @@ -20,7 +20,7 @@ suite('Specs', () => { assert.equal(spec.scheme, 'mem'); assert.equal(spec.path, ''); assert.instanceOf(spec.store(), DataStore); - assert.instanceOf(spec.store()._cs, MemoryStore); + assert.instanceOf(spec.store()._bs, BatchStoreAdaptor); spec = DataStoreSpec.parse('http://foo'); invariant(spec); @@ -28,7 +28,7 @@ suite('Specs', () => { assert.equal(spec.scheme, 'http'); assert.equal(spec.path, '//foo'); assert.instanceOf(spec.store(), DataStore); - assert.instanceOf(spec.store()._cs, HttpStore); + assert.instanceOf(spec.store()._bs, HttpBatchStore); spec = DataStoreSpec.parse('https://foo'); invariant(spec); @@ -48,7 +48,7 @@ suite('Specs', () => { assert.equal(spec.store.path, ''); let ds = spec.set(); assert.instanceOf(ds, Dataset); - assert.instanceOf(ds.store._cs, MemoryStore); + assert.instanceOf(ds.store._bs, BatchStoreAdaptor); spec = DatasetSpec.parse('http://localhost:8000/foo:ds'); invariant(spec); @@ -57,7 +57,7 @@ suite('Specs', () => { assert.equal(spec.store.path, '//localhost:8000/foo'); ds = spec.set(); assert.instanceOf(ds, Dataset); - assert.instanceOf(ds.store._cs, HttpStore); + assert.instanceOf(ds.store._bs, HttpBatchStore); }); test('RefSpec', () => { diff --git a/js/src/specs.js b/js/src/specs.js index 9a1526a917..405c23daeb 100644 --- a/js/src/specs.js +++ b/js/src/specs.js @@ -1,8 +1,9 @@ // @flow +import BatchStoreAdaptor from './batch-store-adaptor.js'; import Dataset from './dataset.js'; import DataStore from './data-store.js'; -import HttpStore from './http-store.js'; +import HttpBatchStore from './http-batch-store.js'; import MemoryStore from './memory-store.js'; import Ref from './ref.js'; @@ -48,10 +49,10 @@ export class DataStoreSpec { // Constructs a new DataStore based on the parsed spec. store(): DataStore { if (this.scheme === 'mem') { - return new DataStore(new MemoryStore()); + return new DataStore(new BatchStoreAdaptor(new MemoryStore())); } if (this.scheme === 'http') { - return new DataStore(new HttpStore(`${this.scheme}:${this.path}`)); + return new DataStore(new HttpBatchStore(`${this.scheme}:${this.path}`)); } throw new Error('Unreached'); } diff --git a/js/src/struct-test.js b/js/src/struct-test.js index c578109096..55005f379b 100644 --- a/js/src/struct-test.js +++ b/js/src/struct-test.js @@ -1,6 +1,6 @@ // @flow -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {default as Struct, newStruct, StructMirror, createStructClass} from './struct.js'; import {assert} from 'chai'; import { @@ -32,8 +32,7 @@ suite('Struct', () => { }); test('chunks', () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const bt = boolType; const refOfBoolType = makeRefType(bt); diff --git a/js/src/type-test.js b/js/src/type-test.js index 4a6230d332..bad45b4cf5 100644 --- a/js/src/type-test.js +++ b/js/src/type-test.js @@ -1,6 +1,6 @@ // @flow -import MemoryStore from './memory-store.js'; +import {makeTestingBatchStore} from './batch-store-adaptor.js'; import {assert} from 'chai'; import { boolType, @@ -18,8 +18,7 @@ import DataStore from './data-store.js'; suite('Type', () => { test('types', async () => { - const ms = new MemoryStore(); - const ds = new DataStore(ms); + const ds = new DataStore(makeTestingBatchStore()); const mapType = makeMapType(stringType, numberType); const setType = makeSetType(stringType); diff --git a/js/src/value-store-test.js b/js/src/value-store-test.js new file mode 100644 index 0000000000..3570dbacb4 --- /dev/null +++ b/js/src/value-store-test.js @@ -0,0 +1,124 @@ +// @flow + +import {suite, test} from 'mocha'; +import {assert} from 'chai'; +import MemoryStore from './memory-store.js'; +import type {ChunkStore} from './chunk-store.js'; +import BatchStore from './batch-store.js'; +import {BatchStoreAdaptorDelegate} from './batch-store-adaptor.js'; +import ValueStore from './value-store.js'; +import {newList} from './list.js'; +import {stringType} from './type.js'; +import {encodeNomsValue} from './encode.js'; + +export class FakeBatchStore extends BatchStore { + constructor(cs: ChunkStore) { + super(3, new BatchStoreAdaptorDelegate(cs)); + } +} + +suite('ValueStore', () => { + test('readValue', async () => { + const ms = new MemoryStore(); + const vs = new ValueStore(new FakeBatchStore(ms)); + const input = 'abc'; + + const c = encodeNomsValue(input, stringType); + const v1 = await vs.readValue(c.ref); + assert.equal(null, v1); + + ms.put(c); + const v2 = await vs.readValue(c.ref); + assert.equal('abc', v2); + }); + + test('writeValue primitives', async () => { + const vs = new ValueStore(new FakeBatchStore(new MemoryStore())); + + const r1 = vs.writeValue('hello').targetRef; + const r2 = vs.writeValue(false).targetRef; + const r3 = vs.writeValue(2).targetRef; + + const v1 = await vs.readValue(r1); + assert.equal('hello', v1); + const v2 = await vs.readValue(r2); + assert.equal(false, v2); + const v3 = await vs.readValue(r3); + assert.equal(2, v3); + }); + + test('writeValue rejects invalid', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + let vs = new ValueStore(bs); + const r = vs.writeValue('hello'); + vs.flush().then(() => { vs.close(); }); + + vs = new ValueStore(bs); + const l = await newList([r]); + let ex; + try { + await vs.writeValue(l); + } catch (e) { + ex = e; + } + assert.instanceOf(ex, Error); + }); + + test('write coalescing', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + const vs = new ValueStore(bs, 1e6); + + const r1 = vs.writeValue('hello').targetRef; + (bs: any).schedulePut = () => { assert.fail('unreachable'); }; + const r2 = vs.writeValue('hello').targetRef; + assert.isTrue(r1.equals(r2)); + }); + + test('read caching', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + const vs = new ValueStore(bs, 1e6); + + const r1 = vs.writeValue('hello').targetRef; + const v1 = await vs.readValue(r1); + assert.equal(v1, 'hello'); + (bs: any).get = () => { throw new Error(); }; + const v2 = await vs.readValue(r1); + assert.equal(v1, v2); + }); + + test('caching eviction', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + const vs = new ValueStore(bs, 15); + + const r1 = vs.writeValue('hello').targetRef; + const r2 = vs.writeValue('world').targetRef; + + // Prime the cache + const v1 = await vs.readValue(r1); + assert.equal(v1, 'hello'); + // Evict v1 from the cache + const v2 = await vs.readValue(r2); + assert.equal(v2, 'world'); + + (bs: any).get = () => { throw new Error(); }; + let ex; + try { + await vs.readValue(r1); + } catch (e) { + ex = e; + } + assert.instanceOf(ex, Error); + }); + + test('hints on cache', async () => { + const bs = new FakeBatchStore(new MemoryStore()); + const vs = new ValueStore(bs, 15); + + const l = await newList([vs.writeValue(1), vs.writeValue(2)]); + const r = vs.writeValue(l); + // await vs.flush(); + + const v = await vs.readValue(r.targetRef); + assert.isTrue(l.equals(v)); + }); +}); diff --git a/js/src/value-store.js b/js/src/value-store.js new file mode 100644 index 0000000000..87e1824157 --- /dev/null +++ b/js/src/value-store.js @@ -0,0 +1,248 @@ +// @flow + +import Chunk from './chunk.js'; +import {default as Ref, emptyRef} from './ref.js'; +import {default as RefValue, refValueFromValue} from './ref-value.js'; +import type BatchStore from './batch-store.js'; +import type {valueOrPrimitive} from './value.js'; +import { + getTypeOfValue, + Type, + valueType, +} from './type.js'; +import {Kind} from './noms-kind.js'; +import {Value} from './value.js'; +import {decodeNomsValue} from './decode.js'; +import {invariant, notNull} from './assert.js'; +import {encodeNomsValue} from './encode.js'; +import {describeType, describeTypeOfValue} from './encode-human-readable.js'; + +export interface Cache { // eslint-disable-line no-undef + entry(ref: Ref): ?CacheEntry; // eslint-disable-line no-undef + get(ref: Ref): ?T; // eslint-disable-line no-undef + add(ref: Ref, size: number, value: T): void; // eslint-disable-line no-undef +} + +export default class ValueStore { + _bs: BatchStore; + _knownRefs: RefCache; + _valueCache: Cache; + + constructor(bs: BatchStore, cacheSize: number = 0) { + this._bs = bs; + this._knownRefs = new RefCache(); + this._valueCache = cacheSize > 0 ? new SizeCache(cacheSize) : new NoopCache(); + } + + // TODO: This should return Promise + async readValue(ref: Ref): Promise { + const entry = this._valueCache.entry(ref); + if (entry) { + return entry.value; + } + const chunk: Chunk = await this._bs.get(ref); + if (chunk.isEmpty()) { + this._valueCache.add(ref, 0, null); + this._knownRefs.addIfNotPresent(ref, new RefCacheEntry(false)); + return null; + } + + const v = decodeNomsValue(chunk, this); + this._valueCache.add(ref, chunk.data.length, v); + this._knownRefs.cacheChunks(v, ref); + // ref is trivially a hint for v, so consider putting that in the cache. + // If we got to v by reading some higher-level chunk, this entry gets dropped on + // the floor because r already has a hint in the cache. If we later read some other + // chunk that references v, cacheChunks will overwrite this with a hint pointing to that chunk. + // If we don't do this, top-level Values that get read but not written -- such as the + // existing Head of a DataStore upon a Commit -- can be erroneously left out during a pull. + this._knownRefs.addIfNotPresent(ref, new RefCacheEntry(true, getTypeOfValue(v), ref)); + return v; + } + + writeValue(v: T): RefValue { + const t = getTypeOfValue(v); + const chunk = encodeNomsValue(v, t, this); + invariant(!chunk.isEmpty()); + const {ref} = chunk; + const refValue = refValueFromValue(v); + const entry = this._knownRefs.get(ref); + if (entry && entry.present) { + return refValue; + } + const hints = this._knownRefs.checkChunksInCache(v); + this._bs.schedulePut(chunk, hints); + this._knownRefs.add(ref, new RefCacheEntry(true, t)); + return refValue; + } + + async flush(): Promise { + return this._bs.flush(); + } + + close() { + this._bs.close(); + } +} + +export class CacheEntry { + size: number; + value: ?T; + + constructor(size: number, value: ?T) { + this.size = size; + this.value = value; + } + + get present(): boolean { + return this.value !== null; + } +} + +/** + * This uses a Map as an LRU cache. It uses the behavior that iteration of keys in a Map is done in + * insertion order and any time a value is checked it is taken out and reinserted which puts it last + * in the iteration. + */ +export class SizeCache { + _size: number; + _maxSize: number; + _cache: Map>; + + constructor(size: number) { + this._maxSize = size; + this._cache = new Map(); + this._size = 0; + } + + entry(ref: Ref): ?CacheEntry { + const key = ref.toString(); + const entry = this._cache.get(key); + if (!entry) { + return undefined; + } + this._cache.delete(key); + this._cache.set(key, entry); + return entry; + } + + get(ref: Ref): ?T { + const entry = this.entry(ref); + return entry ? entry.value : undefined; + } + + add(ref: Ref, size: number, value: ?T) { + const key = ref.toString(); + if (this._cache.has(key)) { + this._cache.delete(key); + } else { + this._size += size; + } + this._cache.set(key, new CacheEntry(size, value)); + + if (this._size > this._maxSize) { + for (const [key, {size}] of this._cache) { + if (this._size <= this._maxSize) { + break; + } + this._cache.delete(key); + this._size -= size; + } + } + } +} + +export class NoopCache { + entry(ref: Ref): ?CacheEntry {} // eslint-disable-line no-unused-vars + + get(ref: Ref): ?T {} // eslint-disable-line no-unused-vars + + add(ref: Ref, size: number, value: T) {} // eslint-disable-line no-unused-vars +} + + +class RefCacheEntry { + present: boolean; + type: ?Type; + provenance: Ref; + + constructor(present: boolean = false, type: ?Type = null, provenance: Ref = emptyRef) { + invariant((!present && !type) || (present && type), `present = ${present}, type = ${type}`); + this.present = present; + this.type = type; + this.provenance = provenance; + } +} + +class RefCache { + _cache: Map; + + constructor() { + this._cache = new Map(); + } + + get(ref: Ref): ?RefCacheEntry { + return this._cache.get(ref.toString()); + } + + add(ref: Ref, entry: RefCacheEntry) { + this._cache.set(ref.toString(), entry); + } + + addIfNotPresent(ref: Ref, entry: RefCacheEntry) { + const refStr = ref.toString(); + const cur = this._cache.get(refStr); + if (!cur || cur.provenance.isEmpty()) { + this._cache.set(refStr, entry); + } + } + + cacheChunks(v: valueOrPrimitive, ref: Ref) { + if (v instanceof Value) { + v.chunks.forEach(reachable => { + const hash = reachable.targetRef; + const cur = this.get(hash); + if (!cur || cur.provenance.isEmpty() || cur.provenance.equals(hash)) { + this.add(hash, new RefCacheEntry(true, getTargetType(reachable), ref)); + } + }); + } + } + + checkChunksInCache(v: valueOrPrimitive): Set { + const hints = new Set(); + if (v instanceof Value) { + const chunks = v.chunks; + for (let i = 0; i < chunks.length; i++) { + const reachable = chunks[i]; + const entry = this.get(reachable.targetRef); + invariant(entry && entry.present, () => + `Value to write -- Type ${describeTypeOfValue(v)} -- contains ref ` + + `${reachable.targetRef.toString()}, which points to a non-existent Value.`); + if (!entry.provenance.isEmpty()) { + hints.add(entry.provenance); + } + + // BUG 1121 + // It's possible that entry.type will be simply 'Value', but that 'reachable' is actually a + // properly-typed object -- that is, a Ref to some specific Type. The Exp below would fail, + // though it's possible that the Type is actually correct. We wouldn't be able to verify + // without reading it, though, so we'll dig into this later. + const targetType = getTargetType(reachable); + if (targetType.equals(valueType)) { + continue; + } + const entryType = notNull(entry.type); + invariant(entryType.equals(targetType), () => + `Value to write contains ref ${reachable.targetRef.toString()}, which points to a ` + + `value of a different type: ${describeType(entryType)} != ${describeType(targetType)}`); + } + } + return hints; + } +} + +function getTargetType(refVal: RefValue): Type { + invariant(refVal.type.kind === Kind.Ref, refVal.type.kind); + return refVal.type.elemTypes[0]; +}