mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-05 02:45:34 -05:00
Code Issues Packages Projects Releases 159 Wiki Activity

Insert sequential (#1675)

Browse Source 
Insert sequential correctly into prolly tree
This commit is contained in:
Rafael Weinstein
2016-05-31 13:10:45 -07:00
parent 12f3c6a0ec
commit d3a3b72caa
9 changed files with 257 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/src/list-test.js
+22 -13
View File
@@ -8,10 +8,15 @@ import {assert} from 'chai';
import {suite, setup, teardown, test} from 'mocha';
import Database from './database.js';
import {makeTestingBatchStore} from './batch-store-adaptor.js';
import List, {ListWriter} from './list.js';
import Ref from './ref.js';
import {newStruct} from './struct.js';
import {MetaTuple, newListMetaSequence} from './meta-sequence.js';
import {calcSplices} from './edit-distance.js';
import {equals} from './compare.js';
import {invariant, notNull} from './assert.js';
import {makeTestingBatchStore} from './batch-store-adaptor.js';
import {newStruct} from './struct.js';
import {
makeRefType,
makeListType,
@@ -28,10 +33,6 @@ import {
intSequence,
testRoundTripAndValidate,
} from './test-util.js';
import {MetaTuple, newListMetaSequence} from './meta-sequence.js';
import {invariant, notNull} from './assert.js';
import List, {ListWriter} from './list.js';
import {equals} from './compare.js';
const testListSize = 5000;
const listOfNRef = 'sha1-aa1605484d993e89dbc0431acb9f2478282f9d94';
@@ -43,6 +44,13 @@ async function assertToJS(list: List, nums: Array<any>, start: number = 0,
assert.deepEqual(expect, jsArray);
}
async function validateList(l: List, values: number[]): Promise<void> {
assert.isTrue(equals(new List(values), l));
const out = [];
await l.forEach(v => void(out.push(v)));
assert.deepEqual(values, out);
}
// IMPORTANT: These tests and in particular the hash of the values should stay in sync with the
// corresponding tests in go
@@ -159,15 +167,16 @@ suite('List', () => {
assert.strictEqual(height + 1, s.sequence.items[0].ref.height);
});
test('LONG: insert', async () => {
const nums = intSequence(testListSize - 10);
let s = new List(nums);
for (let i = testListSize - 10; i < testListSize; i++) {
s = await s.insert(i, i);
async function validateInsertion(values: number[]): Promise<void> {
let l = new List();
for (let i = 0; i < values.length; i++) {
l = await l.insert(i, values[i]);
await validateList(l, values.slice(0, i + 1));
}
}
assert.strictEqual(s.hash.toString(), listOfNRef);
test('LONG: validate - insert ascending', async () => {
await validateInsertion(intSequence(300));
});
test('LONG: append', async () => {
js/src/map-test.js
+33 -17
View File
@@ -41,6 +41,22 @@ class CountingMemoryStore extends MemoryStore {
}
}
function intKVs(count: number): [[number, number]] {
const kvs = [];
for (let i = 0; i < count; i++) {
kvs.push([i, i + 1]);
}
return kvs;
}
async function validateMap(m: Map, kvs: [[number, number]]): Promise<void> {
assert.isTrue(equals(new Map(kvs), m));
const out = [];
await m.forEach((v, k) => void(out.push([k, v])));
assert.deepEqual(kvs, out);
}
suite('BuildMap', () => {
test('unique keys - strings', async () => {
@@ -69,11 +85,7 @@ suite('BuildMap', () => {
});
test('LONG: set of n numbers', () => {
const kvs = [];
for (let i = 0; i < testMapSize; i++) {
kvs.push([i, i + 1]);
}
const kvs = intKVs(testMapSize);
const m = new Map(kvs);
assert.strictEqual(m.hash.toString(), mapOfNRef);
@@ -95,10 +107,7 @@ suite('BuildMap', () => {
});
test('LONG: map of ref to ref, set of n numbers', () => {
const kvs = [];
for (let i = 0; i < testMapSize; i++) {
kvs.push([i, i + 1]);
}
const kvs = intKVs(testMapSize);
const kvRefs = kvs.map(entry => entry.map(n => new Ref(newStruct('num', {n}))));
const m = new Map(kvRefs);
@@ -124,11 +133,21 @@ suite('BuildMap', () => {
assert.strictEqual(m.hash.toString(), mapOfNRef);
});
test('LONG: set existing', async () => {
const kvs = [];
for (let i = 0; i < testMapSize; i++) {
kvs.push([i, i + 1]);
async function validateSet(kvs: [[number, number]]): Promise<void> {
let m = new Map();
for (let i = 0; i < kvs.length; i++) {
const kv = kvs[i];
m = await m.set(kv[0], kv[1]);
await validateMap(m, kvs.slice(0, i + 1));
}
}
test('LONG: validate - set ascending', async () => {
await validateSet(intKVs(300));
});
test('LONG: set existing', async () => {
const kvs = intKVs(testMapSize);
let m = new Map(kvs);
for (let i = 0; i < testMapSize; i++) {
@@ -157,10 +176,7 @@ suite('BuildMap', () => {
test('LONG: write, read, modify, read', async () => {
const db = new Database(makeTestingBatchStore());
const kvs = [];
for (let i = 0; i < testMapSize; i++) {
kvs.push([i, i + 1]);
}
const kvs = intKVs(testMapSize);
const m = new Map(kvs);
js/src/sequence-chunker.js
+33 -6
View File
@@ -97,14 +97,41 @@ export default class SequenceChunker<C: Collection, S, U: Sequence> {
await notNull(this._parent).resume();
}
// TODO: Only call maxNPrevItems once.
const prev =
await cursor.maxNPrevItems(this._boundaryChecker.windowSize - 1);
for (let i = 0; i < prev.length; i++) {
this._boundaryChecker.write(prev[i]);
// Number of previous items which must be hashed into the boundary checker.
let primeHashCount = this._boundaryChecker.windowSize - 1;
// If the cursor is beyond the final position in the sequence, the preceeding
// item may have been a chunk boundary. In that case, we must test at least the preceeding item.
const appendPenultimate = cursor.idx === cursor.length;
if (appendPenultimate) {
// In that case, we prime enough items *prior* to the penultimate item to be correct.
primeHashCount++;
}
// Number of items preceeding initial cursor in present chunk.
const primeCurrentCount = cursor.indexInChunk;
// Number of items to fetch prior to cursor position
const prevCount = Math.max(primeHashCount, primeCurrentCount);
const prev = await cursor.maxNPrevItems(prevCount);
for (let i = 0; i < prev.length; i++) {
const item = prev[i];
const backIdx = prev.length - i;
if (appendPenultimate && backIdx === 1) {
// Test the penultimate item for a boundary.
this.append(item);
continue;
}
if (backIdx <= primeHashCount) {
this._boundaryChecker.write(item);
}
if (backIdx <= primeCurrentCount) {
this._current.push(item);
}
}
this._current = await cursor.maxNPrevItems(cursor.indexInChunk);
this._used = this._current.length > 0;
}
js/src/set-test.js
+41 -28
View File
@@ -7,21 +7,21 @@
import {assert} from 'chai';
import {suite, setup, teardown, test} from 'mocha';
import BatchStore from './batch-store.js';
import Chunk from './chunk.js';
import Database from './database.js';
import Hash from './hash.js';
import MemoryStore from './memory-store.js';
import Ref from './ref.js';
import BatchStore from './batch-store.js';
import {BatchStoreAdaptorDelegate, makeTestingBatchStore} from './batch-store-adaptor.js';
import {newStruct} from './struct.js';
import {flatten, flattenParallel, deriveCollectionHeight} from './test-util.js';
import {invariant, notNull} from './assert.js';
import {MetaTuple, newSetMetaSequence} from './meta-sequence.js';
import Set from './set.js';
import {OrderedSequence} from './ordered-sequence.js';
import Hash from './hash.js';
import type {ValueReadWriter} from './value-store.js';
import {BatchStoreAdaptorDelegate, makeTestingBatchStore} from './batch-store-adaptor.js';
import {MetaTuple, newSetMetaSequence} from './meta-sequence.js';
import {OrderedSequence} from './ordered-sequence.js';
import {compare, equals} from './compare.js';
import {flatten, flattenParallel, intSequence, deriveCollectionHeight} from './test-util.js';
import {invariant, notNull} from './assert.js';
import {newStruct} from './struct.js';
const testSetSize = 5000;
const setOfNRef = 'sha1-8186877fb71711b8e6a516ed5c8ad1ccac8c6c00';
@@ -42,12 +42,12 @@ class CountingMemoryStore extends MemoryStore {
}
}
function firstNNumbers(n: number): Array<number> {
const nums = [];
for (let i = 0; i < n; i++) {
nums.push(i);
}
return nums;
async function validateSet(s: Set, values: number[]): Promise<void> {
assert.isTrue(equals(new Set(values), s));
const out = [];
await s.forEach(v => { out.push(v); });
assert.deepEqual(values, out);
}
suite('BuildSet', () => {
@@ -72,7 +72,7 @@ suite('BuildSet', () => {
});
test('LONG: set of n numbers', async () => {
const nums = firstNNumbers(testSetSize);
const nums = intSequence(testSetSize);
const s = new Set(nums);
assert.strictEqual(s.hash.toString(), setOfNRef);
@@ -83,7 +83,7 @@ suite('BuildSet', () => {
});
test('LONG: set of struct, set of n numbers', async () => {
const nums = firstNNumbers(testSetSize);
const nums = intSequence(testSetSize);
const structs = nums.map(n => newStruct('num', {n}));
const s = new Set(structs);
assert.strictEqual(s.hash.toString(), 'sha1-f10d8ccbc2270bb52bb988a0cadff912e2723eed');
@@ -98,7 +98,7 @@ suite('BuildSet', () => {
});
test('LONG: set of ref, set of n numbers', async () => {
const nums = firstNNumbers(testSetSize);
const nums = intSequence(testSetSize);
const refs = nums.map(n => new Ref(newStruct('num', {n})));
const s = new Set(refs);
assert.strictEqual(s.hash.toString(), 'sha1-14eeb2d1835011bf3e018121ba3274bc08e634e5');
@@ -109,18 +109,32 @@ suite('BuildSet', () => {
});
test('LONG: insert', async () => {
const nums = firstNNumbers(testSetSize - 10);
let s = new Set(nums);
for (let i = testSetSize - 10; i < testSetSize; i++) {
s = await s.insert(i);
assert.strictEqual(i + 1, s.size);
const nums = intSequence(testSetSize);
const build = nums.slice(0, testSetSize - 10);
const insert = nums.slice(testSetSize - 10);
let s = new Set(build);
for (let i = 0; i < insert.length; i++) {
s = await s.insert(insert[i]);
assert.strictEqual(build.length + i + 1, s.size);
}
assert.strictEqual(s.hash.toString(), 'sha1-b41aab13e8de940d998c1f55a2f48f63159a19e0');
await validateSet(s, nums);
});
async function validateInsertion(values: number[]): Promise<void> {
let s = new Set();
for (let i = 0; i < values.length; i++) {
s = await s.insert(values[i]);
await validateSet(s, values.slice(0, i + 1));
}
}
test('LONG: validate - insert ascending', async () => {
await validateInsertion(intSequence(300));
});
test('LONG: remove', async () => {
const nums = firstNNumbers(testSetSize + 10);
const nums = intSequence(testSetSize + 10);
let s = new Set(nums);
let count = 10;
while (count-- > 0) {
@@ -134,7 +148,7 @@ suite('BuildSet', () => {
test('LONG: write, read, modify, read', async () => {
const db = new Database(makeTestingBatchStore());
const nums = firstNNumbers(testSetSize);
const nums = intSequence(testSetSize);
const s = new Set(nums);
const r = db.writeValue(s).targetHash;
const s2 = await db.readValue(r);
@@ -153,11 +167,10 @@ suite('BuildSet', () => {
await db.close();
});
test('LONG: union write, read, modify, read', async () => {
const db = new Database(makeTestingBatchStore());
const tmp = firstNNumbers(testSetSize);
const tmp = intSequence(testSetSize);
const numbers = [];
const strings = [];
const structs = [];
@@ -552,7 +565,7 @@ suite('CompoundSet', () => {
});
test('LONG: canned set diff', async () => {
let s1 = new Set(firstNNumbers(testSetSize));
let s1 = new Set(intSequence(testSetSize));
s1 = await db.readValue(db.writeValue(s1).targetHash);
{