Abstract JS list tests (#1398)

js/src/blob-test.js

@@ -4,9 +4,14 @@ import {blobType, refOfBlobType} from './type.js';
 import {assert} from 'chai';
 import {newBlob, BlobWriter, NomsBlob} from './blob.js';
 import {suite, test} from 'mocha';
-import {testRoundTripAndValidate} from './test-util.js';
+import {
+  assertChunkCountAndType,
+  assertValueRef,
+  assertValueType,
+  chunkDiffCount,
+  testRoundTripAndValidate,
+} from './test-util.js';
 import {invariant} from './assert.js';
-import RefValue from './ref-value.js';
 
 // IMPORTANT: These tests and in particular the hash of the values should stay in sync with the
 // corresponding tests in go
@@ -37,8 +42,8 @@ suite('Blob', () => {
       nb[i + 1] = buff[i];
     }
 
-    const b2 = await newBlob(nb);
-    assert.strictEqual(expectCount, chunkDiffCount(blob.chunks, b2.chunks));
+    const v2 = await newBlob(nb);
+    assert.strictEqual(expectCount, chunkDiffCount(blob, v2));
   }
 
   async function testAppendChunkDiff(buff: Uint8Array, blob: NomsBlob, expectCount: number):
@@ -48,33 +53,8 @@ suite('Blob', () => {
       nb[i] = buff[i];
     }
 
-    const b2 = await newBlob(nb);
-    assert.strictEqual(expectCount, chunkDiffCount(blob.chunks, b2.chunks));
-  }
-
-  function chunkDiffCount(c1: Array<RefValue>, c2: Array<RefValue>): number {
-    let diffCount = 0;
-    const refs = Object.create(null);
-    c1.forEach(r => {
-      const refStr = r.targetRef.toString();
-      let count = refs[refStr];
-      count = count === undefined ? 1 : count + 1;
-      refs[refStr] = count;
-    });
-
-    c2.forEach(r => {
-      const refStr = r.targetRef.toString();
-      const count = refs[refStr];
-      if (count === undefined) {
-        diffCount++;
-      } else if (count === 1) {
-        delete refs[refStr];
-      } else {
-        refs[refStr] = count - 1;
-      }
-    });
-
-    return diffCount + Object.keys(refs).length;
+    const v2 = await newBlob(nb);
+    assert.strictEqual(expectCount, chunkDiffCount(blob, v2));
   }
 
   function randomBuff(len: number): Uint8Array {
@@ -93,33 +73,18 @@ suite('Blob', () => {
     const buff = randomBuff(length);
     const blob = await newBlob(buff);
 
-    // Ref
-    assert.strictEqual(expectRefStr, blob.ref.toString());
-
-    // Type
-    assert.isTrue(blobType.equals(blob.type));
-
-    // Length
+    assertValueRef(expectRefStr, blob);
+    assertValueType(blobType, blob);
     assert.strictEqual(length, blob.length);
-
-    // Chunk Count
-    assert.strictEqual(expectChunkCount, blob.chunks.length);
-
-    // ChunkRef Type
-    blob.chunks.forEach(r => assert.isTrue(refOfBlobType.equals(r.type)));
+    assertChunkCountAndType(expectChunkCount, refOfBlobType, blob);
 
     await testRoundTripAndValidate(blob, async(b2) => {
       await assertReadFull(buff, b2);
-
-      // Equals
-      assert.isTrue(b2.equals(blob));
-      assert.isTrue(blob.equals(b2));
     });
 
     // TODO: Random Read
 
     await testPrependChunkDiff(buff, blob, expectPrependChunkDiff);
-
     await testAppendChunkDiff(buff, blob, expectAppendChunkDiff);
   }
 
@@ -128,29 +93,28 @@ suite('Blob', () => {
     _value: number;
     _count: number;
 
-  constructor(seed: number = 0) {
-    this._z = seed;
-    this._value = seed;
-    this._count = 4;
-  }
-
-  nextUint8(): number {
-    // Increment number
-    if (this._count === 0) {
-      this._z = this._z + 1;
-      this._value = this._z;
+    constructor(seed: number = 0) {
+      this._z = seed;
+      this._value = seed;
       this._count = 4;
     }
 
-    // Unshift a uint8 from our current number
-    const retval = this._value & 0xff;
-    this._value = this._value >>> 8;
-    this._count--;
+    nextUint8(): number {
+      // Increment number
+      if (this._count === 0) {
+        this._z++;
+        this._value = this._z;
+        this._count = 4;
+      }
 
-    return retval;
+      // Unshift a uint8 from our current number
+      const retval = this._value & 0xff;
+      this._value = this._value >>> 8;
+      this._count--;
+
+      return retval;
+    }
   }
-}
-
 
   test('Blob 1K', async () => {
     await blobTestSuite(10, 'sha1-cb21e6231cbcf57ff8a9e80c9cbc5b1e798bf9ea', 3, 2, 2);

js/src/list-test.js

@@ -14,13 +14,20 @@ import {
   makeListType,
   numberType,
   stringType,
-  valueType,
 } from './type.js';
-import {flatten, flattenParallel} from './test-util.js';
+import {
+  assertChunkCountAndType,
+  assertValueRef,
+  assertValueType,
+  chunkDiffCount,
+  flatten,
+  flattenParallel,
+  intSequence,
+  testRoundTripAndValidate,
+} from './test-util.js';
 import {IndexedMetaSequence, MetaTuple} from './meta-sequence.js';
 import {invariant} from './assert.js';
 import {ListLeafSequence, newList, NomsList} from './list.js';
-import type {Type} from './type.js';
 
 const testListSize = 5000;
 const listOfNRef = 'sha1-df0a58e5fb11b2bc0adbab07c2f39c6b3e02b42b';
@@ -32,31 +39,99 @@ async function assertToJS(list: NomsList, nums: Array<any>, start: number = 0,
   assert.deepEqual(expect, jsArray);
 }
 
-suite('BuildList', () => {
-  function intSequence(start: number, end: number): Array<number> {
-    const nums = [];
+// IMPORTANT: These tests and in particular the hash of the values should stay in sync with the
+// corresponding tests in go
 
-    for (let i = start; i < end; i++) {
-      nums.push(i);
+suite('List', () => {
+
+  async function testPrependChunkDiff(nums: Array<any>, list: NomsList, expectCount: number):
+      Promise<void> {
+    const nn = new Array(nums.length + 1);
+    nn[0] = 0;
+    for (let i = 0; i < nums.length; i++) {
+      nn[i + 1] = nums[i];
     }
 
-    return nums;
+    const v2 = await newList(nn, list.type);
+    assert.strictEqual(expectCount, chunkDiffCount(list, v2));
   }
 
-  function firstNNumbers(n: number): Array<number> {
-    return intSequence(0, n);
+  async function testAppendChunkDiff(nums: Array<any>, list: NomsList, expectCount: number):
+      Promise<void> {
+    const nn = new Array(nums.length + 1);
+    nn[0] = 0;
+    for (let i = 0; i < nums.length; i++) {
+      nn[i] = nums[i];
+    }
+    nn[nums.length] = 0;
+
+    const v2 = await newList(nn, list.type);
+    assert.strictEqual(expectCount, chunkDiffCount(list, v2));
   }
 
-  test('LONG: set of n numbers, length', async () => {
-    const nums = firstNNumbers(testListSize);
+  async function testToJS(expect: Array<any>, list: NomsList): Promise<void> {
+    const length = expect.length;
+    let start = 0;
+
+    for (let count = Math.round(length / 2); count > 2;) {
+      assert.deepEqual(expect.slice(start, start + count), await list.toJS(start, start + count));
+      start = start + count;
+      count = (length - start) / 2;
+    }
+  }
+
+  async function testGet(nums: Array<any>, list: NomsList): Promise<void> {
+    const incr = Math.round(nums.length / 256); // test 256 indices
+
+    for (let i = 0; i < nums.length; i += incr) {
+      assert.strictEqual(nums[i], await list.get(i));
+    }
+  }
+
+  async function testForEach(nums: Array<any>, list: NomsList): Promise<void> {
+    const out = [];
+    await list.forEach(v => {
+      out.push(v);
+    });
+
+    assert.deepEqual(nums, out);
+  }
+
+  async function listTestSuite(size: number, expectRefStr: string, expectChunkCount: number,
+                               expectPrependChunkDiff: number,
+                               expectAppendChunkDiff: number): Promise<void> {
+    const length = 1 << size;
+    const nums = intSequence(length);
     const tr = makeListType(numberType);
-    const s = await newList(nums, tr);
-    assert.strictEqual(s.ref.toString(), listOfNRef);
-    assert.strictEqual(testListSize, s.length);
+    const list = await newList(nums, tr);
+
+    assertValueRef(expectRefStr, list);
+    assertValueType(tr, list);
+    assert.isFalse(list.isEmpty());
+    assert.strictEqual(length, list.length);
+    assertChunkCountAndType(expectChunkCount, makeRefType(tr), list);
+
+    await testRoundTripAndValidate(list, async(v2) => {
+      await assertToJS(v2, nums);
+    });
+
+    await testForEach(nums, list);
+    await testToJS(nums, list);
+    await testGet(nums, list);
+    await testPrependChunkDiff(nums, list, expectPrependChunkDiff);
+    await testAppendChunkDiff(nums, list, expectAppendChunkDiff);
+  }
+
+  test('List 1K', async () => {
+    await listTestSuite(10, 'sha1-12f9f864bfb61ae76ac3201f564757f1b576237c', 19, 19, 2);
+  });
+
+  test('LONG: List 4K', async () => {
+    await listTestSuite(12, 'sha1-0bb374c248fcc987ef93dddef91af693032240a9', 2, 3, 2);
   });
 
   test('LONG: list of ref, set of n numbers, length', async () => {
-    const nums = firstNNumbers(testListSize);
+    const nums = intSequence(testListSize);
 
     const structType = makeStructType('num', {
       'n': numberType,
@@ -75,28 +150,8 @@ suite('BuildList', () => {
     assert.strictEqual(testListSize, s.length);
   });
 
-  test('LONG: toJS', async () => {
-    const nums = firstNNumbers(5000);
-    const tr = makeListType(numberType);
-    const s = await newList(nums, tr);
-    assert.strictEqual(s.ref.toString(), listOfNRef);
-    assert.strictEqual(testListSize, s.length);
-
-    await assertToJS(s, nums, 1000, 2000);
-    await assertToJS(s, nums, 3000, 3500);
-    await assertToJS(s, nums);
-    await assertToJS(s, nums, 0, -100);
-    await assertToJS(s, nums, -300, -100);
-    await assertToJS(s, nums, -2000, 4000);
-    await assertToJS(s, nums, -300, -300);
-    await assertToJS(s, nums, -300, -400);
-    await assertToJS(s, nums, 10000, 10000);
-    await assertToJS(s, nums, 0, 1);
-    await assertToJS(s, nums, -1);
-  });
-
   test('LONG: insert', async () => {
-    const nums = firstNNumbers(testListSize - 10);
+    const nums = intSequence(testListSize - 10);
     const tr = makeListType(numberType);
     let s = await newList(nums, tr);
 
@@ -108,7 +163,7 @@ suite('BuildList', () => {
   });
 
   test('LONG: append', async () => {
-    const nums = firstNNumbers(testListSize - 10);
+    const nums = intSequence(testListSize - 10);
     const tr = makeListType(numberType);
     let s = await newList(nums, tr);
 
@@ -120,7 +175,7 @@ suite('BuildList', () => {
   });
 
   test('LONG: remove', async () => {
-    const nums = firstNNumbers(testListSize + 10);
+    const nums = intSequence(testListSize + 10);
     const tr = makeListType(numberType);
     let s = await newList(nums, tr);
 
@@ -133,13 +188,13 @@ suite('BuildList', () => {
   });
 
   test('LONG: splice', async () => {
-    const nums = firstNNumbers(testListSize);
+    const nums = intSequence(testListSize);
     const tr = makeListType(numberType);
     let s = await newList(nums, tr);
 
     const splice500At = async (idx: number) => {
       s = await s.splice(idx, 500);
-      s = await s.splice(idx, 0, ...intSequence(idx, idx + 500));
+      s = await s.splice(idx, 0, ...intSequence(idx + 500, idx));
     };
 
 
@@ -153,7 +208,7 @@ suite('BuildList', () => {
   test('LONG: write, read, modify, read', async () => {
     const ds = new DataStore(makeTestingBatchStore());
 
-    const nums = firstNNumbers(testListSize);
+    const nums = intSequence(testListSize);
     const tr = makeListType(numberType);
     const s = await newList(nums, tr);
     const r = ds.writeValue(s).targetRef;
@@ -170,32 +225,11 @@ suite('BuildList', () => {
 });
 
 suite('ListLeafSequence', () => {
-  test('isEmpty', () => {
+  test('Empty list isEmpty', () => {
     const ds = new DataStore(makeTestingBatchStore());
     const tr = makeListType(stringType);
     const newList = items => new NomsList(tr, new ListLeafSequence(ds, tr, items));
     assert.isTrue(newList([]).isEmpty());
-    assert.isFalse(newList(['z', 'x', 'a', 'b']).isEmpty());
-  });
-
-  test('get', async () => {
-    const ds = new DataStore(makeTestingBatchStore());
-    const tr = makeListType(stringType);
-    const l = new NomsList(tr, new ListLeafSequence(ds, tr, ['z', 'x', 'a', 'b']));
-    assert.strictEqual('z', await l.get(0));
-    assert.strictEqual('x', await l.get(1));
-    assert.strictEqual('a', await l.get(2));
-    assert.strictEqual('b', await l.get(3));
-  });
-
-  test('forEach', async () => {
-    const ds = new DataStore(makeTestingBatchStore());
-    const tr = makeListType(numberType);
-    const l = new NomsList(tr, new ListLeafSequence(ds, tr, [4, 2, 10, 16]));
-
-    const values = [];
-    await l.forEach((v, i) => { values.push(v, i); });
-    assert.deepEqual([4, 0, 2, 1, 10, 2, 16, 3], values);
   });
 
   test('iterator', async () => {
@@ -230,27 +264,6 @@ suite('ListLeafSequence', () => {
     await test([42]);
     await test([4, 2, 10, 16]);
   });
-
-  function testChunks(elemType: Type) {
-    const ds = new DataStore(makeTestingBatchStore());
-    const tr = makeListType(elemType);
-    const r1 = ds.writeValue('x');
-    const r2 = ds.writeValue('a');
-    const r3 = ds.writeValue('b');
-    const l = new NomsList(tr, new ListLeafSequence(ds, tr, ['z', r1, r2, r3]));
-    assert.strictEqual(3, l.chunks.length);
-    assert.isTrue(r1.equals(l.chunks[0]));
-    assert.isTrue(r2.equals(l.chunks[1]));
-    assert.isTrue(r3.equals(l.chunks[2]));
-  }
-
-  test('chunks, list of value', () => {
-    testChunks(valueType);
-  });
-
-  test('chunks', () => {
-    testChunks(stringType);
-  });
 });
 
 suite('CompoundList', () => {
@@ -278,34 +291,6 @@ suite('CompoundList', () => {
     return l;
   }
 
-  test('isEmpty', () => {
-    assert.isFalse(build().isEmpty());
-  });
-
-  test('toJS', async () => {
-    const l = build();
-    await assertToJS(l, ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n'] , 0, 8);
-  });
-
-  test('get', async () => {
-    const l = build();
-    assert.strictEqual('a', await l.get(0));
-    assert.strictEqual('b', await l.get(1));
-    assert.strictEqual('e', await l.get(2));
-    assert.strictEqual('f', await l.get(3));
-    assert.strictEqual('h', await l.get(4));
-    assert.strictEqual('i', await l.get(5));
-    assert.strictEqual('m', await l.get(6));
-    assert.strictEqual('n', await l.get(7));
-  });
-
-  test('forEach', async () => {
-    const l = build();
-    const values = [];
-    await l.forEach((k, i) => { values.push(k, i); });
-    assert.deepEqual(['a', 0, 'b', 1, 'e', 2, 'f', 3, 'h', 4, 'i', 5, 'm', 6, 'n', 7], values);
-  });
-
   test('iterator', async () => {
     const l = build();
     const expected = ['a', 'b', 'e', 'f', 'h', 'i', 'm', 'n'];
@@ -344,44 +329,11 @@ suite('CompoundList', () => {
         [{done: false, value: 'a'}, {done: false, value: 'b'}, {done: true}, {done: true}],
         values);
   });
-
-  test('chunks', () => {
-    const l = build();
-    assert.strictEqual(2, l.chunks.length);
-  });
-
-  test('length', () => {
-    const l = build();
-    assert.equal(l.length, 8);
-  });
-
-  test('chunks', () => {
-    const l = build();
-    const chunks = l.chunks;
-    const sequence = l.sequence;
-    assert.equal(2, chunks.length);
-    assert.isTrue(sequence.items[0].ref.equals(chunks[0]));
-    assert.isTrue(sequence.items[1].ref.equals(chunks[1]));
-  });
 });
 
 suite('Diff List', () => {
-  function intSequence(start: number, end: number): Array<number> {
-    const nums = [];
-
-    for (let i = start; i < end; i++) {
-      nums.push(i);
-    }
-
-    return nums;
-  }
-
-  function firstNNumbers(n: number): Array<number> {
-    return intSequence(0, n);
-  }
-
   test('LONG: Remove 5x100', async () => {
-    const nums1 = firstNNumbers(5000);
+    const nums1 = intSequence(5000);
     const nums2 = nums1.slice(0);
 
     let count = 5;
@@ -400,7 +352,7 @@ suite('Diff List', () => {
   });
 
   test('LONG: Add 5x5', async () => {
-    const nums1 = firstNNumbers(5000);
+    const nums1 = intSequence(5000);
     const nums2 = nums1.slice(0);
 
     let count = 5;
@@ -419,7 +371,7 @@ suite('Diff List', () => {
   });
 
   test('LONG: Replace reverse 5x100', async () => {
-    const nums1 = firstNNumbers(5000);
+    const nums1 = intSequence(5000);
     const nums2 = nums1.slice(0);
 
     let count = 5;
@@ -438,8 +390,8 @@ suite('Diff List', () => {
   });
 
   test('LONG: Load Limit', async () => {
-    const nums1 = firstNNumbers(5);
-    const nums2 = firstNNumbers(5000);
+    const nums1 = intSequence(5);
+    const nums2 = intSequence(5000);
 
     const directDiff = calcSplices(nums1.length, nums2.length, (i, j) => nums1[i] === nums2[j]);
     const tr = makeListType(numberType);

js/src/test-util.js

@@ -1,12 +1,15 @@
 // @flow
 
 import DataStore from './data-store.js';
+import type {Collection} from './collection.js';
 import type {valueOrPrimitive} from './value.js';
-import {AsyncIterator} from './async-iterator.js';
-import {Value} from './value.js';
 import {assert} from 'chai';
-import {notNull} from './assert.js';
+import {AsyncIterator} from './async-iterator.js';
+import {getChunksOfValue, Value} from './value.js';
+import {getRefOfValue} from './get-ref.js';
+import {getTypeOfValue, Type} from './type.js';
 import {makeTestingBatchStore} from './batch-store-adaptor.js';
+import {notNull} from './assert.js';
 
 export async function flatten<T>(iter: AsyncIterator<T>): Promise<Array<T>> {
   const values = [];
@@ -25,6 +28,19 @@ export async function flattenParallel<T>(iter: AsyncIterator<T>, count: number):
   return results.map(res => notNull(res.value));
 }
 
+export function assertValueRef(expectRefStr: string, v: valueOrPrimitive) {
+  assert.strictEqual(expectRefStr, getRefOfValue(v).toString());
+}
+
+export function assertValueType(expectType: Type, v: valueOrPrimitive) {
+  assert.isTrue(expectType.equals(getTypeOfValue(v)));
+}
+
+export function assertChunkCountAndType(expectCount: number, expectType: Type,
+    v: Collection) {
+  v.chunks.forEach(r => assert.isTrue(expectType.equals(r.type)));
+}
+
 export async function testRoundTripAndValidate<T: valueOrPrimitive>(v: T,
       validateFn: (v2: T) => Promise<void>): Promise<void> {
   const bs = makeTestingBatchStore();
@@ -42,3 +58,41 @@ export async function testRoundTripAndValidate<T: valueOrPrimitive>(v: T,
   }
   await validateFn(v2);
 }
+
+export function chunkDiffCount(v1: valueOrPrimitive, v2: valueOrPrimitive): number {
+  const c1 = getChunksOfValue(v1);
+  const c2 = getChunksOfValue(v2);
+
+  let diffCount = 0;
+  const refs = Object.create(null);
+  c1.forEach(r => {
+    const refStr = r.targetRef.toString();
+    let count = refs[refStr];
+    count = count === undefined ? 1 : count + 1;
+    refs[refStr] = count;
+  });
+
+  c2.forEach(r => {
+    const refStr = r.targetRef.toString();
+    const count = refs[refStr];
+    if (count === undefined) {
+      diffCount++;
+    } else if (count === 1) {
+      delete refs[refStr];
+    } else {
+      refs[refStr] = count - 1;
+    }
+  });
+
+  return diffCount + Object.keys(refs).length;
+}
+
+export function intSequence(count: number, start: number = 0): Array<number> {
+  const nums = [];
+
+  for (let i = start; i < count; i++) {
+    nums.push(i);
+  }
+
+  return nums;
+}

js/src/value.js

@@ -35,3 +35,11 @@ export class Value {
 }
 
 export type valueOrPrimitive = primitive | Value;
+
+export function getChunksOfValue(v: valueOrPrimitive): Array<RefValue> {
+  if (v instanceof Value) {
+    return v.chunks;
+  }
+
+  return [];
+}