mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-04 03:11:52 -05:00
Code Issues Packages Projects Releases 159 Wiki Activity

Meta nodes encode a fake simple ref of non-ordered values (#1893)

Browse Source 
Previously when we had an ordered (set/map) prolly tree containing
non-ordered values (blobs/refs/etc), we'd put the ref of the largest
value in each meta node, complete with its full type info and height.

This is wasteful, all we really need is the hash of the largest item for
searching the tree. In lieu of encoding just the hash - which isn't a
value - this patch creates a fake Ref<Boolean> with height 0.
This commit is contained in:
Ben Kalman
2016-06-23 15:59:26 -07:00
committed by GitHub
parent e7da64ad98
commit 0f4ed16641
44 changed files with 606 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/src/blob.js
+9 -4
View File
@@ -10,8 +10,12 @@ import {SequenceCursor} from './sequence.js';
import {invariant} from './assert.js';
import type {ValueReader, ValueWriter, ValueReadWriter} from './value-store.js';
import {blobType} from './type.js';
import {MetaTuple, newIndexedMetaSequenceChunkFn, newIndexedMetaSequenceBoundaryChecker} from
'./meta-sequence.js';
import {
OrderedKey,
MetaTuple,
newIndexedMetaSequenceChunkFn,
newIndexedMetaSequenceBoundaryChecker,
} from './meta-sequence.js';
import BuzHashBoundaryChecker from './buzhash-boundary-checker.js';
import Ref from './ref.js';
import SequenceChunker from './sequence-chunker.js';
@@ -150,11 +154,12 @@ function newBlobLeafChunkFn(vr: ?ValueReader, vw: ?ValueWriter): makeChunkFn {
return (items: Array<number>) => {
const blobLeaf = new BlobLeafSequence(vr, Bytes.fromValues(items));
const blob = Blob.fromSequence(blobLeaf);
const key = new OrderedKey(items.length);
let mt;
if (vw) {
mt = new MetaTuple(vw.writeValue(blob), items.length, items.length, null);
mt = new MetaTuple(vw.writeValue(blob), key, items.length, null);
} else {
mt = new MetaTuple(new Ref(blob), items.length, items.length, blob);
mt = new MetaTuple(new Ref(blob), key, items.length, blob);
}
return [mt, blobLeaf];
};
js/src/encoding-test.js
+32 -9
View File
@@ -26,6 +26,7 @@ import {equals} from './compare.js';
import {invariant} from './assert.js';
import {newStruct, newStructWithType} from './struct.js';
import {
OrderedKey,
MetaTuple,
newBlobMetaSequence,
newListMetaSequence,
@@ -85,7 +86,10 @@ suite('Encoding - roundtrip', () => {
test('compound list', () => {
const leaf = List.fromSequence(newListLeafSequence(null, [4, 5, 6, 7]));
const mts = [new MetaTuple(new Ref(leaf), 10, 10, null), new MetaTuple(new Ref(leaf), 20, 20, null)];
const mts = [
new MetaTuple(new Ref(leaf), new OrderedKey(10), 10, null),
new MetaTuple(new Ref(leaf), new OrderedKey(20), 20, null),
];
assertRoundTrips(List.fromSequence(newListMetaSequence(null, mts)));
});
});
@@ -336,9 +340,9 @@ suite('Encoding', () => {
uint8(RefKind), uint8(BlobKind), r3.toString(), uint64(33), uint8(NumberKind), float64(60), uint64(60),
],
Blob.fromSequence(newBlobMetaSequence(null, [
new MetaTuple(constructRef(refOfBlobType, r1, 11), 20, 20, null),
new MetaTuple(constructRef(refOfBlobType, r2, 22), 40, 40, null),
new MetaTuple(constructRef(refOfBlobType, r3, 33), 60, 60, null),
new MetaTuple(constructRef(refOfBlobType, r1, 11), new OrderedKey(20), 20, null),
new MetaTuple(constructRef(refOfBlobType, r2, 22), new OrderedKey(40), 40, null),
new MetaTuple(constructRef(refOfBlobType, r3, 33), new OrderedKey(60), 60, null),
]))
);
});
@@ -403,13 +407,12 @@ suite('Encoding', () => {
uint8(RefKind), uint8(ListKind), uint8(NumberKind), list2.hash.toString(), uint64(1), uint8(NumberKind), float64(4), uint64(4),
],
List.fromSequence(newListMetaSequence(null, [
new MetaTuple(new Ref(list1), 1, 1, null),
new MetaTuple(new Ref(list2), 4, 4, null),
new MetaTuple(new Ref(list1), new OrderedKey(1), 1, null),
new MetaTuple(new Ref(list2), new OrderedKey(4), 4, null),
]))
);
});
test('compound set', () => {
const set1 = Set.fromSequence(newSetLeafSequence(null, [0, 1]));
const set2 = Set.fromSequence(newSetLeafSequence(null, [2, 3, 4]));
@@ -422,8 +425,28 @@ suite('Encoding', () => {
uint8(RefKind), uint8(SetKind), uint8(NumberKind), set2.hash.toString(), uint64(1), uint8(NumberKind), float64(4), uint64(3),
],
Set.fromSequence(newSetMetaSequence(null, [
new MetaTuple(new Ref(set1), 1, 2, null),
new MetaTuple(new Ref(set2), 4, 3, null),
new MetaTuple(new Ref(set1), new OrderedKey(1), 2, null),
new MetaTuple(new Ref(set2), new OrderedKey(4), 3, null),
]))
);
});
test('compound set of blobs', () => {
const blobs = [0, 1, 2, 3, 4].map(i => new Blob(new Uint8Array([i])));
const set1 = Set.fromSequence(newSetLeafSequence(null, blobs.slice(0, 2)));
const set2 = Set.fromSequence(newSetLeafSequence(null, blobs.slice(2)));
assertEncoding(
[
uint8(SetKind), uint8(BlobKind), true,
uint32(2), // len,
// See https://github.com/attic-labs/noms/issues/1688#issuecomment-227528987
uint8(RefKind), uint8(SetKind), uint8(BlobKind), set1.hash.toString(), uint64(1), uint8(RefKind), uint8(BoolKind), blobs[1].hash.toString(), uint64(0), uint64(2),
uint8(RefKind), uint8(SetKind), uint8(BlobKind), set2.hash.toString(), uint64(1), uint8(RefKind), uint8(BoolKind), blobs[4].hash.toString(), uint64(0), uint64(3),
],
Set.fromSequence(newSetMetaSequence(null, [
new MetaTuple(new Ref(set1), new OrderedKey(blobs[1]), 2, null),
new MetaTuple(new Ref(set2), new OrderedKey(blobs[4]), 3, null),
]))
);
});
js/src/list-test.js
+10 -7
View File
@@ -9,7 +9,7 @@ import {suite, setup, teardown, test} from 'mocha';
import List, {ListWriter, ListLeafSequence} from './list.js';
import Ref from './ref.js';
import {MetaTuple, newListMetaSequence} from './meta-sequence.js';
import {OrderedKey, MetaTuple, newListMetaSequence} from './meta-sequence.js';
import {DEFAULT_MAX_SPLICE_MATRIX_SIZE, calcSplices} from './edit-distance.js';
import {equals} from './compare.js';
import {invariant, notNull} from './assert.js';
@@ -310,15 +310,18 @@ suite('CompoundList', () => {
const l4 = new List(['m', 'n']);
const r4 = db.writeValue(l4);
const m1 = List.fromSequence(newListMetaSequence(
db, [new MetaTuple(r1, 2, 2, null), new MetaTuple(r2, 2, 2, null)]));
const m1 = List.fromSequence(newListMetaSequence(db, [
new MetaTuple(r1, new OrderedKey(2), 2, null),
new MetaTuple(r2, new OrderedKey(2), 2, null)]));
const rm1 = db.writeValue(m1);
const m2 = List.fromSequence(newListMetaSequence(
db, [new MetaTuple(r3, 2, 2, null), new MetaTuple(r4, 2, 2, null)]));
const m2 = List.fromSequence(newListMetaSequence(db, [
new MetaTuple(r3, new OrderedKey(2), 2, null),
new MetaTuple(r4, new OrderedKey(2), 2, null)]));
const rm2 = db.writeValue(m2);
const l = List.fromSequence(newListMetaSequence(
db, [new MetaTuple(rm1, 4, 4, null), new MetaTuple(rm2, 4, 4, null)]));
const l = List.fromSequence(newListMetaSequence(db, [
new MetaTuple(rm1, new OrderedKey(4), 4, null),
new MetaTuple(rm2, new OrderedKey(4), 4, null)]));
return l;
}
js/src/list.js
+4 -2
View File
@@ -17,6 +17,7 @@ import {diff} from './indexed-sequence-diff.js';
import {getHashOfValue} from './get-hash.js';
import {invariant} from './assert.js';
import {
OrderedKey,
MetaTuple,
newIndexedMetaSequenceBoundaryChecker,
newIndexedMetaSequenceChunkFn,
@@ -37,11 +38,12 @@ function newListLeafChunkFn<T: Value>(vr: ?ValueReader, vw: ?ValueWriter): makeC
return (items: Array<T>) => {
const seq = newListLeafSequence(vr, items);
const list = List.fromSequence(seq);
const key = new OrderedKey(items.length);
let mt;
if (vw) {
mt = new MetaTuple(vw.writeValue(list), items.length, items.length, null);
mt = new MetaTuple(vw.writeValue(list), key, items.length, null);
} else {
mt = new MetaTuple(new Ref(list), items.length, items.length, list);
mt = new MetaTuple(new Ref(list), key, items.length, list);
}
return [mt, seq];
};
js/src/map-test.js
+62 -11
View File
@@ -7,7 +7,9 @@
import {assert} from 'chai';
import {suite, setup, teardown, test} from 'mocha';
import Blob from './blob.js';
import Ref from './ref.js';
import Set from './set.js';
import Struct, {newStruct} from './struct.js';
import {
assertChunkCountAndType,
@@ -23,8 +25,9 @@ import {
import {getTypeOfValue} from './type.js';
import type Value from './value.js';
import {invariant, notNull} from './assert.js';
import List from './list.js';
import Map, {MapLeafSequence} from './map.js';
import {MetaTuple, newMapMetaSequence} from './meta-sequence.js';
import {OrderedKey, MetaTuple, newMapMetaSequence} from './meta-sequence.js';
import type {ValueReadWriter} from './value-store.js';
import {compare, equals} from './compare.js';
import {OrderedMetaSequence} from './meta-sequence.js';
@@ -92,11 +95,11 @@ suite('BuildMap', () => {
const newNumberStruct = i => newStruct('', {n: i});
test('Map 1K structs', async () => {
await mapTestSuite(10, 'sha1-a9fb9df66b63d1a74a1d1a486d05d34b30d84441', 18, newNumberStruct);
await mapTestSuite(10, 'sha1-73ab90e854ea52001e59ea4b097c9f3b40565d8c', 18, newNumberStruct);
});
test('LONG: Map 4K structs', async () => {
await mapTestSuite(12, 'sha1-c6256462bb26942e33e4b9e5ece3136a458b8ae8', 76, newNumberStruct);
await mapTestSuite(12, 'sha1-b48765e4423ec48eb5925276794b2864a4b48928', 76, newNumberStruct);
});
test('unique keys - strings', async () => {
@@ -151,7 +154,7 @@ suite('BuildMap', () => {
const kvRefs = kvs.map(entry => entry.map(n => new Ref(newStruct('num', {n}))));
const m = new Map(kvRefs);
assert.strictEqual(m.hash.toString(), 'sha1-b119c8145a3ed519d1271a35a4b25723ed0b61d2');
assert.strictEqual(m.hash.toString(), 'sha1-c43b17db2fa433aa1842b8b0b25acfd10e035be3');
const height = deriveCollectionHeight(m);
assert.isTrue(height > 0);
// height + 1 because the leaves are Ref values (with height 1).
@@ -265,7 +268,7 @@ suite('BuildMap', () => {
const sortedKeys = numbers.concat(strings, structs);
const m = new Map(kvs);
assert.strictEqual(m.hash.toString(), 'sha1-c4b291de9dfb466f3afbd0c9bd0afb0ce5f33c70');
assert.strictEqual(m.hash.toString(), 'sha1-96504f677dae417f0714af77b3e313ca1c3764e6');
const height = deriveCollectionHeight(m);
assert.isTrue(height > 0);
assert.strictEqual(height, m.sequence.items[0].ref.height);
@@ -454,19 +457,19 @@ suite('CompoundMap', () => {
const r4 = vwr.writeValue(l4);
const m1 = Map.fromSequence(newMapMetaSequence(vwr, [
new MetaTuple(r1, 'b', 2, null),
new MetaTuple(r2, 'f', 2, null),
new MetaTuple(r1, new OrderedKey('b'), 2, null),
new MetaTuple(r2, new OrderedKey('f'), 2, null),
]));
const rm1 = vwr.writeValue(m1);
const m2 = Map.fromSequence(newMapMetaSequence(vwr, [
new MetaTuple(r3, 'i', 2, null),
new MetaTuple(r4, 'n', 2, null),
new MetaTuple(r3, new OrderedKey('i'), 2, null),
new MetaTuple(r4, new OrderedKey('n'), 2, null),
]));
const rm2 = vwr.writeValue(m2);
const c = Map.fromSequence(newMapMetaSequence(vwr, [
new MetaTuple(rm1, 'f', 4, null),
new MetaTuple(rm2, 'n', 4, null),
new MetaTuple(rm1, new OrderedKey('f'), 4, null),
new MetaTuple(rm2, new OrderedKey('n'), 4, null),
]));
return [c, m1, m2];
}
@@ -725,4 +728,52 @@ suite('CompoundMap', () => {
await t(10, MapLeafSequence);
await t(100, OrderedMetaSequence);
});
test('compound map with values of every type', async () => {
const v = 42;
const kvs = [
// Values
[true, v],
[0, v],
['hello', v],
[new Blob(new Uint8Array([0])), v],
[new Set([true]), v],
[new List([true]), v],
[new Map([[true, true]]), v],
[newStruct('', {field: true}), v],
// Refs of values
[new Ref(true), v],
[new Ref(0), v],
[new Ref('hello'), v],
[new Ref(new Blob(new Uint8Array([0]))), v],
[new Ref(new Set([true])), v],
[new Ref(new List([true])), v],
[new Ref(new Map([[true, true]])), v],
[new Ref(newStruct('', {field: true})), v],
];
let m = new Map(kvs);
for (let i = 1; !m.sequence.isMeta; i++) {
kvs.push([i, v]);
m = await m.set(i, v);
}
assert.strictEqual(kvs.length, m.size);
const [fk, fv] = notNull(await m.first());
assert.strictEqual(true, fk);
assert.strictEqual(v, fv);
for (const kv of kvs) {
assert.isTrue(await m.has(kv[0]));
assert.strictEqual(v, await m.get(kv[0]));
assert.strictEqual(v, kv[1]);
}
while (kvs.length > 0) {
const kv = kvs.shift();
m = await m.delete(kv[0]);
assert.isFalse(await m.has(kv[0]));
assert.strictEqual(kvs.length, m.size);
}
});
});
js/src/map.js
+23 -23
View File
@@ -17,10 +17,17 @@ import {compare, equals} from './compare.js';
import {sha1Size} from './hash.js';
import {getHashOfValue} from './get-hash.js';
import {getTypeOfValue, makeMapType, makeUnionType} from './type.js';
import {MetaTuple, newOrderedMetaSequenceBoundaryChecker, newOrderedMetaSequenceChunkFn} from
'./meta-sequence.js';
import {OrderedSequence, OrderedSequenceCursor, OrderedSequenceIterator} from
'./ordered-sequence.js';
import {
OrderedKey,
MetaTuple,
newOrderedMetaSequenceBoundaryChecker,
newOrderedMetaSequenceChunkFn,
} from './meta-sequence.js';
import {
OrderedSequence,
OrderedSequenceCursor,
OrderedSequenceIterator,
} from './ordered-sequence.js';
import diff from './ordered-sequence-diff.js';
import {ValueBase} from './value.js';
import {Kind} from './noms-kind.js';
@@ -37,17 +44,10 @@ const mapPattern = ((1 << 6) | 0) - 1;
function newMapLeafChunkFn<K: Value, V: Value>(vr: ?ValueReader):
makeChunkFn {
return (items: Array<MapEntry<K, V>>) => {
let indexValue: ?Value = null;
if (items.length > 0) {
indexValue = items[items.length - 1][KEY];
if (indexValue instanceof ValueBase) {
indexValue = new Ref(indexValue);
}
}
const key = new OrderedKey(items.length > 0 ? items[items.length - 1][KEY] : false);
const seq = newMapLeafSequence(vr, items);
const nm = Map.fromSequence(seq);
const mt = new MetaTuple(new Ref(nm), indexValue, items.length, nm);
const mt = new MetaTuple(new Ref(nm), key, seq.length, nm);
return [mt, seq];
};
}
@@ -101,8 +101,8 @@ export default class Map<K: Value, V: Value> extends
}
async has(key: K): Promise<boolean> {
const cursor = await this.sequence.newCursorAt(key);
return cursor.valid && equals(cursor.getCurrentKey(), key);
const cursor = await this.sequence.newCursorAtValue(key);
return cursor.valid && equals(cursor.getCurrentKey().value(), key);
}
async _firstOrLast(last: boolean): Promise<?MapEntry<K, V>> {
@@ -123,7 +123,7 @@ export default class Map<K: Value, V: Value> extends
}
async get(key: K): Promise<?V> {
const cursor = await this.sequence.newCursorAt(key);
const cursor = await this.sequence.newCursorAtValue(key);
if (!cursor.valid) {
return undefined;
}
@@ -146,7 +146,7 @@ export default class Map<K: Value, V: Value> extends
}
iteratorAt(k: K): AsyncIterator<MapEntry<K, V>> {
return new OrderedSequenceIterator(this.sequence.newCursorAt(k));
return new OrderedSequenceIterator(this.sequence.newCursorAtValue(k));
}
_splice(cursor: OrderedSequenceCursor, insert: Array<MapEntry<K, V>>, remove: number):
@@ -160,8 +160,8 @@ export default class Map<K: Value, V: Value> extends
async set(key: K, value: V): Promise<Map<K, V>> {
let remove = 0;
const cursor = await this.sequence.newCursorAt(key, true);
if (cursor.valid && equals(cursor.getCurrentKey(), key)) {
const cursor = await this.sequence.newCursorAtValue(key, true);
if (cursor.valid && equals(cursor.getCurrentKey().value(), key)) {
const entry = cursor.getCurrent();
if (equals(entry[VALUE], value)) {
return this;
@@ -174,8 +174,8 @@ export default class Map<K: Value, V: Value> extends
}
async delete(key: K): Promise<Map<K, V>> {
const cursor = await this.sequence.newCursorAt(key);
if (cursor.valid && equals(cursor.getCurrentKey(), key)) {
const cursor = await this.sequence.newCursorAtValue(key);
if (cursor.valid && equals(cursor.getCurrentKey().value(), key)) {
return this._splice(cursor, [], 1);
}
@@ -196,8 +196,8 @@ export default class Map<K: Value, V: Value> extends
export class MapLeafSequence<K: Value, V: Value> extends
OrderedSequence<K, MapEntry<K, V>> {
getKey(idx: number): K {
return this.items[idx][KEY];
getKey(idx: number): OrderedKey {
return new OrderedKey(this.items[idx][KEY]);
}
getCompareFn(other: OrderedSequence): EqualsFn {
js/src/meta-sequence-test.js
+10 -6
View File
@@ -8,8 +8,12 @@ import {assert} from 'chai';
import {suite, test} from 'mocha';
import List from './list.js';
import {MetaTuple, newOrderedMetaSequenceChunkFn, newIndexedMetaSequenceChunkFn} from
'./meta-sequence.js';
import {
OrderedKey,
MetaTuple,
newOrderedMetaSequenceChunkFn,
newIndexedMetaSequenceChunkFn,
} from './meta-sequence.js';
import Ref from './ref.js';
import Set from './set.js';
import {Kind} from './noms-kind.js';
@@ -18,8 +22,8 @@ suite('MetaSequence', () => {
test('calculate ordered sequence MetaTuple height', async () => {
const set1 = new Set(['bar', 'baz']);
const set2 = new Set(['foo', 'qux', 'zoo']);
const mt1 = new MetaTuple(new Ref(set1), 'baz', 2, set1);
const mt2 = new MetaTuple(new Ref(set2), 'zoo', 3, set2);
const mt1 = new MetaTuple(new Ref(set1), new OrderedKey('baz'), 2, set1);
const mt2 = new MetaTuple(new Ref(set2), new OrderedKey('zoo'), 3, set2);
assert.strictEqual(1, mt1.ref.height);
assert.strictEqual(1, mt2.ref.height);
@@ -36,8 +40,8 @@ suite('MetaSequence', () => {
test('calculate indexed sequence MetaTuple height', async () => {
const list1 = new List(['bar', 'baz']);
const list2 = new List(['foo', 'qux', 'zoo']);
const mt1 = new MetaTuple(new Ref(list1), 2, 2, list1);
const mt2 = new MetaTuple(new Ref(list2), 3, 3, list2);
const mt1 = new MetaTuple(new Ref(list1), new OrderedKey(2), 2, list1);
const mt2 = new MetaTuple(new Ref(list2), new OrderedKey(3), 3, list2);
assert.strictEqual(1, mt1.ref.height);
assert.strictEqual(1, mt2.ref.height);
js/src/meta-sequence.js
+74 -23
View File
@@ -5,10 +5,12 @@
// http://www.apache.org/licenses/LICENSE-2.0
import BuzHashBoundaryChecker from './buzhash-boundary-checker.js';
import {sha1Size} from './hash.js';
import {compare} from './compare.js';
import {default as Hash, sha1Size} from './hash.js';
import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js';
import type {ValueReader, ValueWriter} from './value-store.js';
import type Value from './value.js'; // eslint-disable-line no-unused-vars
import {ValueBase} from './value.js';
import type Collection from './collection.js';
import type {Type} from './type.js';
import {makeListType, makeUnionType, blobType, makeSetType, makeMapType} from './type.js';
@@ -27,15 +29,15 @@ import type {EqualsFn} from './edit-distance.js';
export type MetaSequence = Sequence<MetaTuple>;
export class MetaTuple<K> {
export class MetaTuple<T: Value> {
ref: Ref;
value: K;
key: OrderedKey<T>;
numLeaves: number;
child: ?Collection;
constructor(ref: Ref, value: K, numLeaves: number, child: ?Collection) {
constructor(ref: Ref, key: OrderedKey<T>, numLeaves: number, child: ?Collection) {
this.ref = ref;
this.value = value;
this.key = key;
this.numLeaves = numLeaves;
this.child = child;
}
@@ -54,30 +56,77 @@ export class MetaTuple<K> {
}
}
export class OrderedKey<T: Value> {
isOrderedByValue: boolean;
v: ?T;
h: ?Hash;
constructor(v: T) {
this.v = v;
if (v instanceof ValueBase) {
this.isOrderedByValue = false;
this.h = v.hash;
} else {
this.isOrderedByValue = true;
this.h = null;
}
}
static fromHash(h: Hash): OrderedKey {
const k = Object.create(this.prototype);
k.isOrderedByValue = false;
k.v = null;
k.h = h;
return k;
}
value(): T {
return notNull(this.v);
}
numberValue(): number {
invariant(typeof this.v === 'number');
return this.v;
}
compare(other: OrderedKey): number {
if (this.isOrderedByValue && other.isOrderedByValue) {
return compare(notNull(this.v), notNull(other.v));
}
if (this.isOrderedByValue) {
return -1;
}
if (other.isOrderedByValue) {
return 1;
}
return notNull(this.h).compare(notNull(other.h));
}
}
// The elemTypes of the collection inside the Ref<Collection<?, ?>>
function getCollectionTypes<K>(tuple: MetaTuple<K>): Type[] {
function getCollectionTypes(tuple: MetaTuple): Type[] {
return tuple.ref.type.desc.elemTypes[0].desc.elemTypes;
}
export function newListMetaSequence(vr: ?ValueReader, items: Array<MetaTuple<number>>):
export function newListMetaSequence(vr: ?ValueReader, items: Array<MetaTuple>):
IndexedMetaSequence {
const t = makeListType(makeUnionType(items.map(tuple => getCollectionTypes(tuple)[0])));
return new IndexedMetaSequence(vr, t, items);
}
export function newBlobMetaSequence(vr: ?ValueReader, items: Array<MetaTuple<number>>):
export function newBlobMetaSequence(vr: ?ValueReader, items: Array<MetaTuple>):
IndexedMetaSequence {
return new IndexedMetaSequence(vr, blobType, items);
}
export class IndexedMetaSequence extends IndexedSequence<MetaTuple<number>> {
export class IndexedMetaSequence extends IndexedSequence<MetaTuple> {
_offsets: Array<number>;
constructor(vr: ?ValueReader, t: Type, items: Array<MetaTuple<number>>) {
constructor(vr: ?ValueReader, t: Type, items: Array<MetaTuple>) {
super(vr, t, items);
let cum = 0;
this._offsets = this.items.map(i => {
cum += i.value;
cum += i.key.numberValue();
return cum;
});
}
@@ -100,7 +149,7 @@ export class IndexedMetaSequence extends IndexedSequence<MetaTuple<number>> {
const childRanges = [];
for (let i = 0; i < this.items.length && end > start; i++) {
const cum = this.getOffset(i) + 1;
const seqLength = this.items[i].value;
const seqLength = this.items[i].key.numberValue();
if (start < cum) {
const seqStart = cum - seqLength;
const childStart = start - seqStart;
@@ -166,7 +215,7 @@ export class IndexedMetaSequence extends IndexedSequence<MetaTuple<number>> {
}
export function newMapMetaSequence<K: Value>(vr: ?ValueReader,
tuples: Array<MetaTuple<K>>): OrderedMetaSequence<K> {
tuples: Array<MetaTuple>): OrderedMetaSequence<K> {
const kt = makeUnionType(tuples.map(mt => getCollectionTypes(mt)[0]));
const vt = makeUnionType(tuples.map(mt => getCollectionTypes(mt)[1]));
const t = makeMapType(kt, vt);
@@ -174,15 +223,15 @@ export function newMapMetaSequence<K: Value>(vr: ?ValueReader,
}
export function newSetMetaSequence<K: Value>(vr: ?ValueReader,
tuples: Array<MetaTuple<K>>): OrderedMetaSequence<K> {
tuples: Array<MetaTuple>): OrderedMetaSequence<K> {
const t = makeSetType(makeUnionType(tuples.map(mt => getCollectionTypes(mt)[0])));
return new OrderedMetaSequence(vr, t, tuples);
}
export class OrderedMetaSequence<K: Value> extends OrderedSequence<K, MetaTuple<K>> {
export class OrderedMetaSequence<K: Value> extends OrderedSequence<K, MetaTuple> {
_numLeaves: number;
constructor(vr: ?ValueReader, t: Type, items: Array<MetaTuple<K>>) {
constructor(vr: ?ValueReader, t: Type, items: Array<MetaTuple>) {
super(vr, t, items);
this._numLeaves = items.reduce((l, mt) => l + mt.numLeaves, 0);
}
@@ -217,8 +266,8 @@ export class OrderedMetaSequence<K: Value> extends OrderedSequence<K, MetaTuple<
return mt.getSequenceSync();
}
getKey(idx: number): K {
return this.items[idx].value;
getKey(idx: number): OrderedKey {
return this.items[idx].key;
}
getCompareFn(other: OrderedSequence): EqualsFn {
@@ -241,7 +290,7 @@ export function newOrderedMetaSequenceChunkFn(kind: NomsKind, vr: ?ValueReader):
seq = newSetMetaSequence(vr, tuples);
col = Set.fromSequence(seq);
}
return [new MetaTuple(new Ref(col), last.value, numLeaves, col), seq];
return [new MetaTuple(new Ref(col), last.key, numLeaves, col), seq];
};
}
@@ -259,8 +308,9 @@ export function newIndexedMetaSequenceChunkFn(kind: NomsKind, vr: ?ValueReader,
vw: ?ValueWriter): makeChunkFn {
return (tuples: Array<MetaTuple>) => {
const sum = tuples.reduce((l, mt) => {
invariant(mt.value === mt.numLeaves);
return l + mt.value;
const nv = mt.key.numberValue();
invariant(nv === mt.numLeaves);
return l + nv;
}, 0);
let seq: IndexedMetaSequence;
let col: Collection;
@@ -272,11 +322,12 @@ export function newIndexedMetaSequenceChunkFn(kind: NomsKind, vr: ?ValueReader,
seq = newBlobMetaSequence(vr, tuples);
col = Blob.fromSequence(seq);
}
const key = new OrderedKey(sum);
let mt;
if (vw) {
mt = new MetaTuple(vw.writeValue(col), sum, sum, null);
mt = new MetaTuple(vw.writeValue(col), key, sum, null);
} else {
mt = new MetaTuple(new Ref(col), sum, sum, col);
mt = new MetaTuple(new Ref(col), key, sum, col);
}
return [mt, seq];
};
js/src/ordered-sequence-diff-test.js
+1 -1
View File
@@ -22,7 +22,7 @@ suite('OrderedSequenceCursor', () => {
const newMetaSequenceCursor = nums => {
const lst = new Set(nums);
assert.isTrue(lst.sequence.isMeta);
return lst.sequence.newCursorAt(0);
return lst.sequence.newCursorAt();
};
{
js/src/ordered-sequence-diff.js
+10 -11
View File
@@ -5,7 +5,6 @@
// http://www.apache.org/licenses/LICENSE-2.0
import {invariant} from './assert.js';
import {equals, less} from './compare.js';
import {OrderedSequence, OrderedSequenceCursor} from './ordered-sequence.js';
import {SequenceCursor} from './sequence.js';
import type Value from './value.js'; // eslint-disable-line no-unused-vars
@@ -31,25 +30,25 @@ export default async function diff<K: Value, T>(
while (lastCur.valid && currentCur.valid &&
!lastCur.sequence.getCompareFn(currentCur.sequence)(lastCur.idx, currentCur.idx)) {
const lastKey = lastCur.getCurrentKey(), currentKey = currentCur.getCurrentKey();
if (equals(lastKey, currentKey)) {
modified.push(lastKey);
await Promise.all([lastCur.advance(), currentCur.advance()]);
} else if (less(lastKey, currentKey)) {
removed.push(lastKey);
const compare = currentKey.compare(lastKey);
if (compare < 0) {
added.push(currentKey.value());
await currentCur.advance();
} else if (compare > 0) {
removed.push(lastKey.value());
await lastCur.advance();
} else {
added.push(currentKey);
await currentCur.advance();
modified.push(currentKey.value());
await Promise.all([lastCur.advance(), currentCur.advance()]);
}
}
}
for (; lastCur.valid; await lastCur.advance()) {
removed.push(lastCur.getCurrentKey());
removed.push(lastCur.getCurrentKey().value());
}
for (; currentCur.valid; await currentCur.advance()) {
added.push(currentCur.getCurrentKey());
added.push(currentCur.getCurrentKey().value());
}
return [added, removed, modified];
js/src/ordered-sequence.js
+16 -25
View File
@@ -6,22 +6,31 @@
import {AsyncIterator} from './async-iterator.js';
import type {AsyncIteratorResult} from './async-iterator.js';
import {OrderedKey} from './meta-sequence.js';
import type Value from './value.js'; // eslint-disable-line no-unused-vars
import {invariant, notNull} from './assert.js';
import {compare} from './compare.js';
import search from './binary-search.js';
import type {EqualsFn} from './edit-distance.js';
import Sequence, {SequenceCursor} from './sequence.js';
import {ValueBase} from './value.js';
export class OrderedSequence<K: Value, T> extends Sequence<T> {
// See newCursorAt().
newCursorAtValue(val: ?K, forInsertion: boolean = false, last: boolean = false):
Promise<OrderedSequenceCursor> {
let key;
if (val !== null && val !== undefined) {
key = new OrderedKey(val);
}
return this.newCursorAt(key, forInsertion, last);
}
// Returns:
// -null, if sequence is empty.
// -null, if all values in sequence are < key.
// -cursor positioned at
// -first value, if |key| is null
// -first value >= |key|
async newCursorAt(key: ?K, forInsertion: boolean = false, last: boolean = false):
async newCursorAt(key: ?OrderedKey, forInsertion: boolean = false, last: boolean = false):
Promise<OrderedSequenceCursor> {
let cursor: ?OrderedSequenceCursor = null;
let sequence: ?OrderedSequence = this;
@@ -44,7 +53,7 @@ export class OrderedSequence<K: Value, T> extends Sequence<T> {
/**
* Gets the key used for ordering the sequence at index |idx|.
*/
getKey(idx: number): K { // eslint-disable-line no-unused-vars
getKey(idx: number): OrderedKey { // eslint-disable-line no-unused-vars
throw new Error('override');
}
@@ -55,7 +64,7 @@ export class OrderedSequence<K: Value, T> extends Sequence<T> {
export class OrderedSequenceCursor<T, K: Value> extends
SequenceCursor<T, OrderedSequence> {
getCurrentKey(): K {
getCurrentKey(): OrderedKey {
invariant(this.idx >= 0 && this.idx < this.length);
return this.sequence.getKey(this.idx);
}
@@ -66,8 +75,8 @@ export class OrderedSequenceCursor<T, K: Value> extends
// Moves the cursor to the first value in sequence >= key and returns true.
// If none exists, returns false.
_seekTo(key: K, lastPositionIfNotfound: boolean = false): boolean {
this.idx = search(this.length, getSearchFunction(this.sequence, key));
_seekTo(key: OrderedKey, lastPositionIfNotfound: boolean = false): boolean {
this.idx = search(this.length, i => this.sequence.getKey(i).compare(key));
if (this.idx === this.length && lastPositionIfNotfound) {
invariant(this.idx > 0);
@@ -94,21 +103,3 @@ export class OrderedSequenceIterator<T, K: Value> extends AsyncIterator<T> {
return this._iterator.then(it => it.return());
}
}
function getSearchFunction(sequence: OrderedSequence, key: Value):
(i: number) => number {
if (sequence.isMeta) {
const keyRef = key instanceof ValueBase ? key.hash : null;
return i => {
const sk = sequence.getKey(i);
if (sk instanceof ValueBase) {
if (keyRef) {
return sk.targetHash.compare(keyRef);
}
return 1;
}
return compare(sk, key);
};
}
return i => compare(sequence.getKey(i), key);
}
js/src/ref.js
-1
View File
@@ -18,7 +18,6 @@ import {ValueBase, getChunksOfValue} from './value.js';
export function constructRef(t: Type, targetHash: Hash, height: number): Ref {
invariant(t.kind === Kind.Ref, () => `Not a Ref type: ${describeType(t)}`);
invariant(!targetHash.isEmpty());
invariant(height > 0);
const rv = Object.create(Ref.prototype);
rv._type = t;
rv.targetHash = targetHash;
js/src/set-test.js
+8 -8
View File
@@ -10,7 +10,7 @@ import {suite, setup, teardown, test} from 'mocha';
import Ref from './ref.js';
import Set, {SetLeafSequence} from './set.js';
import type {ValueReadWriter} from './value-store.js';
import {MetaTuple, newSetMetaSequence} from './meta-sequence.js';
import {OrderedKey, MetaTuple, newSetMetaSequence} from './meta-sequence.js';
import {compare, equals} from './compare.js';
import {
assertChunkCountAndType,
@@ -82,11 +82,11 @@ suite('BuildSet', () => {
const newNumberStruct = i => newStruct('', {n: i});
test('Set 1K structs', async () => {
await setTestSuite(10, 'sha1-5cccc0406d9f1102592e9cd63f794c0b508a5ef8', 18, newNumberStruct);
await setTestSuite(10, 'sha1-9d904bcb2b06b0361a73f9ccbdfeca53f081030f', 18, newNumberStruct);
});
test('LONG: Set 4K structs', async () => {
await setTestSuite(12, 'sha1-b9a1403102103acbc3c780a0177278661b49b2e0', 2, newNumberStruct);
await setTestSuite(12, 'sha1-a8f3b3362cde66638e6e1fb8359ad5675b7b5292', 2, newNumberStruct);
});
test('unique keys - strings', async () => {
@@ -124,7 +124,7 @@ suite('BuildSet', () => {
const nums = intSequence(testSetSize);
const structs = nums.map(n => newStruct('num', {n}));
const s = new Set(structs);
assert.strictEqual('sha1-4e6c7eb66b2cc252611a38cb06eed751d2bdf3c3', s.hash.toString());
assert.strictEqual('sha1-4f207ac30b7922b5c2181769ce827d9a3cbc8b9b', s.hash.toString());
const height = deriveCollectionHeight(s);
assert.isTrue(height > 0);
assert.strictEqual(height, s.sequence.items[0].ref.height);
@@ -139,7 +139,7 @@ suite('BuildSet', () => {
const nums = intSequence(testSetSize);
const refs = nums.map(n => new Ref(newStruct('num', {n})));
const s = new Set(refs);
assert.strictEqual('sha1-c8396816c8fb961939cc7c9dcf8011efe9040103', s.hash.toString());
assert.strictEqual('sha1-084c21ccfb81d18d1b6da8bae6b5de1f52d1bd00', s.hash.toString());
const height = deriveCollectionHeight(s);
assert.isTrue(height > 0);
// height + 1 because the leaves are Ref values (with height 1).
@@ -231,7 +231,7 @@ suite('BuildSet', () => {
vals.sort(compare);
const s = new Set(vals);
assert.strictEqual('sha1-4af5578e97bbe7ae1326087f004e828716ac7b85', s.hash.toString());
assert.strictEqual('sha1-b40b9337b0a87c1c0233a415e28bd5294cab6abb', s.hash.toString());
const height = deriveCollectionHeight(s);
assert.isTrue(height > 0);
assert.strictEqual(height, s.sequence.items[0].ref.height);
@@ -368,7 +368,7 @@ suite('CompoundSet', () => {
for (let i = 0; i < values.length; i += 2) {
const s = new Set([values[i], values[i + 1]]);
const r = vwr.writeValue(s);
tuples.push(new MetaTuple(r, values[i + 1], 2, null));
tuples.push(new MetaTuple(r, new OrderedKey(values[i + 1]), 2, null));
}
let last: ?Set = null;
@@ -377,7 +377,7 @@ suite('CompoundSet', () => {
for (let i = 0; i < tuples.length; i += 2) {
last = Set.fromSequence(newSetMetaSequence(vwr, [tuples[i], tuples[i + 1]]));
const r = vwr.writeValue(last);
next.push(new MetaTuple(r, tuples[i + 1].value,
next.push(new MetaTuple(r, tuples[i + 1].key,
tuples[i].numLeaves + tuples[i + 1].numLeaves, null));
}
js/src/set.js
+17 -21
View File
@@ -9,15 +9,18 @@ import Ref from './ref.js';
import type {ValueReader} from './value-store.js';
import type {BoundaryChecker, makeChunkFn} from './sequence-chunker.js';
import type Value from './value.js'; // eslint-disable-line no-unused-vars
import {ValueBase} from './value.js';
import {AsyncIterator} from './async-iterator.js';
import {chunkSequence, chunkSequenceSync} from './sequence-chunker.js';
import Collection from './collection.js';
import {compare, equals} from './compare.js';
import {getHashOfValue} from './get-hash.js';
import {invariant} from './assert.js';
import {MetaTuple, newOrderedMetaSequenceBoundaryChecker, newOrderedMetaSequenceChunkFn} from
'./meta-sequence.js';
import {
OrderedKey,
MetaTuple,
newOrderedMetaSequenceBoundaryChecker,
newOrderedMetaSequenceChunkFn,
} from './meta-sequence.js';
import {OrderedSequence, OrderedSequenceCursor, OrderedSequenceIterator} from
'./ordered-sequence.js';
import diff from './ordered-sequence-diff.js';
@@ -33,17 +36,10 @@ const setPattern = ((1 << 6) | 0) - 1;
function newSetLeafChunkFn<T:Value>(vr: ?ValueReader): makeChunkFn {
return (items: Array<T>) => {
let indexValue: ?(T | Ref) = null;
if (items.length > 0) {
indexValue = items[items.length - 1];
if (indexValue instanceof ValueBase) {
indexValue = new Ref(indexValue);
}
}
const key = new OrderedKey(items.length > 0 ? items[items.length - 1] : false);
const seq = newSetLeafSequence(vr, items);
const ns = Set.fromSequence(seq);
const mt = new MetaTuple(new Ref(ns), indexValue, items.length, ns);
const mt = new MetaTuple(new Ref(ns), key, items.length, ns);
return [mt, seq];
};
}
@@ -80,8 +76,8 @@ export default class Set<T: Value> extends Collection<OrderedSequence> {
}
async has(key: T): Promise<boolean> {
const cursor = await this.sequence.newCursorAt(key);
return cursor.valid && equals(cursor.getCurrentKey(), key);
const cursor = await this.sequence.newCursorAtValue(key);
return cursor.valid && equals(cursor.getCurrentKey().value(), key);
}
async _firstOrLast(last: boolean): Promise<?T> {
@@ -111,7 +107,7 @@ export default class Set<T: Value> extends Collection<OrderedSequence> {
}
iteratorAt(v: T): AsyncIterator<T> {
return new OrderedSequenceIterator(this.sequence.newCursorAt(v));
return new OrderedSequenceIterator(this.sequence.newCursorAtValue(v));
}
_splice(cursor: OrderedSequenceCursor, insert: Array<T>, remove: number):
@@ -124,8 +120,8 @@ export default class Set<T: Value> extends Collection<OrderedSequence> {
}
async add(value: T): Promise<Set<T>> {
const cursor = await this.sequence.newCursorAt(value, true);
if (cursor.valid && equals(cursor.getCurrentKey(), value)) {
const cursor = await this.sequence.newCursorAtValue(value, true);
if (cursor.valid && equals(cursor.getCurrentKey().value(), value)) {
return this;
}
@@ -133,8 +129,8 @@ export default class Set<T: Value> extends Collection<OrderedSequence> {
}
async delete(value: T): Promise<Set<T>> {
const cursor = await this.sequence.newCursorAt(value);
if (cursor.valid && equals(cursor.getCurrentKey(), value)) {
const cursor = await this.sequence.newCursorAtValue(value);
if (cursor.valid && equals(cursor.getCurrentKey().value(), value)) {
return this._splice(cursor, [], 1);
}
@@ -169,8 +165,8 @@ export default class Set<T: Value> extends Collection<OrderedSequence> {
}
export class SetLeafSequence<K: Value> extends OrderedSequence<K, K> {
getKey(idx: number): K {
return this.items[idx];
getKey(idx: number): OrderedKey {
return new OrderedKey(this.items[idx]);
}
getCompareFn(other: OrderedSequence): EqualsFn {
js/src/value-decoder.js
+3 -2
View File
@@ -19,7 +19,7 @@ import {
StructDesc,
Type,
} from './type.js';
import {MetaTuple} from './meta-sequence.js';
import {OrderedKey, MetaTuple} from './meta-sequence.js';
import {invariant} from './assert.js';
import {isPrimitiveKind, kindToString, Kind} from './noms-kind.js';
import List, {ListLeafSequence} from './list.js';
@@ -126,8 +126,9 @@ export default class ValueDecoder {
for (let i = 0; i < count; i++) {
const ref = this.readValue();
const v = this.readValue();
const key = v instanceof Ref ? OrderedKey.fromHash(v.targetHash) : new OrderedKey(v);
const numLeaves = this._r.readUint64();
data.push(new MetaTuple(ref, v, numLeaves, null));
data.push(new MetaTuple(ref, key, numLeaves, null));
}
return data;
js/src/value-encoder.js
+11 -4
View File
@@ -7,7 +7,7 @@
import Blob, {BlobLeafSequence} from './blob.js';
import List, {ListLeafSequence} from './list.js';
import Map, {MapLeafSequence} from './map.js';
import Ref from './ref.js';
import Ref, {constructRef} from './ref.js';
import Sequence from './sequence.js';
import Set, {SetLeafSequence} from './set.js';
import Struct, {StructMirror} from './struct.js';
@@ -17,9 +17,9 @@ import type {NomsWriter} from './codec.js';
import type {ValueWriter} from './value-store.js';
import type {primitive} from './primitives.js';
import {MetaTuple} from './meta-sequence.js';
import {StructDesc, Type, getTypeOfValue} from './type.js';
import {boolType, getTypeOfValue, makeRefType, StructDesc, Type} from './type.js';
import {describeTypeOfValue} from './encode-human-readable.js';
import {invariant} from './assert.js';
import {invariant, notNull} from './assert.js';
import {isPrimitiveKind, kindToString, Kind} from './noms-kind.js';
type primitiveOrArray = primitive | Array<primitiveOrArray>;
@@ -116,7 +116,14 @@ export default class ValueEncoder {
this._vw.writeValue(child);
}
this.writeValue(tuple.ref);
this.writeValue(tuple.value);
let val = tuple.key.v;
if (!tuple.key.isOrderedByValue) {
// See https://github.com/attic-labs/noms/issues/1688#issuecomment-227528987
val = constructRef(makeRefType(boolType), notNull(tuple.key.h), 0);
} else {
val = notNull(val);
}
this.writeValue(val);
this._w.writeUint64(tuple.numLeaves);
}
return true;
js/src/version.js
+1 -1
View File
@@ -4,4 +4,4 @@
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
export default '1';
export default '2';