Merge pull request #1070 from arv/compound-blob

JS: Implement NomsBlob
This commit is contained in:
Erik Arvidsson
2016-03-16 13:10:43 -07:00
11 changed files with 214 additions and 84 deletions

View File

@@ -18,8 +18,7 @@ for (let i = 0 ; i < chars.length; i++) {
lookup[chars.charCodeAt(i)] = i;
}
export function encode(b: ArrayBuffer): string {
const bytes = new Uint8Array(b);
export function encode(bytes: Uint8Array): string {
const len = bytes.length;
let base64 = '';
@@ -39,7 +38,7 @@ export function encode(b: ArrayBuffer): string {
return base64;
}
export function decode(s: string): ArrayBuffer {
export function decode(s: string): Uint8Array {
let bufferLength = s.length * 0.75;
const len = s.length;
@@ -50,8 +49,7 @@ export function decode(s: string): ArrayBuffer {
}
}
const arraybuffer = new ArrayBuffer(bufferLength);
const bytes = new Uint8Array(arraybuffer);
const bytes = new Uint8Array(bufferLength);
let p = 0;
for (let i = 0; i < len; i += 4) {
@@ -65,5 +63,5 @@ export function decode(s: string): ArrayBuffer {
bytes[p++] = ((encoded3 & 3) << 6) | (encoded4 & 63);
}
return arraybuffer;
return bytes;
}

View File

@@ -4,30 +4,30 @@ import {suite, test} from 'mocha';
import {assert} from 'chai';
import {encode, decode} from './base64.js';
function arrayBufferFromString(s: string): ArrayBuffer {
function uint8ArrayFromString(s: string): Uint8Array {
const ta = new Uint8Array(s.length);
for (let i = 0; i < s.length; i++) {
ta[i] = s.charCodeAt(i);
}
return ta.buffer;
return ta;
}
suite('base64', () => {
test('encode', () => {
assert.deepEqual(encode(arrayBufferFromString('Hello world')), 'SGVsbG8gd29ybGQ=');
assert.deepEqual(encode(arrayBufferFromString('Man')), 'TWFu');
assert.deepEqual(encode(arrayBufferFromString('Ma')), 'TWE=');
assert.deepEqual(encode(arrayBufferFromString('M')), 'TQ==');
assert.deepEqual(encode(arrayBufferFromString('')), '');
assert.deepEqual(encode(arrayBufferFromString('Hello worlds!')), 'SGVsbG8gd29ybGRzIQ==');
assert.deepEqual(encode(uint8ArrayFromString('Hello world')), 'SGVsbG8gd29ybGQ=');
assert.deepEqual(encode(uint8ArrayFromString('Man')), 'TWFu');
assert.deepEqual(encode(uint8ArrayFromString('Ma')), 'TWE=');
assert.deepEqual(encode(uint8ArrayFromString('M')), 'TQ==');
assert.deepEqual(encode(uint8ArrayFromString('')), '');
assert.deepEqual(encode(uint8ArrayFromString('Hello worlds!')), 'SGVsbG8gd29ybGRzIQ==');
});
test('decode', () => {
assert.deepEqual(decode('TWFu'), arrayBufferFromString('Man'));
assert.deepEqual(decode('TWE='), arrayBufferFromString('Ma'));
assert.deepEqual(decode('TQ=='), arrayBufferFromString('M'));
assert.deepEqual(decode(''), arrayBufferFromString(''));
assert.deepEqual(decode('SGVsbG8gd29ybGQ='), arrayBufferFromString('Hello world'));
assert.deepEqual(decode('SGVsbG8gd29ybGRzIQ=='), arrayBufferFromString('Hello worlds!'));
assert.deepEqual(decode('TWFu'), uint8ArrayFromString('Man'));
assert.deepEqual(decode('TWE='), uint8ArrayFromString('Ma'));
assert.deepEqual(decode('TQ=='), uint8ArrayFromString('M'));
assert.deepEqual(decode(''), uint8ArrayFromString(''));
assert.deepEqual(decode('SGVsbG8gd29ybGQ='), uint8ArrayFromString('Hello world'));
assert.deepEqual(decode('SGVsbG8gd29ybGRzIQ=='), uint8ArrayFromString('Hello worlds!'));
});
});

63
js/src/blob.js Normal file
View File

@@ -0,0 +1,63 @@
// @flow
import {Collection} from './collection.js';
import {IndexedSequence} from './indexed_sequence.js';
import {SequenceCursor} from './sequence.js';
import {invariant} from './assert.js';
import type {ChunkStore} from './chunk_store.js';
import {blobType} from './type.js';
import type {uint8} from './primitives.js';
export class NomsBlob extends Collection<IndexedSequence<uint8>> {
constructor(sequence: IndexedSequence<uint8>) {
super(blobType, sequence);
}
getReader(): BlobReader {
return new BlobReader(this.sequence.newCursorAt(0));
}
}
export class BlobReader {
_cursor: Promise<SequenceCursor<number, IndexedSequence<number>>>;
_lock: boolean;
constructor(cursor: Promise<SequenceCursor<number, IndexedSequence<number>>>) {
this._cursor = cursor;
this._lock = false;
}
async read(): Promise<{done: boolean, value?: Uint8Array}> {
invariant(!this._lock, 'cannot read without completing current read');
this._lock = true;
const cur = await this._cursor;
if (!cur.valid) {
return {done: true};
}
const arr = cur.sequence.items;
await cur.advanceChunk();
// No more awaits after this, so we can't be interrupted.
this._lock = false;
invariant(arr instanceof Uint8Array);
return {done: false, value: arr};
}
}
export class BlobLeafSequence extends IndexedSequence<uint8> {
constructor(cs: ChunkStore, items: Uint8Array) {
// $FlowIssue: The super class expects Array<T> but we sidestep that.
super(cs, blobType, items);
}
getOffset(idx: number): number {
return idx;
}
}
export function newBlob(data: Uint8Array, cs: ChunkStore): NomsBlob {
// TODO: Chunk it!
return new NomsBlob(new BlobLeafSequence(cs, data));
}

View File

@@ -1,5 +1,6 @@
// @flow
import {NomsBlob, newBlob} from './blob.js';
import Chunk from './chunk.js';
import Ref from './ref.js';
import Struct from './struct.js';
@@ -16,6 +17,7 @@ import {lookupPackage, Package, readPackage} from './package.js';
import {NomsMap, MapLeafSequence} from './map.js';
import {setDecodeNomsValue} from './read_value.js';
import {NomsSet, SetLeafSequence} from './set.js';
import {IndexedMetaSequence} from './meta_sequence.js';
const typedTag = 't ';
const blobTag = 'b ';
@@ -128,9 +130,8 @@ class JsonArrayReader {
throw new Error('Unreachable');
}
readBlob(): Promise<ArrayBuffer> {
const s = this.readString();
return Promise.resolve(decodeBase64(s));
readBlob(): NomsBlob {
return newBlob(decodeBase64(this.readString()), this._cs);
}
readSequence(t: Type, pkg: ?Package): Array<any> {
@@ -227,12 +228,16 @@ class JsonArrayReader {
readValueWithoutTag(t: Type, pkg: ?Package = null): any {
// TODO: Verify read values match tagged kinds.
switch (t.kind) {
case Kind.Blob:
case Kind.Blob: {
const isMeta = this.readBool();
// https://github.com/attic-labs/noms/issues/798
invariant(!isMeta, 'CompoundBlob not supported');
if (isMeta) {
const r2 = new JsonArrayReader(this.readArray(), this._cs);
const sequence = r2.readMetaSequence(t, pkg);
invariant(sequence instanceof IndexedMetaSequence);
return new NomsBlob(sequence);
}
return this.readBlob();
}
case Kind.Bool:
return this.readBool();
case Kind.Float32:
@@ -419,9 +424,8 @@ function decodeNomsValue(chunk: Chunk, cs: ChunkStore): Promise<any> {
const reader = new JsonArrayReader(payload, cs);
return reader.readTopLevelValue();
}
case blobTag: {
return Promise.resolve(chunk.data.buffer.slice(2));
}
case blobTag:
return Promise.resolve(newBlob(new Uint8Array(chunk.data.buffer, 2), cs));
default:
throw new Error('Not implemented');
}

View File

@@ -1,5 +1,7 @@
// @flow
import {encode as encodeBase64} from './base64.js';
import {NomsBlob, newBlob} from './blob.js';
import Chunk from './chunk.js';
import MemoryStore from './memory_store.js';
import Ref from './ref.js';
@@ -9,8 +11,8 @@ import type {float64, int32, int64, uint8, uint16, uint32, uint64} from './primi
import type {TypeDesc} from './type.js';
import {assert} from 'chai';
import {decodeNomsValue, JsonArrayReader} from './decode.js';
import {Field, makeCompoundType, makeEnumType, makePrimitiveType, makeStructType, makeType, Type,}
from './type.js';
import {Field, makeCompoundType, makeEnumType, makePrimitiveType, makeStructType, makeType, Type,
blobType} from './type.js';
import {IndexedMetaSequence, MetaTuple} from './meta_sequence.js';
import {invariant, notNull} from './assert.js';
import {Kind} from './noms_kind.js';
@@ -24,6 +26,14 @@ import type {Value} from './value.js';
import {writeValue} from './encode.js';
suite('Decode', () => {
function stringToUint8Array(s): Uint8Array {
const bytes = new Uint8Array(s.length);
for (let i = 0; i < s.length; i++) {
bytes[i] = s.charCodeAt(i);
}
return bytes;
}
test('read', async () => {
const ms = new MemoryStore();
const a = [1, 'hi', true];
@@ -87,9 +97,6 @@ suite('Decode', () => {
await doTest(1e20, [Kind.Float64, '1e+20']);
await doTest('hi', [Kind.String, 'hi']);
const blob = new Uint8Array([0x00, 0x01]).buffer;
await doTest(blob, [Kind.Blob, false, 'AAE=']);
});
test('read list of int 32', async () => {
@@ -464,19 +471,13 @@ suite('Decode', () => {
assert.strictEqual(1, await commit.get('value'));
});
test('top level blob', async () => {
function stringToBuffer(s) {
const bytes = new Uint8Array(s.length);
for (let i = 0; i < s.length; i++) {
bytes[i] = s.charCodeAt(i);
}
return bytes.buffer;
}
test('out of line blob', async () => {
const chunk = Chunk.fromString('b hi');
const v = await decodeNomsValue(chunk, new MemoryStore());
assert.equal(2, v.byteLength);
assert.deepEqual(stringToBuffer('hi'), v);
const blob = await decodeNomsValue(chunk, new MemoryStore());
const r = await blob.getReader().read();
assert.isFalse(r.done);
assert.equal(2, r.value.byteLength);
assert.deepEqual(stringToUint8Array('hi'), r.value);
const data = new Uint8Array(2 + 256);
data[0] = 'b'.charCodeAt(0);
@@ -488,8 +489,49 @@ suite('Decode', () => {
}
const chunk2 = new Chunk(data);
const v2 = await decodeNomsValue(chunk2, new MemoryStore());
assert.equal(bytes.buffer.byteLength, v2.byteLength);
assert.deepEqual(bytes.buffer, v2);
const blob2 = await decodeNomsValue(chunk2, new MemoryStore());
const r2 = await blob2.getReader().read();
assert.isFalse(r2.done);
assert.equal(bytes.length, r2.value.length);
assert.deepEqual(bytes, r2.value);
});
test('inline blob', async () => {
const ms = new MemoryStore();
const a = [
Kind.List, Kind.Blob, false,
[false, encodeBase64(stringToUint8Array('hello')),
false, encodeBase64(stringToUint8Array('world'))],
];
const r = new JsonArrayReader(a, ms);
const v: NomsList<NomsBlob> = await r.readTopLevelValue();
invariant(v instanceof NomsList);
assert.strictEqual(2, v.length);
const [b1, b2] = [await v.get(0), await v.get(1)];
assert.deepEqual({done: false, value: stringToUint8Array('hello')},
await b1.getReader().read());
assert.deepEqual({done: false, value: stringToUint8Array('world')},
await b2.getReader().read());
});
test('compound blob', async () => {
const ms = new MemoryStore();
const r1 = writeValue(newBlob(stringToUint8Array('hi'), ms), blobType, ms);
const r2 = writeValue(newBlob(stringToUint8Array('world'), ms), blobType, ms);
const a = [Kind.Blob, true, [r1.ref.toString(), '2', r2.ref.toString(), '5']];
const r = new JsonArrayReader(a, ms);
const v: NomsBlob = await r.readTopLevelValue();
invariant(v instanceof NomsBlob);
const reader = v.getReader();
assert.deepEqual(await reader.read(), {done: false, value: stringToUint8Array('hi')});
// console.log(stringToUint8Array('world'));
const x = await reader.read();
// console.log(x);
assert.deepEqual(x, {done: false, value: stringToUint8Array('world')});
assert.deepEqual(await reader.read(), {done: true});
});
});

View File

@@ -16,6 +16,7 @@ import {MapLeafSequence, NomsMap} from './map.js';
import {NomsSet, SetLeafSequence} from './set.js';
import {Sequence} from './sequence.js';
import {setEncodeNomsValue} from './get_ref.js';
import {NomsBlob, BlobLeafSequence} from './blob.js';
const typedTag = 't ';
@@ -116,14 +117,15 @@ class JsonArrayWriter {
writeValue(v: any, t: Type, pkg: ?Package) {
switch (t.kind) {
case Kind.Blob:
this.write(false);
// TODO: When CompoundBlob is implemented...
// invariant(v instanceof Sequence);
// if (this.maybeWriteMetaSequence(v, t, pkg)) {
// break;
// }
invariant(v instanceof NomsBlob || v instanceof Sequence);
const sequence: Sequence = v instanceof NomsBlob ? v.sequence : v;
this.writeBlob(v);
if (this.maybeWriteMetaSequence(sequence, t, pkg)) {
break;
}
invariant(sequence instanceof BlobLeafSequence);
this.writeBlob(sequence);
break;
case Kind.Bool:
case Kind.String:
@@ -322,8 +324,10 @@ class JsonArrayWriter {
}
}
writeBlob(v: ArrayBuffer) {
this.write(encodeBase64(v));
writeBlob(seq: BlobLeafSequence) {
// HACK: The items property is declared as Array<T> in Flow.
invariant(seq.items instanceof Uint8Array);
this.write(encodeBase64(seq.items));
}
writeStruct(s: Struct, type: Type, typeDef: Type, pkg: Package) {
@@ -386,21 +390,24 @@ function encodeEmbeddedNomsValue(v: any, t: Type, cs: ?ChunkStore): Chunk {
// Top level blobs are not encoded using JSON but prefixed with 'b ' followed
// by the raw bytes.
function encodeTopLevelBlob(v: ArrayBuffer): Chunk {
const data = new Uint8Array(2 + v.byteLength);
const view = new DataView(v);
function encodeTopLevelBlob(sequence: BlobLeafSequence): Chunk {
const arr = sequence.items;
const data = new Uint8Array(2 + arr.length);
data[0] = 98; // 'b'
data[1] = 32; // ' '
for (let i = 0; i < view.byteLength; i++) {
data[i + 2] = view.getUint8(i);
for (let i = 0; i < arr.length; i++) {
data[i + 2] = arr[i];
}
return new Chunk(data);
}
function encodeNomsValue(v: any, t: Type, cs: ?ChunkStore): Chunk {
if (t.kind === Kind.Blob) {
invariant(v instanceof ArrayBuffer);
return encodeTopLevelBlob(v);
invariant(v instanceof NomsBlob || v instanceof Sequence);
const sequence: BlobLeafSequence = v instanceof NomsBlob ? v.sequence : v;
if (!sequence.isMeta) {
return encodeTopLevelBlob(sequence);
}
}
return encodeEmbeddedNomsValue(v, t, cs);
}

View File

@@ -18,6 +18,7 @@ import {MapLeafSequence, NomsMap} from './map.js';
import {NomsSet, SetLeafSequence} from './set.js';
import {Package, registerPackage} from './package.js';
import {writeValue} from './encode.js';
import {newBlob} from './blob.js';
suite('Encode', () => {
test('write primitives', () => {
@@ -53,7 +54,8 @@ suite('Encode', () => {
test('write simple blob', () => {
const ms = new MemoryStore();
const w = new JsonArrayWriter(ms);
w.writeTopLevel(makePrimitiveType(Kind.Blob), new Uint8Array([0x00, 0x01]).buffer);
const blob = newBlob(new Uint8Array([0x00, 0x01]), ms);
w.writeTopLevel(makePrimitiveType(Kind.Blob), blob);
assert.deepEqual([Kind.Blob, false, 'AAE='], w.array);
});
@@ -371,19 +373,20 @@ suite('Encode', () => {
});
test('top level blob', () => {
function stringToBuffer(s) {
function stringToUint8Array(s) {
const bytes = new Uint8Array(s.length);
for (let i = 0; i < s.length; i++) {
bytes[i] = s.charCodeAt(i);
}
return bytes.buffer;
return bytes;
}
const ms = new MemoryStore();
const blob = stringToBuffer('hi');
const blob = newBlob(stringToUint8Array('hi'), ms);
const chunk = encodeNomsValue(blob, makePrimitiveType(Kind.Blob), ms);
assert.equal(4, chunk.data.length);
assert.deepEqual(stringToBuffer('b hi'), chunk.data.buffer);
assert.deepEqual(stringToUint8Array('b hi'), chunk.data);
const buffer2 = new ArrayBuffer(2 + 256);
const view = new DataView(buffer2);
@@ -394,7 +397,7 @@ suite('Encode', () => {
bytes[i] = i;
view.setUint8(2 + i, i);
}
const blob2 = bytes.buffer;
const blob2 = newBlob(bytes, ms);
const chunk2 = encodeNomsValue(blob2, makePrimitiveType(Kind.Blob), ms);
assert.equal(buffer2.byteLength, chunk2.data.buffer.byteLength);
assert.deepEqual(buffer2, chunk2.data.buffer);

View File

@@ -49,7 +49,6 @@ export class IndexedMetaSequence extends IndexedSequence<MetaTuple<number>> {
constructor(cs: ?ChunkStore, type: Type, items: Array<MetaTuple<number>>) {
super(cs, type, items);
this.isMeta = true;
this.offsets = [];
let cum = 0;
for (let i = 0; i < items.length; i++) {
@@ -59,6 +58,10 @@ export class IndexedMetaSequence extends IndexedSequence<MetaTuple<number>> {
}
}
get isMeta(): boolean {
return true;
}
range(start: number, end: number): Promise<Array<any>> {
invariant(start >= 0 && end >= 0 && end >= start);
@@ -117,9 +120,8 @@ export class IndexedMetaSequence extends IndexedSequence<MetaTuple<number>> {
}
export class OrderedMetaSequence<K: valueOrPrimitive> extends OrderedSequence<K, MetaTuple<K>> {
constructor(cs: ?ChunkStore, type: Type, items: Array<MetaTuple>) {
super(cs, type, items);
this.isMeta = true;
get isMeta(): boolean {
return true;
}
getChildSequence(idx: number): Promise<?Sequence> {
@@ -142,10 +144,9 @@ export function newMetaSequenceFromData(cs: ChunkStore, type: Type, tuples: Arra
case Kind.Map:
case Kind.Set:
return new OrderedMetaSequence(cs, type, tuples);
case Kind.Blob:
case Kind.List:
return new IndexedMetaSequence(cs, type, tuples);
case Kind.Blob:
throw new Error('Not implemented');
default:
throw new Error('Not reached');
}
@@ -205,4 +206,3 @@ export function newIndexedMetaSequenceBoundaryChecker(): BoundaryChecker<MetaTup
(mt: MetaTuple) => mt.ref.digest
);
}

View File

@@ -2,6 +2,7 @@
export {AsyncIterator} from './async_iterator.js';
export {DataStore, newCommit} from './datastore.js';
export {NomsBlob, BlobReader} from './blob.js';
export {decodeNomsValue} from './decode.js';
export {default as Chunk} from './chunk.js';
export {default as HttpStore} from './http_store.js';

View File

@@ -10,14 +10,16 @@ import {ValueBase} from './value.js';
export class Sequence<T> extends ValueBase {
cs: ?ChunkStore;
items: Array<T>;
isMeta: boolean;
constructor(cs: ?ChunkStore, type: Type, items: Array<T>) {
super(type);
this.cs = cs;
this.items = items;
this.isMeta = false;
}
get isMeta(): boolean {
return false;
}
getChildSequence(idx: number): // eslint-disable-line no-unused-vars
@@ -30,7 +32,7 @@ export class Sequence<T> extends ValueBase {
}
}
export class SequenceCursor<T, S:Sequence> {
export class SequenceCursor<T, S: Sequence> {
parent: ?SequenceCursor;
sequence: S;
idx: number;
@@ -49,7 +51,7 @@ export class SequenceCursor<T, S:Sequence> {
}
get length(): number {
return this.sequence.items.length;
return this.sequence.length;
}
getItem(idx: number): T {
@@ -86,6 +88,10 @@ export class SequenceCursor<T, S:Sequence> {
return this._advanceMaybeAllowPastEnd(true);
}
/**
* Advances the cursor in the local chunk and returns false if advancing would advance past the
* end.
*/
advanceLocal(): boolean {
if (this.idx < this.length - 1) {
this.idx++;
@@ -117,6 +123,11 @@ export class SequenceCursor<T, S:Sequence> {
return false;
}
advanceChunk(): Promise<boolean> {
this.idx = this.length - 1;
return this._advanceMaybeAllowPastEnd(true);
}
retreat(): Promise<boolean> {
return this._retreatMaybeAllowBeforeStart(true);
}
@@ -169,7 +180,7 @@ export class SequenceCursor<T, S:Sequence> {
}
}
export class SequenceIterator<T, S:Sequence> extends AsyncIterator<T> {
export class SequenceIterator<T, S: Sequence> extends AsyncIterator<T> {
_cursor: SequenceCursor<T, S>;
_advance: Promise<boolean>;
_closed: boolean;

View File

@@ -417,6 +417,7 @@ export const int64Type = makePrimitiveType(Kind.Int64);
export const float32Type = makePrimitiveType(Kind.Float32);
export const float64Type = makePrimitiveType(Kind.Float64);
export const stringType = makePrimitiveType(Kind.String);
export const blobType = makePrimitiveType(Kind.Blob);
export const typeType = makePrimitiveType(Kind.Type);
export const packageType = makePrimitiveType(Kind.Package);