JS: Support encoding and decoding embedded blobs

Embedded blobs use base64 encoded strings

js/src/base64.js

@@ -0,0 +1,69 @@
+/* @flow */
+
+'use strict';
+
+// Based on https://github.com/niklasvh/base64-arraybuffer
+//
+// base64-arraybuffer
+// https://github.com/niklasvh/base64-arraybuffer
+//
+// Copyright (c) 2012 Niklas von Hertzen
+// Licensed under the MIT license.
+
+const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+
+// Build charCode -> index
+const lookup: Uint8Array = new Uint8Array(256);
+for (let i = 0 ; i < chars.length; i++) {
+  lookup[chars.charCodeAt(i)] = i;
+}
+
+export function encode(b: ArrayBuffer): string {
+  let bytes = new Uint8Array(b);
+  let len = bytes.length;
+  let base64 = '';
+
+  for (let i = 0; i < len; i += 3) {
+    base64 += chars[bytes[i] >> 2];
+    base64 += chars[((bytes[i] & 3) << 4) | (bytes[i + 1] >> 4)];
+    base64 += chars[((bytes[i + 1] & 15) << 2) | (bytes[i + 2] >> 6)];
+    base64 += chars[bytes[i + 2] & 63];
+  }
+
+  if (len % 3 === 2) {
+    base64 = base64.substring(0, base64.length - 1) + '=';
+  } else if (len % 3 === 1) {
+    base64 = base64.substring(0, base64.length - 2) + '==';
+  }
+
+  return base64;
+}
+
+export function decode(s: string): ArrayBuffer {
+  let bufferLength = s.length * 0.75;
+  let len = s.length;
+
+  if (s[len - 1] === '=') {
+    bufferLength--;
+    if (s[len - 2] === '=') {
+      bufferLength--;
+    }
+  }
+
+  let arraybuffer = new ArrayBuffer(bufferLength);
+  let bytes = new Uint8Array(arraybuffer);
+  let p = 0;
+
+  for (let i = 0; i < len; i += 4) {
+    let encoded1 = lookup[s.charCodeAt(i)];
+    let encoded2 = lookup[s.charCodeAt(i + 1)];
+    let encoded3 = lookup[s.charCodeAt(i + 2)];
+    let encoded4 = lookup[s.charCodeAt(i + 3)];
+
+    bytes[p++] = (encoded1 << 2) | (encoded2 >> 4);
+    bytes[p++] = ((encoded2 & 15) << 4) | (encoded3 >> 2);
+    bytes[p++] = ((encoded3 & 3) << 6) | (encoded4 & 63);
+  }
+
+  return arraybuffer;
+}

js/src/base64_test.js

@@ -0,0 +1,33 @@
+/* @flow */
+
+import {suite, test} from 'mocha';
+import {assert} from 'chai';
+import {encode, decode} from './base64.js';
+
+function arrayBufferFromString(s: string): ArrayBuffer {
+  let ta = new Uint8Array(s.length);
+  for (let i = 0; i < s.length; i++) {
+    ta[i] = s.charCodeAt(i);
+  }
+  return ta.buffer;
+}
+
+suite('base64', () => {
+  test('encode', () => {
+    assert.deepEqual(encode(arrayBufferFromString('Hello world')), 'SGVsbG8gd29ybGQ=');
+    assert.deepEqual(encode(arrayBufferFromString('Man')), 'TWFu');
+    assert.deepEqual(encode(arrayBufferFromString('Ma')), 'TWE=');
+    assert.deepEqual(encode(arrayBufferFromString('M')), 'TQ==');
+    assert.deepEqual(encode(arrayBufferFromString('')), '');
+    assert.deepEqual(encode(arrayBufferFromString('Hello worlds!')), 'SGVsbG8gd29ybGRzIQ==');
+  });
+
+  test('decode', () => {
+    assert.deepEqual(decode('TWFu'), arrayBufferFromString('Man'));
+    assert.deepEqual(decode('TWE='), arrayBufferFromString('Ma'));
+    assert.deepEqual(decode('TQ=='), arrayBufferFromString('M'));
+    assert.deepEqual(decode(''), arrayBufferFromString(''));
+    assert.deepEqual(decode('SGVsbG8gd29ybGQ='), arrayBufferFromString('Hello world'));
+    assert.deepEqual(decode('SGVsbG8gd29ybGRzIQ=='), arrayBufferFromString('Hello worlds!'));
+  });
+});

js/src/browser/fetch.js

@@ -6,7 +6,7 @@ type FetchOptions = {
   headers?: {[key: string]: string}
 };
 
-function fetch<T>(url: string, responseType: string, options: FetchOptions = {}) : Promise<T> {
+function fetch<T>(url: string, responseType: string, options: FetchOptions = {}): Promise<T> {
   let xhr = new XMLHttpRequest();
   xhr.responseType = responseType;
   let method = options.method || 'GET';
@@ -29,10 +29,10 @@ function fetch<T>(url: string, responseType: string, options: FetchOptions = {})
   return p;
 }
 
-export function fetchText(url: string, options: FetchOptions = {}) : Promise<string> {
+export function fetchText(url: string, options: FetchOptions = {}): Promise<string> {
   return fetch(url, 'text', options);
 }
 
-export function fetchArrayBuffer(url: string, options: FetchOptions = {}) : Promise<ArrayBuffer> {
+export function fetchArrayBuffer(url: string, options: FetchOptions = {}): Promise<ArrayBuffer> {
   return fetch(url, 'arraybuffer', options);
 }

js/src/decode.js

@@ -5,6 +5,7 @@ import Ref from './ref.js';
 import Struct from './struct.js';
 import type {ChunkStore} from './chunk_store.js';
 import type {NomsKind} from './noms_kind.js';
+import {decode as decodeBase64} from './base64.js';
 import {Field, makeCompoundType, makePrimitiveType, makeStructType, makeType, makeUnresolvedType, StructDesc, Type} from './type.js';
 import {invariant, notNull} from './assert.js';
 import {isPrimitiveKind, Kind} from './noms_kind.js';
@@ -98,6 +99,11 @@ class JsonArrayReader {
     throw new Error('Unreachable');
   }
 
+  readBlob(): Promise<ArrayBuffer> {
+    let s = this.readString();
+    return Promise.resolve(decodeBase64(s));
+  }
+
   async readList(t: Type, pkg: ?Package): Promise<Array<any>> {
     let elemType = t.elemTypes[0];
     let list = [];
@@ -152,7 +158,7 @@ class JsonArrayReader {
     // TODO: Verify read values match tagged kinds.
     switch (t.kind) {
       case Kind.Blob:
-        throw new Error('Not implemented');
+        return this.readBlob();
       case Kind.Bool:
         return Promise.resolve(this.readBool());
       case Kind.UInt8:

js/src/decode_test.js

@@ -55,25 +55,26 @@ suite('Decode', () => {
     async function doTest(expected: any, a: Array<any>): Promise<void> {
       let r = new JsonArrayReader(a, ms);
       let v = await r.readTopLevelValue();
-      assert.strictEqual(expected, v);
+      assert.deepEqual(expected, v);
     }
 
-    doTest(true, [Kind.Bool, true]);
-    doTest(false, [Kind.Bool, false]);
-    doTest(0, [Kind.UInt8, 0]);
-    doTest(0, [Kind.UInt16, 0]);
-    doTest(0, [Kind.UInt32, 0]);
-    doTest(0, [Kind.UInt64, 0]);
-    doTest(0, [Kind.Int8, 0]);
-    doTest(0, [Kind.Int16, 0]);
-    doTest(0, [Kind.Int32, 0]);
-    doTest(0, [Kind.Int64, 0]);
-    doTest(0, [Kind.Float32, 0]);
-    doTest(0, [Kind.Float64, 0]);
+    await doTest(true, [Kind.Bool, true]);
+    await doTest(false, [Kind.Bool, false]);
+    await doTest(0, [Kind.UInt8, 0]);
+    await doTest(0, [Kind.UInt16, 0]);
+    await doTest(0, [Kind.UInt32, 0]);
+    await doTest(0, [Kind.UInt64, 0]);
+    await doTest(0, [Kind.Int8, 0]);
+    await doTest(0, [Kind.Int16, 0]);
+    await doTest(0, [Kind.Int32, 0]);
+    await doTest(0, [Kind.Int64, 0]);
+    await doTest(0, [Kind.Float32, 0]);
+    await doTest(0, [Kind.Float64, 0]);
 
-    doTest('hi', [Kind.String, 'hi']);
+    await doTest('hi', [Kind.String, 'hi']);
 
-    // TODO: Blob
+    let blob = new Uint8Array([0x00, 0x01]).buffer;
+    await doTest(blob, [Kind.Blob, 'AAE=']);
   });
 
   test('read list of int 32', async () => {

js/src/encode.js

@@ -5,6 +5,7 @@ import Ref from './ref.js';
 import Struct from './struct.js';
 import type {ChunkStore} from './chunk_store.js';
 import type {NomsKind} from './noms_kind.js';
+import {encode as encodeBase64} from './base64.js';
 import {invariant, notNull} from './assert.js';
 import {isPrimitiveKind, Kind} from './noms_kind.js';
 import {lookupPackage, Package} from './package.js';
@@ -77,7 +78,8 @@ class JsonArrayWriter {
   writeValue(v: any, t: Type, pkg: ?Package) {
     switch (t.kind) {
       case Kind.Blob:
-        throw new Error('Not implemented');
+        this.writeBlob(v);
+        break;
       case Kind.Bool:
       case Kind.UInt8:
       case Kind.UInt16:
@@ -233,6 +235,10 @@ class JsonArrayWriter {
     }
   }
 
+  writeBlob(v: ArrayBuffer) {
+    this.write(encodeBase64(v));
+  }
+
   writeStruct(s: Struct, type: Type, typeDef: Type, pkg: Package) {
     let desc = typeDef.desc;
     invariant(desc instanceof StructDesc);

js/src/encode_test.js

@@ -11,8 +11,37 @@ import {Field, makeCompoundType, makePrimitiveType, makeStructType, makeType} fr
 import {JsonArrayWriter} from './encode.js';
 import {Kind} from './noms_kind.js';
 import {Package, registerPackage} from './package.js';
+import type {NomsKind} from './noms_kind.js';
 
 suite('Encode', () => {
+  test('write primitives', () => {
+    function f(k: NomsKind, v: any, ex: any) {
+      let ms = new MemoryStore();
+      let w = new JsonArrayWriter(ms);
+      w.writeTopLevel(makePrimitiveType(k), v);
+      assert.deepEqual([k, ex], w.array);
+    }
+
+    f(Kind.Bool, true, true);
+    f(Kind.Bool, false, false);
+
+    f(Kind.UInt8, 0, 0);
+    f(Kind.UInt16, 0, 0);
+    f(Kind.UInt32, 0, 0);
+    f(Kind.UInt64, 0, 0);
+    f(Kind.Int8, 0, 0);
+    f(Kind.Int16, 0, 0);
+    f(Kind.Int32, 0, 0);
+    f(Kind.Int64, 0, 0);
+    f(Kind.Float32, 0, 0);
+    f(Kind.Float64, 0, 0);
+
+    f(Kind.String, 'hi', 'hi');
+
+    let buffer = new Uint8Array([0x00, 0x01]).buffer;
+    f(Kind.Blob, buffer, 'AAE=');
+  });
+
   test('write list', async () => {
     let ms = new MemoryStore();
     let w = new JsonArrayWriter(ms);

js/src/fetch.js

@@ -9,7 +9,7 @@ type FetchOptions = {
   headers?: {[key: string]: string}
 };
 
-function fetch<T>(url: string, f: (buf: Buffer) => T, options: FetchOptions = {}) : Promise<T> {
+function fetch<T>(url: string, f: (buf: Buffer) => T, options: FetchOptions = {}): Promise<T> {
   let opts: any = parse(url);
   opts.method = options.method || 'GET';
   if (options.headers) {
@@ -39,7 +39,7 @@ function fetch<T>(url: string, f: (buf: Buffer) => T, options: FetchOptions = {}
   });
 }
 
-function bufferToArrayBuffer(buf: Buffer) : ArrayBuffer {
+function bufferToArrayBuffer(buf: Buffer): ArrayBuffer {
   let ab = new ArrayBuffer(buf.length);
   let view = new Uint8Array(ab);
   for (let i = 0; i < buf.length; i++) {
@@ -49,14 +49,14 @@ function bufferToArrayBuffer(buf: Buffer) : ArrayBuffer {
 }
 
 
-function bufferToString(buf: Buffer) : string {
+function bufferToString(buf: Buffer): string {
   return buf.toString();
 }
 
-export function fetchText(url: string, options: FetchOptions = {}) : Promise<string> {
+export function fetchText(url: string, options: FetchOptions = {}): Promise<string> {
   return fetch(url, bufferToString, options);
 }
 
-export function fetchArrayBuffer(url: string, options: FetchOptions = {}) : Promise<ArrayBuffer> {
+export function fetchArrayBuffer(url: string, options: FetchOptions = {}): Promise<ArrayBuffer> {
   return fetch(url, bufferToArrayBuffer, options);
 }