From 141b5678a0a73abc2434ccb36b410e4cc9d7516f Mon Sep 17 00:00:00 2001 From: Rafael Weinstein Date: Tue, 31 May 2016 16:58:09 -0700 Subject: [PATCH] Post-binaryEnc cleanup (#1684) Post-binaryEnc cleanup --- js/src/base64-test.js | 37 ---------- js/src/base64.js | 69 ------------------- js/src/batch-store-test.js | 6 +- js/src/codec.js | 27 ++++++++ js/src/database-test.js | 4 +- js/src/{encode-test.js => encoding-test.js} | 9 +-- js/src/get-hash.js | 8 +-- js/src/noms.js | 4 +- js/src/{decode.js => value-decoder.js} | 18 +---- js/src/{encode.js => value-encoder.js} | 38 ++++------ js/src/value-store-test.js | 4 +- js/src/value-store.js | 8 +-- ...de_noms_value_test.go => encoding_test.go} | 0 ...{decode_noms_value.go => value_decoder.go} | 0 ...{encode_noms_value.go => value_encoder.go} | 0 15 files changed, 63 insertions(+), 169 deletions(-) delete mode 100644 js/src/base64-test.js delete mode 100644 js/src/base64.js rename js/src/{encode-test.js => encoding-test.js} (98%) rename js/src/{decode.js => value-decoder.js} (94%) rename js/src/{encode.js => value-encoder.js} (94%) rename types/{encode_noms_value_test.go => encoding_test.go} (100%) rename types/{decode_noms_value.go => value_decoder.go} (100%) rename types/{encode_noms_value.go => value_encoder.go} (100%) diff --git a/js/src/base64-test.js b/js/src/base64-test.js deleted file mode 100644 index 80f9149e58..0000000000 --- a/js/src/base64-test.js +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2016 The Noms Authors. All rights reserved. -// Licensed under the Apache License, version 2.0: -// http://www.apache.org/licenses/LICENSE-2.0 - -// @flow - -import {suite, test} from 'mocha'; -import {assert} from 'chai'; -import {encode, decode} from './base64.js'; - -function uint8ArrayFromString(s: string): Uint8Array { - const ta = new Uint8Array(s.length); - for (let i = 0; i < s.length; i++) { - ta[i] = s.charCodeAt(i); - } - return ta; -} - -suite('base64', () => { - test('encode', () => { - assert.deepEqual(encode(uint8ArrayFromString('Hello world')), 'SGVsbG8gd29ybGQ='); - assert.deepEqual(encode(uint8ArrayFromString('Man')), 'TWFu'); - assert.deepEqual(encode(uint8ArrayFromString('Ma')), 'TWE='); - assert.deepEqual(encode(uint8ArrayFromString('M')), 'TQ=='); - assert.deepEqual(encode(uint8ArrayFromString('')), ''); - assert.deepEqual(encode(uint8ArrayFromString('Hello worlds!')), 'SGVsbG8gd29ybGRzIQ=='); - }); - - test('decode', () => { - assert.deepEqual(decode('TWFu'), uint8ArrayFromString('Man')); - assert.deepEqual(decode('TWE='), uint8ArrayFromString('Ma')); - assert.deepEqual(decode('TQ=='), uint8ArrayFromString('M')); - assert.deepEqual(decode(''), uint8ArrayFromString('')); - assert.deepEqual(decode('SGVsbG8gd29ybGQ='), uint8ArrayFromString('Hello world')); - assert.deepEqual(decode('SGVsbG8gd29ybGRzIQ=='), uint8ArrayFromString('Hello worlds!')); - }); -}); diff --git a/js/src/base64.js b/js/src/base64.js deleted file mode 100644 index 8cd2a74027..0000000000 --- a/js/src/base64.js +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2016 The Noms Authors. All rights reserved. -// Licensed under the Apache License, version 2.0: -// http://www.apache.org/licenses/LICENSE-2.0 - -// @flow - -// Based on https://github.com/niklasvh/base64-arraybuffer -// -// base64-arraybuffer -// https://github.com/niklasvh/base64-arraybuffer -// -// Copyright (c) 2012 Niklas von Hertzen -// Licensed under the MIT license. - -const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; - -// Build charCode -> index -const lookup: Uint8Array = new Uint8Array(256); -for (let i = 0 ; i < chars.length; i++) { - lookup[chars.charCodeAt(i)] = i; -} - -export function encode(bytes: Uint8Array): string { - const len = bytes.length; - let base64 = ''; - - for (let i = 0; i < len; i += 3) { - base64 += chars[bytes[i] >> 2]; - base64 += chars[((bytes[i] & 3) << 4) | (bytes[i + 1] >> 4)]; - base64 += chars[((bytes[i + 1] & 15) << 2) | (bytes[i + 2] >> 6)]; - base64 += chars[bytes[i + 2] & 63]; - } - - if (len % 3 === 2) { - base64 = base64.substring(0, base64.length - 1) + '='; - } else if (len % 3 === 1) { - base64 = base64.substring(0, base64.length - 2) + '=='; - } - - return base64; -} - -export function decode(s: string): Uint8Array { - let bufferLength = s.length * 0.75; - const len = s.length; - - if (s[len - 1] === '=') { - bufferLength--; - if (s[len - 2] === '=') { - bufferLength--; - } - } - - const bytes = new Uint8Array(bufferLength); - let p = 0; - - for (let i = 0; i < len; i += 4) { - const encoded1 = lookup[s.charCodeAt(i)]; - const encoded2 = lookup[s.charCodeAt(i + 1)]; - const encoded3 = lookup[s.charCodeAt(i + 2)]; - const encoded4 = lookup[s.charCodeAt(i + 3)]; - - bytes[p++] = (encoded1 << 2) | (encoded2 >> 4); - bytes[p++] = ((encoded2 & 15) << 4) | (encoded3 >> 2); - bytes[p++] = ((encoded3 & 3) << 6) | (encoded4 & 63); - } - - return bytes; -} diff --git a/js/src/batch-store-test.js b/js/src/batch-store-test.js index cc11a46d04..b32020a04c 100644 --- a/js/src/batch-store-test.js +++ b/js/src/batch-store-test.js @@ -9,7 +9,7 @@ import {assert} from 'chai'; import MemoryStore from './memory-store.js'; import BatchStore from './batch-store.js'; import {BatchStoreAdaptorDelegate} from './batch-store-adaptor.js'; -import {encodeNomsValue} from './encode.js'; +import {encodeValue} from './codec.js'; suite('BatchStore', () => { test('get after schedulePut works immediately', async () => { @@ -17,7 +17,7 @@ suite('BatchStore', () => { const bs = new BatchStore(3, new BatchStoreAdaptorDelegate(ms)); const input = 'abc'; - const c = encodeNomsValue(input); + const c = encodeValue(input); bs.schedulePut(c, new Set()); const chunk = await bs.get(c.hash); @@ -30,7 +30,7 @@ suite('BatchStore', () => { const bs = new BatchStore(3, new BatchStoreAdaptorDelegate(ms)); const input = 'abc'; - const c = encodeNomsValue(input); + const c = encodeValue(input); bs.schedulePut(c, new Set()); let chunk = await bs.get(c.hash); diff --git a/js/src/codec.js b/js/src/codec.js index 037f01b956..05b9a0e72d 100644 --- a/js/src/codec.js +++ b/js/src/codec.js @@ -4,9 +4,36 @@ // @flow +import Chunk from './chunk.js'; import Hash, {sha1Size} from './hash.js'; +import ValueDecoder from './value-decoder.js'; +import ValueEncoder from './value-encoder.js'; import {encode, decode} from './utf8.js'; import {invariant} from './assert.js'; +import {setEncodeValue} from './get-hash.js'; +import {setHash, ValueBase} from './value.js'; + +export function encodeValue(v: Value, vw: ?ValueWriter): Chunk { + const w = new BinaryNomsWriter(); + const enc = new ValueEncoder(w, vw); + enc.writeValue(v); + return new Chunk(w.data); +} + +setEncodeValue(encodeValue); + +export function decodeValue(chunk: Chunk, vr: ValueReader): Value { + const data = chunk.data; + const dec = new ValueDecoder(new BinaryNomsReader(data), vr); + const v = dec.readValue(); + + if (v instanceof ValueBase) { + setHash(v, chunk.hash); + } + + return v; +} + const maxUInt32 = Math.pow(2, 32); const littleEndian = true; diff --git a/js/src/database-test.js b/js/src/database-test.js index 6440c84fa1..0720c51244 100644 --- a/js/src/database-test.js +++ b/js/src/database-test.js @@ -13,7 +13,7 @@ import Database from './database.js'; import {invariant, notNull} from './assert.js'; import List from './list.js'; import Map from './map.js'; -import {encodeNomsValue} from './encode.js'; +import {encodeValue} from './codec.js'; import NomsSet from './set.js'; // namespace collision with JS Set import {equals} from './compare.js'; @@ -23,7 +23,7 @@ suite('Database', () => { const ds = new Database(bs); const input = 'abc'; - const c = encodeNomsValue(input); + const c = encodeValue(input); const v1 = await ds.readValue(c.hash); assert.equal(null, v1); diff --git a/js/src/encode-test.js b/js/src/encoding-test.js similarity index 98% rename from js/src/encode-test.js rename to js/src/encoding-test.js index 918bd5c62d..2c9921515f 100644 --- a/js/src/encode-test.js +++ b/js/src/encoding-test.js @@ -18,8 +18,9 @@ import Set, {newSetLeafSequence} from './set.js'; import type Value from './value.js'; import type {NomsKind} from './noms-kind.js'; import {Kind} from './noms-kind.js'; -import {decodeNomsValue, ValueDecoder} from './decode.js'; -import {encodeNomsValue, ValueEncoder} from './encode.js'; +import ValueDecoder from './value-decoder.js'; +import ValueEncoder from './value-encoder.js'; +import {encodeValue, decodeValue} from './codec.js'; import {equals} from './compare.js'; import {invariant} from './assert.js'; import {makeTestingBatchStore} from './batch-store-adaptor.js'; @@ -45,8 +46,8 @@ import { function assertRoundTrips(v: Value) { const db = new Database(makeTestingBatchStore()); - const c = encodeNomsValue(v, db); - const out = decodeNomsValue(c, db); + const c = encodeValue(v, db); + const out = decodeValue(c, db); assert.isTrue(equals(v, out)); } diff --git a/js/src/get-hash.js b/js/src/get-hash.js index 969f5f7702..aebe5af853 100644 --- a/js/src/get-hash.js +++ b/js/src/get-hash.js @@ -13,7 +13,7 @@ import {getTypeOfValue} from './type.js'; import {ValueBase} from './value.js'; type encodeFn = (v: Value, vw: ?ValueWriter) => Chunk; -let encodeNomsValue: ?encodeFn = null; +let encodeValue: ?encodeFn = null; export function getHashOfValue(v: Value): Hash { if (v instanceof ValueBase) { @@ -24,7 +24,7 @@ export function getHashOfValue(v: Value): Hash { } export function getHash(v: Value): Hash { - return notNull(encodeNomsValue)(v, null).hash; + return notNull(encodeValue)(v, null).hash; } export function ensureHash(h: ?Hash, v: Value): Hash { @@ -35,6 +35,6 @@ export function ensureHash(h: ?Hash, v: Value): Hash { return getHash(v); } -export function setEncodeNomsValue(encode: encodeFn) { - encodeNomsValue = encode; +export function setEncodeValue(encode: encodeFn) { + encodeValue = encode; } diff --git a/js/src/noms.js b/js/src/noms.js index e3160f0750..52a1d41f2c 100644 --- a/js/src/noms.js +++ b/js/src/noms.js @@ -10,7 +10,7 @@ export {default as Commit} from './commit.js'; export {default as Database} from './database.js'; export {default as Dataset} from './dataset.js'; export {default as Blob, BlobReader, BlobWriter} from './blob.js'; -export {decodeNomsValue} from './decode.js'; +export {decodeValue} from './codec.js'; export {default as Chunk} from './chunk.js'; export {default as HttpBatchStore} from './http-batch-store.js'; export {default as MemoryStore} from './memory-store.js'; @@ -24,7 +24,7 @@ export { createStructClass, escapeStructField, } from './struct.js'; -export {encodeNomsValue} from './encode.js'; +export {encodeValue} from './codec.js'; export {invariant, notNull} from './assert.js'; export {isPrimitiveKind, Kind, kindToString} from './noms-kind.js'; export {default as List, ListWriter, ListLeafSequence} from './list.js'; diff --git a/js/src/decode.js b/js/src/value-decoder.js similarity index 94% rename from js/src/decode.js rename to js/src/value-decoder.js index 4e23f20b89..1235882174 100644 --- a/js/src/decode.js +++ b/js/src/value-decoder.js @@ -5,7 +5,6 @@ // @flow import Blob, {BlobLeafSequence} from './blob.js'; -import Chunk from './chunk.js'; import Ref, {constructRef} from './ref.js'; import {newStructWithTypeNoValidation} from './struct.js'; import type Struct from './struct.js'; @@ -28,13 +27,11 @@ import List, {ListLeafSequence} from './list.js'; import Map, {MapLeafSequence} from './map.js'; import Set, {SetLeafSequence} from './set.js'; import {IndexedMetaSequence, OrderedMetaSequence} from './meta-sequence.js'; -import {ValueBase, setHash} from './value.js'; import type Value from './value.js'; import type {ValueReader} from './value-store.js'; import type {NomsReader} from './codec.js'; -import {BinaryNomsReader} from './codec.js'; -export class ValueDecoder { +export default class ValueDecoder { _r: NomsReader; _ds: ValueReader; @@ -53,7 +50,6 @@ export class ValueDecoder { return constructRef(t, hash, height); } - readType(parentStructTypes: Type[]): Type { const k = this.readKind(); switch (k) { @@ -234,15 +230,3 @@ export class ValueDecoder { return structType; } } - -export function decodeNomsValue(chunk: Chunk, vr: ValueReader): Value { - const data = chunk.data; - const dec = new ValueDecoder(new BinaryNomsReader(data), vr); - const v = dec.readValue(); - - if (v instanceof ValueBase) { - setHash(v, chunk.hash); - } - - return v; -} diff --git a/js/src/encode.js b/js/src/value-encoder.js similarity index 94% rename from js/src/encode.js rename to js/src/value-encoder.js index 82324291aa..8ef7c85ccf 100644 --- a/js/src/encode.js +++ b/js/src/value-encoder.js @@ -4,30 +4,27 @@ // @flow -import Chunk from './chunk.js'; -import Ref from './ref.js'; -import Struct, {StructMirror} from './struct.js'; -import type {NomsKind} from './noms-kind.js'; -import {StructDesc, Type, getTypeOfValue} from './type.js'; -import {MetaTuple} from './meta-sequence.js'; -import {invariant} from './assert.js'; -import {isPrimitiveKind, kindToString, Kind} from './noms-kind.js'; +import Blob, {BlobLeafSequence} from './blob.js'; import List, {ListLeafSequence} from './list.js'; import Map, {MapLeafSequence} from './map.js'; -import Set, {SetLeafSequence} from './set.js'; +import Ref from './ref.js'; import Sequence from './sequence.js'; -import {setEncodeNomsValue} from './get-hash.js'; -import Blob, {BlobLeafSequence} from './blob.js'; -import {describeTypeOfValue} from './encode-human-readable.js'; -import type {primitive} from './primitives.js'; +import Set, {SetLeafSequence} from './set.js'; +import Struct, {StructMirror} from './struct.js'; import type Value from './value.js'; -import type {ValueWriter} from './value-store.js'; +import type {NomsKind} from './noms-kind.js'; import type {NomsWriter} from './codec.js'; -import {BinaryNomsWriter} from './codec.js'; +import type {ValueWriter} from './value-store.js'; +import type {primitive} from './primitives.js'; +import {MetaTuple} from './meta-sequence.js'; +import {StructDesc, Type, getTypeOfValue} from './type.js'; +import {describeTypeOfValue} from './encode-human-readable.js'; +import {invariant} from './assert.js'; +import {isPrimitiveKind, kindToString, Kind} from './noms-kind.js'; type primitiveOrArray = primitive | Array; -export class ValueEncoder { +export default class ValueEncoder { _w: NomsWriter; _vw: ?ValueWriter; @@ -251,12 +248,3 @@ export class ValueEncoder { parentStructTypes.pop(); } } - -export function encodeNomsValue(v: Value, vw: ?ValueWriter): Chunk { - const w = new BinaryNomsWriter(); - const enc = new ValueEncoder(w, vw); - enc.writeValue(v); - return new Chunk(w.data); -} - -setEncodeNomsValue(encodeNomsValue); diff --git a/js/src/value-store-test.js b/js/src/value-store-test.js index dd1ff7100c..a7a0672b59 100644 --- a/js/src/value-store-test.js +++ b/js/src/value-store-test.js @@ -12,7 +12,7 @@ import BatchStore from './batch-store.js'; import {BatchStoreAdaptorDelegate} from './batch-store-adaptor.js'; import ValueStore from './value-store.js'; import List from './list.js'; -import {encodeNomsValue} from './encode.js'; +import {encodeValue} from './codec.js'; import {equals} from './compare.js'; export class FakeBatchStore extends BatchStore { @@ -27,7 +27,7 @@ suite('ValueStore', () => { const vs = new ValueStore(new FakeBatchStore(ms)); const input = 'abc'; - const c = encodeNomsValue(input); + const c = encodeValue(input); const v1 = await vs.readValue(c.hash); assert.equal(null, v1); diff --git a/js/src/value-store.js b/js/src/value-store.js index daa26020f3..a531c65ebd 100644 --- a/js/src/value-store.js +++ b/js/src/value-store.js @@ -18,9 +18,9 @@ import { } from './type.js'; import {Kind} from './noms-kind.js'; import {ValueBase} from './value.js'; -import {decodeNomsValue} from './decode.js'; +import {decodeValue} from './codec.js'; import {invariant, notNull} from './assert.js'; -import {encodeNomsValue} from './encode.js'; +import {encodeValue} from './codec.js'; import {describeType, describeTypeOfValue} from './encode-human-readable.js'; import {equals} from './compare.js'; @@ -63,7 +63,7 @@ export default class ValueStore { return null; } - const v = decodeNomsValue(chunk, this); + const v = decodeValue(chunk, this); this._valueCache.add(hash, chunk.data.length, v); this._knownHashes.cacheChunks(v, hash); // hash is trivially a hint for v, so consider putting that in the cache. @@ -78,7 +78,7 @@ export default class ValueStore { writeValue(v: T): Ref { const t = getTypeOfValue(v); - const chunk = encodeNomsValue(v, this); + const chunk = encodeValue(v, this); invariant(!chunk.isEmpty()); const {hash} = chunk; const height = maxChunkHeight(v) + 1; diff --git a/types/encode_noms_value_test.go b/types/encoding_test.go similarity index 100% rename from types/encode_noms_value_test.go rename to types/encoding_test.go diff --git a/types/decode_noms_value.go b/types/value_decoder.go similarity index 100% rename from types/decode_noms_value.go rename to types/value_decoder.go diff --git a/types/encode_noms_value.go b/types/value_encoder.go similarity index 100% rename from types/encode_noms_value.go rename to types/value_encoder.go