Add Read-only datastore

clients/crunchbase/ui/package.json

@@ -24,7 +24,7 @@
     "babel-preset-react": "^6.3.13",
     "eslint": "^1.9.0",
     "eslint-plugin-react": "^3.8.0",
-    "flow-bin": "^0.19.1",
+    "flow-bin": "^0.21.0",
     "http-server": "^0.8.5"
   },
   "scripts": {

clients/crunchbase/ui/src/chart.js

@@ -17,6 +17,15 @@ type State = {
 };
 
 export default class Chart extends React.Component<void, Props, State> {
+  state: State;
+
+  constructor(props: Props) {
+    super(props);
+    this.state = {
+      chart: null,
+    };
+  }
+
   componentDidMount() {
     nv.addGraph(() => {
       const chart = nv.models.lineChart();

clients/crunchbase/ui/src/data.js

@@ -1,17 +1,18 @@
 // @flow
 
 import {
+  DataStore,
   invariant,
   Kind,
   makeCompoundType,
   makePrimitiveType,
   makeType,
+  newSet,
   NomsMap,
   NomsSet,
   readValue,
   Ref,
   registerPackage,
-  SetLeafSequence,
   Struct,
 } from 'noms';
 import type {ChunkStore, Package} from 'noms';
@@ -42,11 +43,10 @@ type DataEntry = {values: Array<DataPoint>, key: string, color?: string};
 export type DataArray = Array<DataEntry>;
 
 export default class DataManager {
-  _store: ChunkStore;
-  _datasetId: string;
+  _datastore: DataStore;
   _keyClass: any;
   _quarterClass: any;
-  _datasetP: ?Promise<NomsMap<Ref, Ref>>;
+  _datasetP: ?Promise<NomsMap<Ref, NomsSet<Ref>>>;
   _packageP: ?Promise<Package>;
 
   _categorySetP: ?Promise<NomsSet<Ref>>;
@@ -59,12 +59,15 @@ export default class DataManager {
   _time: ?TimeOption;
   _category: string;
 
-  constructor(store: ChunkStore, datasetId: string) {
-    this._datasetId = datasetId;
-    this._store = store;
+  constructor(datastore: DataStore, datasetId: string) {
+    this._datastore = datastore;
+    this._datasetP = this._datastore.head(datasetId).then(commit => {
+      invariant(commit);
+      return commit.get('value');
+    });
+
     this._keyClass = null;
     this._quarterClass = null;
-    this._datasetP = null;
     this._packageP = null;
 
     this._timeSetP = null;
@@ -78,20 +81,14 @@ export default class DataManager {
     this._category = '';
   }
 
-  async _getDataset(): Promise<NomsMap<Ref, Ref>> {
-    if (this._datasetP) {
-      return this._datasetP;
-    }
-    return this._datasetP = getDataset(this._datasetId, this._store);
-  }
-
   async _getPackage(): Promise<Package> {
     if (this._packageP) {
       return this._packageP;
     }
 
-    const ds = await this._getDataset();
-    this._packageP = getKeyPackage(ds, this._store);
+    const ds = await this._datasetP;
+    invariant(ds);
+    this._packageP = getKeyPackage(ds, this._datastore);
     return this._packageP;
   }
 
@@ -146,7 +143,7 @@ export default class DataManager {
         await Promise.all([this._seedSetP, this._seriesASetP, this._seriesBSetP,
             this._timeSetP, this._categorySetP]);
 
-    const store = this._store;
+    const store = this._datastore;
     const getAmountRaised = (r: Ref): Promise<number> =>
         readValue(r, store).then(round => round.get('RaisedAmountUsd'));
 
@@ -190,14 +187,13 @@ export default class DataManager {
     const map = await this._datasetP;
     const set = await map.get(r);
     if (set === undefined) {
-      // TODO: Cleanup the NomsSet api (it shouldn't be this hard to create an emptySet)
-      return new NomsSet(this._store, setType, new SetLeafSequence(setType, []));
-    } else {
-      // Update the type to something that is correct.
-      // An alternative would be to hardcode the ref/ordinal.
-      setType = set.type;
+      return newSet(setType, []);
     }
 
+    invariant(set);
+    // Update the type to something that is correct.
+    // An alternative would be to hardcode the ref/ordinal.
+    setType = set.type;
     return set;
   }
 }
@@ -208,7 +204,7 @@ let setType = makeCompoundType(Kind.Set, makeCompoundType(Kind.Ref, makePrimitiv
 /**
  * Loads the first key in the index and gets the package from the type.
  */
-async function getKeyPackage(index: NomsMap<Ref, Ref>, store: ChunkStore):
+async function getKeyPackage(index: NomsMap<Ref, NomsSet<Ref>>, store: ChunkStore):
     Promise<Package> {
   const kv = await index.first();
   invariant(kv);
@@ -233,17 +229,6 @@ function getStructClass(pkg, name) {
   };
 }
 
-async function getDataset(id: string, httpStore: ChunkStore): Promise<NomsMap<Ref, Ref>> {
-  const rootRef = await httpStore.getRoot();
-  const datasets: Map<string, Ref> = await readValue(rootRef, httpStore);
-  const commitRef = await datasets.get(id);
-  invariant(commitRef);
-  const commit: Struct = await readValue(commitRef, httpStore);
-  invariant(commit);
-  return commit.get('value');
-}
-
-
 function percentiles(s: Array<number>): Array<{x: number, y: number}> {
   const arr: Array<number> = [];
   s.forEach(v => {

clients/crunchbase/ui/src/main.js

@@ -1,6 +1,6 @@
 // @flow
 
-import {HttpStore} from 'noms';
+import {DataStore, HttpStore} from 'noms';
 import Chart from './chart.js';
 import DataManager from './data.js';
 import List from './list.js';
@@ -38,6 +38,7 @@ if (!datasetId) {
 }
 
 class Main extends React.Component<void, Props, State> {
+  state: State;
   _dataManager: DataManager;
 
   constructor(props: Props) {
@@ -46,7 +47,8 @@ class Main extends React.Component<void, Props, State> {
     const selectedTimeItem = props.timeItems[0];
     const selectedCategoryItem = props.categories[0];
 
-    this._dataManager = new DataManager(new HttpStore(nomsServer), datasetId);
+    const datastore = new DataStore(new HttpStore(nomsServer));
+    this._dataManager = new DataManager(datastore, datasetId);
 
     this.state = {
       selectedSeries: new Set(this.props.series),

clients/pitchmap/README.md

@@ -20,7 +20,7 @@ cd /tmp/mlb_data
 
 wget -e robots=off -x -i urls.txt
 
-<noms>/clients/xml_importer/xml_importer -fs=/tmp/mlb_data -ds=mlb/xml  gd2.mlb.com/
+<noms>/clients/xml_importer/xml_importer -ldb=/tmp/mlb_data -ds=mlb/xml  gd2.mlb.com/
 
-<noms>/clients/pitchmap/index/index -fs=/tmp/mlb_data -input-ds=mlb/xml -output-ds=mlb/heatmap
+<noms>/clients/pitchmap/index/index -ldb=/tmp/mlb_data -input-ds=mlb/xml -output-ds=mlb/heatmap
 ```

clients/pitchmap/ui/package.json

@@ -22,7 +22,8 @@
     "babel-preset-react": "^6.3.13",
     "eslint": "^1.9.0",
     "eslint-plugin-react": "^3.8.0",
-    "flow-bin": "^0.19.1"
+    "flow-bin": "^0.21.0",
+    "http-server": "^0.8.5"
   },
   "scripts": {
     "start": "python .npm_build_helper.py development",

clients/pitchmap/ui/src/heat_map.js

@@ -1,7 +1,8 @@
 // @flow
 
 import React from 'react';
-import {HttpStore, NomsList, notNull, readValue, Ref} from 'noms';
+import {NomsList, notNull, readValue, Ref} from 'noms';
+import type {ChunkStore} from 'noms';
 
 const IMAGE_WIDTH_PX = 286;
 const IMAGE_HEIGHT_PX = 324;
@@ -18,7 +19,7 @@ function feetToPixels(f: number): number {
 
 type Props = {
   pitchListRefP: Promise<?Ref>,
-  httpStore: HttpStore
+  chunkStore: ChunkStore
 };
 
 type Point = {
@@ -32,6 +33,8 @@ type State = {
 };
 
 export default class HeatMap extends React.Component<void, Props, State> {
+  state: State;
+
   constructor(props: Props) {
     super(props);
 
@@ -47,7 +50,7 @@ export default class HeatMap extends React.Component<void, Props, State> {
     }
 
     const pitchListRef = notNull(await this.props.pitchListRefP);
-    const pitchList = await readValue(pitchListRef, this.props.httpStore);
+    const pitchList = await readValue(pitchListRef, this.props.chunkStore);
 
     if (pitchList instanceof NomsList) {
       const pointList = [];

clients/pitchmap/ui/src/main.js

@@ -3,9 +3,9 @@
 import HeatMap from './heat_map.js';
 import React from 'react';
 import ReactDOM from 'react-dom';
-import {HttpStore, invariant, NomsMap, readValue, Ref, Struct} from 'noms';
+import {DataStore, HttpStore, invariant, NomsMap, Ref} from 'noms';
 
-let httpStore: HttpStore;
+let datastore: DataStore;
 
 const nomsServer: ?string = process.env.NOMS_SERVER;
 if (!nomsServer) {
@@ -17,13 +17,11 @@ if (!datasetId) {
 }
 
 window.addEventListener('load', async () => {
-  httpStore = new HttpStore(nomsServer);
-  const rootRef = await httpStore.getRoot();
-  const datasets: NomsMap<string, Ref> = await readValue(rootRef, httpStore);
-  const commitRef = await datasets.get(datasetId);
-  invariant(commitRef);
-  const commit:Struct = await readValue(commitRef, httpStore);
-  const pitchersMap = commit.get('value');
+  datastore = new DataStore(new HttpStore(nomsServer));
+  const head = await datastore.head(datasetId);
+  invariant(head);
+  const pitchersMap: NomsMap<string, Ref> = head.get('value');
+  invariant(pitchersMap);
   const pitchers = [];
   await pitchersMap.forEach((ref, pitcher) => {
     pitchers.push(pitcher);
@@ -43,6 +41,8 @@ type State = {
 };
 
 class PitcherList extends React.Component<void, Props, State> {
+  state: State;
+
   constructor(props) {
     super(props);
 
@@ -65,7 +65,7 @@ class PitcherList extends React.Component<void, Props, State> {
       <select onChange={onChangePitcher} defaultValue={currentPitcher}>{
         this.props.pitchers.map(pitcher => <option key={pitcher} value={pitcher}>{pitcher}</option>)
       }</select>
-      <HeatMap key={currentPitcher} pitchListRefP={pitchListRefP} httpStore={httpStore}/>
+      <HeatMap key={currentPitcher} pitchListRefP={pitchListRefP} chunkStore={datastore}/>
     </div>;
   }
 }

clients/splore/package.json

@@ -23,7 +23,7 @@
     "babel-preset-react": "^6.3.13",
     "eslint": "^1.9.0",
     "eslint-plugin-react": "^3.8.0",
-    "flow-bin": "^0.19.1",
+    "flow-bin": "^0.21.0",
     "http-server": "^0.8.5"
   },
   "scripts": {

clients/splore/src/main.js

@@ -167,7 +167,7 @@ function handleChunkLoad(ref: Ref, val: any, fromRef: ?string) {
   render();
 }
 
-function handleNodeClick(e: Event, id: string) {
+function handleNodeClick(e: MouseEvent, id: string) {
   if (e.button === 0 && !e.shiftKey && !e.ctrlKey && !e.altKey && !e.metaKey) {
     e.preventDefault();
   }
@@ -199,6 +199,8 @@ type PromptState = {
 }
 
 class Prompt extends React.Component<void, {}, PromptState> {
+  state: PromptState;
+
   render() {
     const fontStyle = {
       fontFamily: 'Menlo',
@@ -214,7 +216,7 @@ class Prompt extends React.Component<void, {}, PromptState> {
               style={Object.assign(fontStyle, {}, {width: '50ex'})}
               defaultValue='http://api.noms.io/-/ds/[user]'/>
           </div>
-        </label> 
+        </label>
         <div><button onClick={() => setServer(this.refs.input.value)}>OK</button></div>
       </div>
     </div>;

clients/splore/src/node.js

@@ -25,6 +25,8 @@ type State = {
 };
 
 export default class Node extends React.Component<void, Props, State> {
+  state: State;
+
   constructor(props: Props) {
     super(props);
 

clients/tagshow/package.json

@@ -24,7 +24,7 @@
     "chai": "^3.2.0",
     "eslint": "^1.9.0",
     "eslint-plugin-react": "^3.8.0",
-    "flow-bin": "^0.19.1",
+    "flow-bin": "^0.21.0",
     "http-server": "^0.8.5",
     "mocha": "^2.3.0"
   },

clients/tagshow/src/datasetpicker.js

@@ -20,6 +20,7 @@ type State = {
 };
 
 export default class DatasetPicker extends React.Component<DefaultProps, Props, State> {
+  state: State;
   static defaultProps: DefaultProps;
 
   constructor(props: Props) {

clients/tagshow/src/main.js

@@ -1,11 +1,10 @@
 // @flow
 
-import {HttpStore} from 'noms';
+import {DataStore, HttpStore} from 'noms';
 import queryString from 'query-string';
 import React from 'react'; // eslint-disable-line no-unused-vars
 import ReactDOM from 'react-dom';
 import Root from './root.js';
-import type {ChunkStore} from 'noms';
 
 window.onload = window.onhashchange = render;
 
@@ -21,7 +20,7 @@ function updateQuery(qs: {[key: string]: string}) {
 function render() {
   const qs = Object.freeze(queryString.parse(location.hash));
   const target = document.getElementById('root');
-  const store: ChunkStore = new HttpStore(nomsServer);
+  const store = new DataStore(new HttpStore(nomsServer));
 
   ReactDOM.render(
       <Root qs={qs} store={store} updateQuery={updateQuery}/>,

clients/tagshow/src/photo.js

@@ -22,6 +22,7 @@ type State = {
 };
 
 export default class Photo extends React.Component<DefaultProps, Props, State> {
+  state: State;
   static defaultProps: DefaultProps;
 
   constructor(props: Props) {

clients/tagshow/src/root.js

@@ -5,13 +5,13 @@ import eq from './eq.js';
 import React from 'react';
 import SlideShow from './slideshow.js';
 import TagChooser from './tagchooser.js';
-import type {ChunkStore} from 'noms';
-import {invariant, NomsMap, NomsSet, readValue, Ref, Struct} from 'noms';
+import type {DataStore} from 'noms';
+import {invariant, NomsMap, NomsSet, Ref} from 'noms';
 
 type QueryStringObject = {[key: string]: string};
 
 type Props = {
-  store: ChunkStore,
+  store: DataStore,
   qs: QueryStringObject,
   updateQuery: (qs: QueryStringObject) => void,
 };
@@ -23,6 +23,8 @@ type State = {
 };
 
 export default class Root extends React.Component<void, Props, State> {
+  state: State;
+
   constructor(props: Props) {
     super(props);
     this.state = {
@@ -39,11 +41,8 @@ export default class Root extends React.Component<void, Props, State> {
 
     if (props.qs.ds) {
       const {store} = props;
-      const rootRef = await props.store.getRoot();
-      const datasets: NomsMap<string, Ref> = await readValue(rootRef, props.store);
-      const commitRef = await datasets.get(props.qs.ds);
-      invariant(commitRef);
-      const commit: Struct = await readValue(commitRef, store);
+      const commit = await store.head(props.qs.ds);
+      invariant(commit);
       const v = commit.get('value');
       if (v instanceof NomsMap) {
         const seenRefs: Set<string> = new Set();

clients/tagshow/src/slideshow.js

@@ -27,6 +27,8 @@ type State = {
 };
 
 export default class SlideShow extends React.Component<void, Props, State> {
+  state: State;
+
   constructor(props: Props) {
     super(props);
     this.state = {
@@ -70,6 +72,8 @@ type ItemState = {
 };
 
 class Item extends React.Component<void, ItemProps, ItemState> {
+  state: ItemState;
+
   constructor(props: ItemProps) {
     super(props);
     this.state = {

js/src/datastore.js

@@ -0,0 +1,114 @@
+// @flow
+
+import Chunk from './chunk.js';
+import Ref from './ref.js';
+import Struct from './struct.js';
+import type {ChunkStore} from './chunk_store.js';
+import {Field, makeCompoundType, makePrimitiveType, makeStructType, makeType,
+  Type} from './type.js';
+import {Kind} from './noms_kind.js';
+import {newMap, NomsMap} from './map.js';
+import {Package, registerPackage} from './package.js';
+import {readValue} from './read_value.js';
+
+type DatasTypes = {
+  commitTypeDef: Type,
+  datasPackage: Package,
+  commitType: Type,
+  commitSetType: Type,
+  commitMapType: Type,
+};
+
+let emptyCommitMap: Promise<NomsMap<string, Ref>>;
+function getEmptyCommitMap(): Promise<NomsMap<string, Ref>> {
+  if (!emptyCommitMap) {
+    emptyCommitMap = newMap(getDatasTypes().commitMapType, []);
+  }
+  return emptyCommitMap;
+}
+
+
+let datasTypes: DatasTypes;
+export function getDatasTypes(): DatasTypes {
+  if (!datasTypes) {
+    const commitTypeDef = makeStructType('Commit', [
+      new Field('value', makePrimitiveType(Kind.Value), false),
+      new Field('parents', makeCompoundType(Kind.Set,
+        makeCompoundType(Kind.Ref, makeType(new Ref(), 0))), false),
+    ], []);
+
+    const datasPackage = new Package([commitTypeDef], []);
+    registerPackage(datasPackage);
+
+    const commitType = makeType(datasPackage.ref, 0);
+
+    const commitSetType = makeCompoundType(Kind.Set,
+      makeCompoundType(Kind.Ref, makeType(datasPackage.ref, 0)));
+
+    const commitMapType =
+      makeCompoundType(Kind.Map, makePrimitiveType(Kind.String),
+                                 makeCompoundType(Kind.Ref, makeType(datasPackage.ref, 0)));
+
+    datasTypes = {
+      commitTypeDef,
+      datasPackage,
+      commitType,
+      commitSetType,
+      commitMapType,
+    };
+  }
+
+  return datasTypes;
+}
+
+export class DataStore {
+  _cs: ChunkStore;
+  _datasets: Promise<NomsMap<string, Ref>>;
+
+  constructor(cs: ChunkStore) {
+    this._cs = cs;
+    this._datasets = this._datasetsFromRootRef();
+  }
+
+  getRoot(): Promise<Ref> {
+    return this._cs.getRoot();
+  }
+
+  updateRoot(current: Ref, last: Ref): Promise<boolean> {
+    return this._cs.updateRoot(current, last);
+  }
+
+  get(ref: Ref): Promise<Chunk> {
+    return this._cs.get(ref);
+  }
+
+  has(ref: Ref): Promise<boolean> {
+    return this._cs.has(ref);
+  }
+
+  put(c: Chunk): void {
+    this._cs.put(c);
+  }
+
+  close() {}
+
+  _datasetsFromRootRef(): Promise<NomsMap<string, Ref>> {
+    return this._cs.getRoot().then(rootRef => {
+      if (rootRef.isEmpty()) {
+        return getEmptyCommitMap();
+      }
+
+      return readValue(rootRef, this._cs);
+    });
+  }
+
+  head(datasetID: string): Promise<?Struct> {
+    return this._datasets.then(
+      datasets => datasets.get(datasetID).then(commitRef => commitRef ?
+                                                            readValue(commitRef, this._cs) : null));
+  }
+
+  datasets(): Promise<NomsMap<string, Ref>> {
+    return this._datasets;
+  }
+}

js/src/datastore_test.js

@@ -0,0 +1,67 @@
+// @flow
+
+import {suite, test} from 'mocha';
+
+import Chunk from './chunk.js';
+import MemoryStore from './memory_store.js';
+import Ref from './ref.js';
+import Struct from './struct.js';
+import {assert} from 'chai';
+import {DataStore, getDatasTypes} from './datastore.js';
+import {invariant} from './assert.js';
+import {newMap} from './map.js';
+import {newSet} from './set.js';
+import {writeValue} from './encode.js';
+
+suite('DataStore', () => {
+  test('access', async () => {
+    const ms = new MemoryStore();
+    const ds = new DataStore(ms);
+    const input = 'abc';
+
+    const c = Chunk.fromString(input);
+    let c1 = await ds.get(c.ref);
+    assert.isTrue(c1.isEmpty());
+
+    let has = await ds.has(c.ref);
+    assert.isFalse(has);
+
+    ms.put(c);
+    c1 = await ds.get(c.ref);
+    assert.isFalse(c1.isEmpty());
+
+    has = await ds.has(c.ref);
+    assert.isTrue(has);
+  });
+
+  test('empty datasets', async() => {
+    const ms = new MemoryStore();
+    const ds = new DataStore(ms);
+    const datasets = await ds.datasets();
+    assert.strictEqual(0, datasets.size);
+  });
+
+  test('head', async() => {
+    const ms = new MemoryStore();
+    let ds = new DataStore(ms);
+    const types = getDatasTypes();
+
+    const commit = new Struct(types.commitType, types.commitTypeDef, {
+      value: 'foo',
+      parents: await newSet(types.commitSetType, []),
+    });
+
+    const commitRef = writeValue(commit, commit.type, ms);
+    const datasets = await newMap(types.commitMapType, ['foo', commitRef]);
+    const rootRef = writeValue(datasets, datasets.type, ms);
+    assert.isTrue(await ms.updateRoot(rootRef, new Ref()));
+    ds = new DataStore(ms); // refresh the datasets
+
+    assert.strictEqual(1, datasets.size);
+    const fooHead = await ds.head('foo');
+    invariant(fooHead);
+    assert.isTrue(fooHead.equals(commit));
+    const barHead = await ds.head('bar');
+    assert.isNull(barHead);
+  });
+});

js/src/noms.js

@@ -1,6 +1,7 @@
 // @flow
 
 export {AsyncIterator} from './async_iterator.js';
+export {DataStore} from './datastore.js';
 export {decodeNomsValue} from './decode.js';
 export {default as Chunk} from './chunk.js';
 export {default as HttpStore} from './http_store.js';
@@ -10,8 +11,8 @@ export {default as Struct} from './struct.js';
 export {encodeNomsValue, writeValue} from './encode.js';
 export {invariant, notNull} from './assert.js';
 export {isPrimitiveKind, Kind} from './noms_kind.js';
-export {newList, ListLeafSequence, NomsList} from './list.js';
 export {lookupPackage, Package, readPackage, registerPackage} from './package.js';
+export {newList, ListLeafSequence, NomsList} from './list.js';
 export {newMap, NomsMap, MapLeafSequence} from './map.js';
 export {newSet, NomsSet, SetLeafSequence} from './set.js';
 export {OrderedMetaSequence, IndexedMetaSequence} from './meta_sequence.js';

js/src/sequence_chunker.js

@@ -142,7 +142,6 @@ export class SequenceChunker<S, T, U:Sequence, V:Sequence> {
     }
 
     if (this.isRoot()) {
-      invariant(this._current.length > 0);
       return this._makeChunk(this._current)[1];
     }