Implement noms-view and splore-view (a splore compatible with noms-view) (#1194)

* Implement noms-view and make splore compatible

* Comments

* npm test

* Fewer changes to readme/build/etc

* comment in noms_view.go

* More updates

* Typo

clients/splore/.npm_build_helper.py

@@ -9,8 +9,6 @@ def main():
 	env = os.environ
 	env['NODE_ENV'] = sys.argv[1]
 	env['BABEL_ENV'] = sys.argv[1]
-	if 'NOMS_SERVER' not in env:
-		env['NOMS_SERVER'] = 'http://localhost:8000'
 	subprocess.check_call(['node_modules/.bin/webpack'] + SRC + [OUT], env=env, shell=False)
 
 

clients/splore/README.md

@@ -7,27 +7,26 @@ Splore is a general-purpose debug UI for exploring noms data.
 ## Example
 
 ```
-cd $GOPATH/src/github.com/attic-labs/noms/clients/counter
+# Create some data
+cd "$GOPATH/src/github.com/attic-labs/noms/clients/counter"
 go build
 ./counter -ldb="/tmp/sploretest" -ds="counter"
 ./counter -ldb="/tmp/sploretest" -ds="counter"
 
-# Splore requires server to be running
-cd ../server
-go build
-./server -ldb="/tmp/sploretest" &
-
+# Build Splore
 cd ../splore
 ./build.py
-./node_modules/.bin/http-server
+
+# Launch Splore with noms-view
+cd ../../cmd/noms-view
+go build
+./noms-view serve ../../clients/splore store="ldb:/tmp/sploretest" &
 ```
 
-Then, navigate to [http://localhost:8080](http://localhost:8080).
-
+Then, navigate to the URL printed by noms-view, e.g. http://127.0.0.1:12345?store=xyz.
 
 ## Develop
 
-* `./build.py`  # only necessary first time
-* `NOMS_SERVER=http://localhost:8000 npm run start`
-
-This will start watchify which is continually building a non-minified (and thus debuggable) build.
+Same as the example, but:
+* `./build.py` is only necessary the first time.
+* Also run `npm run start`, to continually build a non-minified (and thus debuggable) build.

clients/splore/src/layout.js

@@ -8,7 +8,8 @@ import {TreeNode} from './buchheim.js';
 type Props = {
   data: NodeGraph,
   onNodeClick: (e: Event, s: string) => void,
-  tree: TreeNode
+  tree: TreeNode,
+  nomsStore: string,
 }
 
 export default function Layout(props: Props) : React.Element {
@@ -51,6 +52,7 @@ export default function Layout(props: Props) : React.Element {
         canOpen={hasChildren}
         isOpen={!hasChildren || Boolean(treeNode.data.isOpen)}
         nomsRef={nomsRef}
+        nomsStore={props.nomsStore}
         onClick={(e) => props.onNodeClick(e, treeNode.id)}/>);
     children.push(n);
     lookup[treeNode.id] = treeNode;

clients/splore/src/main.js

@@ -25,34 +25,38 @@ import type {NodeGraph} from './buchheim.js';
 
 const data: NodeGraph = {nodes: {}, links: {}};
 let rootRef: Ref;
-let httpStore: HttpStore;
 let dataStore: DataStore;
 
 let renderNode: ?HTMLElement;
 
-const hash = {};
+const params = {};
 
 window.onload = load;
-window.onhashchange = load;
+window.onpopstate = load;
 window.onresize = render;
 
 function load() {
   renderNode = document.getElementById('splore');
-  location.hash.substr(1).split('&').forEach(pair => {
-    const [k, v] = pair.split('=');
-    hash[k] = v;
-  });
 
-  if (!hash.server) {
+  const paramsIdx = location.href.indexOf('?');
+  if (paramsIdx > -1) {
+    decodeURIComponent(location.href.slice(paramsIdx + 1)).split('&').forEach(pair => {
+      const [k, v] = pair.split('=');
+      params[k] = v;
+    });
+  }
+
+  if (!params.store) {
     renderPrompt();
     return;
   }
 
   const opts = {};
-  if (hash.token) {
-    opts['headers'] = {Authorization: `Bearer ${hash.token}`};
+  if (params.token) {
+    opts['headers'] = {Authorization: `Bearer ${params.token}`};
   }
-  httpStore = new HttpStore(hash.server, undefined, undefined, opts);
+
+  const httpStore = new HttpStore(params.store, undefined, undefined, opts);
   dataStore = new DataStore(httpStore);
 
   const setRootRef = ref => {
@@ -60,8 +64,8 @@ function load() {
     handleChunkLoad(ref, ref);
   };
 
-  if (hash.ref) {
-    setRootRef(Ref.parse(hash.ref));
+  if (params.ref) {
+    setRootRef(Ref.parse(params.ref));
   } else {
     httpStore.getRoot().then(setRootRef);
   }
@@ -232,25 +236,8 @@ function handleNodeClick(e: MouseEvent, id: string) {
   }
 }
 
-function setServer(url: string, token: ?string, ref: ?string) {
-  let hash = `server=${url}`;
-  if (token) {
-    hash += '&token=' + token;
-  }
-  if (ref) {
-    hash += '&ref=' + ref;
-  }
-  location.hash = hash;
-}
-
-type PromptState = {
-  server: string,
-}
-
-class Prompt extends React.Component<void, {}, PromptState> {
-  state: PromptState;
-
-  render() {
+class Prompt extends React.Component<void, {}, void> {
+  render(): React.Element {
     const fontStyle: {[key: string]: any} = {
       fontFamily: 'Menlo',
       fontSize: 14,
@@ -259,14 +246,20 @@ class Prompt extends React.Component<void, {}, PromptState> {
     return <div style={{display: 'flex', height: '100%', alignItems: 'center',
       justifyContent: 'center'}}>
       <div style={fontStyle}>
-        Can haz server?
+        Can haz datastore?
         <form style={{margin:'0.5em 0'}} onSubmit={() => this._handleOnSubmit()}>
-          <input type='text' ref='url' autoFocus={true} style={inputStyle}
-            defaultValue='http://api.noms.io/-/ds/[user]'/><br/>
+          <input type='text' ref='store' autoFocus={true} style={inputStyle}
+            defaultValue={params.store || 'http://api.noms.io/-/ds/[user]'}
+            placeholder='noms store URL'
+          />
           <input type='text' ref='token' style={inputStyle}
-            placeholder='auth token'/>
+            defaultValue={params.token}
+            placeholder='auth token'
+          />
           <input type='text' ref='ref' style={inputStyle}
-            placeholder='sha1-xyz (ref to jump to)' />
+            defaultValue={params.ref}
+            placeholder='sha1-xyz (ref to jump to)'
+          />
           <button type='submit'>OK</button>
         </form>
       </div>
@@ -274,8 +267,16 @@ class Prompt extends React.Component<void, {}, PromptState> {
   }
 
   _handleOnSubmit() {
-    const {url, token, ref} = this.refs;
-    setServer(url.value, token.value, ref.value);
+    const {store, token, ref} = this.refs;
+    let params = 'store=' + store.value;
+    if (token.value) {
+      params += '&token=' + token.value;
+    }
+    if (ref.value) {
+      params += '&ref=' + ref.value;
+    }
+    window.location.pushState({}, undefined, '?' + params);
+    render();
   }
 }
 
@@ -286,5 +287,7 @@ function renderPrompt() {
 function render() {
   const dt = new TreeNode(data, rootRef.toString(), null, 0, 0, {});
   layout(dt);
-  ReactDOM.render(<Layout tree={dt} data={data} onNodeClick={handleNodeClick}/>, renderNode);
+  ReactDOM.render(
+    <Layout tree={dt} data={data} onNodeClick={handleNodeClick} nomsStore={params.store}/>,
+    renderNode);
 }

clients/splore/src/node.js

@@ -3,7 +3,6 @@
 import classNames from 'classnames';
 import React from 'react';
 import {Ref} from '@attic/noms';
-import nomsServer from './noms_server.js';
 
 type Props = {
   canOpen: boolean,
@@ -17,6 +16,7 @@ type Props = {
   y: number,
   spaceX: number,
   nomsRef: ?Ref,
+  nomsStore: string,
   onClick: (e: Event, s: String) => void,
 };
 
@@ -50,7 +50,7 @@ export default class Node extends React.Component<void, Props, State> {
 
     let text = this.props.text;
     if (this.props.nomsRef) {
-      const url = `${nomsServer}/ref/${this.props.nomsRef.toString()}`;
+      const url = `${this.props.nomsStore}/ref/${this.props.nomsRef.toString()}`;
       text = <a href={url}>{text}</a>;
     }
 

clients/splore/src/noms_server.js

@@ -1,8 +0,0 @@
-// @flow
-
-const nomsServer: ?string = process.env.NOMS_SERVER;
-if (!nomsServer) {
-  throw new Error('NOMS_SERVER not set');
-}
-
-export {nomsServer as default};

clients/splore/webpack.config.js

@@ -1,3 +1 @@
-module.exports = require('noms-webpack-config')({
-  requiredEnvVars: ['NOMS_SERVER'],
-});
+module.exports = require('noms-webpack-config')();

cmd/noms-view/.gitignore

@@ -0,0 +1 @@
+noms-view

cmd/noms-view/README.md

@@ -0,0 +1,17 @@
+# noms-view
+
+Manages noms views. Usage:
+```
+noms view <flags> <command> ...
+```
+
+Supported commands:
+* `serve`: serves a noms from from a local server
+
+Examples:
+
+## serve
+
+```
+./noms-view serve ../../clients/splore store=ldb:/tmp/picasa
+```

cmd/noms-view/noms_view.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/attic-labs/noms/chunks"
+	"github.com/attic-labs/noms/constants"
+	"github.com/attic-labs/noms/d"
+	"github.com/attic-labs/noms/datas"
+	"github.com/julienschmidt/httprouter"
+)
+
+const (
+	dsPathPrefix = "/-ds"
+	serveCmd     = "serve"
+)
+
+var (
+	hostFlag = flag.String("host", "localhost:0", "Host to listen on")
+)
+
+type dataStoreRecord struct {
+	ds    datas.DataStore
+	alias string
+}
+
+type dataStoreRecords map[string]dataStoreRecord
+
+func main() {
+	usage := func() {
+		flag.PrintDefaults()
+		fmt.Printf("Usage: %s %s <view-dir> arg1=val1 arg2=val2...\n", os.Args[0], serveCmd)
+	}
+
+	flag.Parse()
+	flag.Usage = usage
+
+	if len(flag.Args()) < 2 || flag.Arg(0) != serveCmd {
+		usage()
+		os.Exit(1)
+	}
+
+	viewDir := flag.Arg(1)
+	qsValues, stores := constructQueryString(flag.Args()[2:])
+
+	router := &httprouter.Router{
+		HandleMethodNotAllowed: true,
+		NotFound:               http.FileServer(http.Dir(viewDir)),
+		RedirectFixedPath:      true,
+	}
+
+	prefix := dsPathPrefix + "/:store"
+	router.GET(prefix+constants.RefPath+":ref", routeToStore(stores, datas.HandleRef))
+	router.OPTIONS(prefix+constants.RefPath+":ref", routeToStore(stores, datas.HandleRef))
+	router.POST(prefix+constants.PostRefsPath, routeToStore(stores, datas.HandlePostRefs))
+	router.OPTIONS(prefix+constants.PostRefsPath, routeToStore(stores, datas.HandlePostRefs))
+	router.POST(prefix+constants.GetHasPath, routeToStore(stores, datas.HandleGetHasRefs))
+	router.OPTIONS(prefix+constants.GetHasPath, routeToStore(stores, datas.HandleGetHasRefs))
+	router.POST(prefix+constants.GetRefsPath, routeToStore(stores, datas.HandleGetRefs))
+	router.OPTIONS(prefix+constants.GetRefsPath, routeToStore(stores, datas.HandleGetRefs))
+	router.GET(prefix+constants.RootPath, routeToStore(stores, datas.HandleRootGet))
+	router.POST(prefix+constants.RootPath, routeToStore(stores, datas.HandleRootPost))
+	router.OPTIONS(prefix+constants.RootPath, routeToStore(stores, datas.HandleRootGet))
+
+	l, err := net.Listen("tcp", *hostFlag)
+	d.Chk.NoError(err)
+
+	srv := &http.Server{
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			router.ServeHTTP(w, req)
+		}),
+	}
+
+	qs := ""
+	if len(qsValues) > 0 {
+		qs = "?" + qsValues.Encode()
+	}
+
+	fmt.Printf("Starting view %s at http://%s%s\n", viewDir, l.Addr().String(), qs)
+	log.Fatal(srv.Serve(l))
+}
+
+func constructQueryString(args []string) (url.Values, dataStoreRecords) {
+	qsValues := url.Values{}
+	stores := dataStoreRecords{}
+
+	for _, arg := range args {
+		k, v, ok := split2(arg, "=")
+		if !ok {
+			continue
+		}
+
+		// Magically assume that ldb: prefixed arguments are references to ldb stores. If so, construct
+		// httpstore proxies to them, and rewrite the path to the client.
+		// TODO: When clients can declare a nomdl interface, this can be much stricter. There should be
+		// no need to search and attempt to string match every argument.
+		if strings.HasPrefix(v, "ldb:") {
+			_, path, _ := split2(v, ":")
+			record, ok := stores[path]
+			if !ok {
+				record.ds = datas.NewDataStore(chunks.NewLevelDBStore(path, "", 24, false))
+				// Identify the stores with a (abridged) hash of the file system path,
+				// so that the same URL always refers to the same database.
+				hash := sha1.Sum([]byte(path))
+				record.alias = hex.EncodeToString(hash[:])[:8]
+				stores[path] = record
+			}
+			v = fmt.Sprintf("%s/%s", dsPathPrefix, record.alias)
+		}
+
+		qsValues.Add(k, v)
+	}
+
+	return qsValues, stores
+}
+
+func routeToStore(stores dataStoreRecords, handler datas.Handler) httprouter.Handle {
+	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
+		store := params.ByName("store")
+		for _, record := range stores {
+			if record.alias == store {
+				handler(w, r, params, record.ds)
+				return
+			}
+		}
+		d.Chk.Fail("No store named", store)
+	}
+}
+
+func split2(s, sep string) (string, string, bool) {
+	substrs := strings.SplitN(s, sep, 2)
+	if len(substrs) != 2 {
+		fmt.Println("Invalid arg %s, must be of form k%sv", s, sep)
+		return "", "", false
+	}
+	return substrs[0], substrs[1], true
+}

cmd/noms-view/noms_view_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConstructQueryString(t *testing.T) {
+	assert := assert.New(t)
+	prefix := "TestConstructQueryString"
+
+	d1, e1 := ioutil.TempDir(os.TempDir(), prefix)
+	defer os.RemoveAll(d1)
+	d2, e2 := ioutil.TempDir(os.TempDir(), prefix)
+	defer os.RemoveAll(d2)
+
+	assert.NoError(e1)
+	assert.NoError(e2)
+
+	qs, stores := constructQueryString([]string{
+		"foo=bar",
+		"store1=ldb:" + d1,
+		"store2=ldb:" + d2,
+		"store3=ldb:" + d1,
+		"hello=world",
+	})
+
+	assert.Equal(5, len(qs))
+	assert.Equal("bar", qs.Get("foo"))
+	assert.True(strings.HasPrefix(qs.Get("store1"), dsPathPrefix))
+	assert.True(strings.HasPrefix(qs.Get("store2"), dsPathPrefix))
+	assert.True(strings.HasPrefix(qs.Get("store3"), dsPathPrefix))
+	assert.Equal(qs.Get("store1"), qs.Get("store3"))
+	assert.NotEqual(qs.Get("store1"), qs.Get("store2"))
+	assert.Equal(2, len(stores))
+}