Nomdex - for indexing and querying. (#2357)

This first version builds indexes and can query against a single one.

samples/go/nomdex/expr.go

@@ -0,0 +1,102 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+
+	"github.com/attic-labs/noms/go/types"
+)
+
+type expr interface {
+	ranges() queryRangeSlice
+	dbgPrintTree(w io.Writer, level int)
+}
+
+// logExpr represents a logical 'and' or 'or' expression between two other expressions. e.g.
+type logExpr struct {
+	op    boolOp
+	expr1 expr
+	expr2 expr
+}
+
+// compExpr represents a comparison between index values and constants. e.g. model-year-index > 1999
+type compExpr struct {
+	idxPath string
+	op      compOp
+	v1      types.Value
+}
+
+func (le logExpr) ranges() (ranges queryRangeSlice) {
+	rslice1 := le.expr1.ranges()
+	rslice2 := le.expr2.ranges()
+	rslice := queryRangeSlice{}
+
+	switch le.op {
+	case and:
+		if len(rslice1) == 0 || len(rslice2) == 0 {
+			return rslice
+		}
+		for _, r1 := range rslice1 {
+			for _, r2 := range rslice2 {
+				rslice = append(rslice, r1.and(r2)...)
+			}
+		}
+		sort.Sort(rslice)
+		return rslice
+	case or:
+		if len(rslice1) == 0 {
+			return rslice2
+		}
+		if len(rslice2) == 0 {
+			return rslice1
+		}
+		for _, r1 := range rslice1 {
+			for _, r2 := range rslice2 {
+				rslice = append(rslice, r1.or(r2)...)
+			}
+		}
+		sort.Sort(rslice)
+		return rslice
+	}
+	return queryRangeSlice{}
+}
+
+func (le logExpr) dbgPrintTree(w io.Writer, level int) {
+	fmt.Fprintf(w, "%*s%s\n", 2*level, "", le.op)
+	if le.expr1 != nil {
+		le.expr1.dbgPrintTree(w, level+1)
+	}
+	if le.expr2 != nil {
+		le.expr2.dbgPrintTree(w, level+1)
+	}
+}
+
+func (re compExpr) ranges() (ranges queryRangeSlice) {
+	var r queryRange
+	switch re.op {
+	case equals:
+		e := bound{value: re.v1, include: true}
+		r = queryRange{lower: e, upper: e}
+	case gt:
+		r = queryRange{lower: bound{re.v1, false, 0}, upper: bound{nil, true, 1}}
+	case gte:
+		r = queryRange{lower: bound{re.v1, true, 0}, upper: bound{nil, true, 1}}
+	case lt:
+		r = queryRange{lower: bound{nil, true, -1}, upper: bound{re.v1, false, 0}}
+	case lte:
+		r = queryRange{lower: bound{nil, true, -1}, upper: bound{re.v1, true, 0}}
+	}
+	return queryRangeSlice{r}
+}
+
+func (re compExpr) dbgPrintTree(w io.Writer, level int) {
+	buf := bytes.Buffer{}
+	types.WriteEncodedValue(&buf, re.v1)
+	fmt.Fprintf(w, "%*s%s %s %s\n", 2*level, "", re.idxPath, re.op, buf.String())
+}

samples/go/nomdex/nomdex.go

@@ -0,0 +1,82 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/attic-labs/noms/cmd/util"
+	"github.com/attic-labs/noms/go/d"
+	flag "github.com/juju/gnuflag"
+)
+
+var commands = []*util.Command{
+	update,
+	find,
+}
+
+var usageLine = `Nomdex builds indexes to support fast data access.`
+
+func main() {
+	progName := path.Base(os.Args[0])
+	util.InitHelp(progName, commands, usageLine)
+	flag.Usage = util.Usage
+	flag.Parse(false)
+
+	args := flag.Args()
+	if len(args) < 1 {
+		util.Usage()
+		return
+	}
+
+	if args[0] == "help" {
+		util.Help(args[1:])
+		return
+	}
+
+	for _, cmd := range commands {
+		if cmd.Name() == args[0] {
+			flags := cmd.Flags()
+			flags.Usage = cmd.Usage
+
+			flags.Parse(true, args[1:])
+			args = flags.Args()
+			if cmd.Nargs != 0 && len(args) < cmd.Nargs {
+				cmd.Usage()
+			}
+			exitCode := cmd.Run(args)
+			if exitCode != 0 {
+				os.Exit(exitCode)
+			}
+			return
+		}
+	}
+
+	fmt.Fprintf(os.Stderr, "noms: unknown command %q\n", args[0])
+	util.Usage()
+}
+
+func printError(err error, msgAndArgs ...interface{}) bool {
+	if err != nil {
+		err := d.Unwrap(err)
+		switch len(msgAndArgs) {
+		case 0:
+			fmt.Fprintf(os.Stderr, "error: %s\n", err)
+		case 1:
+			fmt.Fprintf(os.Stderr, "%s%s\n", msgAndArgs[0], err)
+		default:
+			format, ok := msgAndArgs[0].(string)
+			if ok {
+				s1 := fmt.Sprintf(format, msgAndArgs[1:]...)
+				fmt.Fprintf(os.Stderr, "%s%s\n", s1, err)
+			} else {
+				fmt.Fprintf(os.Stderr, "error: %s\n", err)
+			}
+		}
+	}
+	return err != nil
+}

samples/go/nomdex/nomdex_find.go

@@ -0,0 +1,128 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/attic-labs/noms/cmd/util"
+	"github.com/attic-labs/noms/go/datas"
+	"github.com/attic-labs/noms/go/spec"
+	"github.com/attic-labs/noms/go/types"
+	"github.com/attic-labs/noms/go/util/outputpager"
+	flag "github.com/juju/gnuflag"
+)
+
+var find = &util.Command{
+	Run:       runFind,
+	UsageLine: "find --db <database spec> <query>",
+	Short:     "Print objects from prebuilt indexes",
+	Long:      "Print object from prebuild indexes (long desc)",
+	Flags:     setupFindFlags,
+	Nargs:     1,
+}
+
+func setupFindFlags() *flag.FlagSet {
+	flagSet := flag.NewFlagSet("find", flag.ExitOnError)
+	flagSet.StringVar(&indexPath, "index", "", "dataset containing index")
+	outputpager.RegisterOutputpagerFlags(flagSet)
+	return flagSet
+}
+
+func runFind(args []string) int {
+	query := args[0]
+	if indexPath == "" {
+		fmt.Fprintf(os.Stderr, "Missing required 'index' arg\n")
+		flag.Usage()
+		return 1
+	}
+
+	db, index, err := openIndex(indexPath)
+	if printError(err, "Unable to open database/index\n\terror: ") {
+		return 1
+	}
+	defer db.Close()
+
+	expr, err := parseQuery(query)
+	if err != nil {
+		fmt.Printf("err: %s\n", err)
+		return 1
+	}
+
+	pgr := outputpager.Start()
+	defer pgr.Stop()
+
+	ranges := expr.ranges()
+	printObjects(pgr.Writer, index, ranges)
+
+	return 0
+}
+
+func printObjects(w io.Writer, index types.Map, ranges queryRangeSlice) {
+	cnt := 0
+	first := true
+	printObjectForRange := func(index types.Map, r queryRange) {
+		index.IterFrom(r.lower.value, func(k, v types.Value) bool {
+			if first && r.lower.value != nil && !r.lower.include && r.lower.value.Equals(k) {
+				return false
+			}
+			if r.upper.value != nil {
+				if !r.upper.include && r.upper.value.Equals(k) {
+					return true
+				}
+				if r.upper.value.Less(k) {
+					return true
+				}
+			}
+			s := v.(types.Set)
+			s.IterAll(func(v types.Value) {
+				types.WriteEncodedValue(w, v)
+				fmt.Fprintf(w, "\n")
+				cnt++
+			})
+			return false
+		})
+	}
+	for _, r := range ranges {
+		printObjectForRange(index, r)
+	}
+	fmt.Fprintf(w, "Found %d objects\n", cnt)
+}
+
+func openIndex(idxPath string) (datas.Database, types.Map, error) {
+	db, value, err := spec.GetPath(idxPath)
+	if err != nil {
+		return nil, types.Map{}, err
+	}
+
+	var index types.Map
+	s, ok := value.(types.Struct)
+	if ok && datas.IsCommitType(s.Type()) {
+		index, ok = s.Get("value").(types.Map)
+		if !ok {
+			return nil, types.Map{}, fmt.Errorf("Value of commit is not a valid index")
+		}
+	} else {
+		index, ok = value.(types.Map)
+		if !ok {
+			return nil, types.Map{}, fmt.Errorf("%s is not a valid index", outDsArg)
+		}
+	}
+
+	// Todo: make this type be Map<String | Number>, Set<Value>> once Issue #2326 gets resolved and
+	// IsSubtype() returns the correct value.
+	typ := types.MakeMapType(
+		types.MakeUnionType(types.StringType, types.NumberType),
+		types.ValueType)
+
+	if !types.IsSubtype(typ, index.Type()) {
+		err := fmt.Errorf("%s does not point to a suitable index type:", idxPath)
+		return nil, types.Map{}, err
+	}
+
+	return db, index, nil
+}

samples/go/nomdex/nomdex_update.go

@@ -0,0 +1,142 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/attic-labs/noms/cmd/util"
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/datas"
+	"github.com/attic-labs/noms/go/dataset"
+	"github.com/attic-labs/noms/go/spec"
+	"github.com/attic-labs/noms/go/types"
+	"github.com/attic-labs/noms/go/util/status"
+	"github.com/attic-labs/noms/go/walk"
+	humanize "github.com/dustin/go-humanize"
+	flag "github.com/juju/gnuflag"
+)
+
+var (
+	inPathArg  = ""
+	outDsArg   = ""
+	relPathArg = ""
+)
+
+var update = &util.Command{
+	Run:       runUpdate,
+	UsageLine: "up --in-path <path> --out-ds <dspath> --by <relativepath>",
+	Short:     "Build/Update an index",
+	Long:      "Traverse all values starting at root and add values found at 'relativePath' to a map found at 'out-ds'\n",
+	Flags:     setupUpdateFlags,
+	Nargs:     0,
+}
+
+func setupUpdateFlags() *flag.FlagSet {
+	flagSet := flag.NewFlagSet("up", flag.ExitOnError)
+	flagSet.StringVar(&inPathArg, "in-path", "", "a value to search for items to index within ")
+	flagSet.StringVar(&outDsArg, "out-ds", "", "name of dataset to save the results to")
+	flagSet.StringVar(&relPathArg, "by", "", "a path relative to all the items in <in-path> to index by")
+	return flagSet
+}
+
+type StreamingSetEntry struct {
+	valChan chan<- types.Value
+	setChan <-chan types.Set
+}
+
+type IndexMap map[types.Value]StreamingSetEntry
+
+type Index struct {
+	m     IndexMap
+	mutex sync.Mutex
+}
+
+func runUpdate(args []string) int {
+	db, rootObject, err := spec.GetPath(inPathArg)
+	d.Chk.NoError(err)
+
+	if rootObject == nil {
+		fmt.Printf("Object not found: %s\n", inPathArg)
+		return 1
+	}
+
+	outDs := dataset.NewDataset(db, outDsArg)
+	relPath, err := types.ParsePath(relPathArg)
+	if printError(err, "Error parsing -by value\n\t") {
+		return 1
+	}
+
+	typeCacheMutex := sync.Mutex{}
+	typeCache := map[*types.Type]bool{}
+
+	index := Index{m: IndexMap{}}
+	walk.AllP(rootObject, db, func(v types.Value, r *types.Ref) {
+		typ := v.Type()
+		typeCacheMutex.Lock()
+		hasPath, ok := typeCache[typ]
+		typeCacheMutex.Unlock()
+		if !ok || hasPath {
+			pathResolved := false
+			tv := relPath.Resolve(v)
+			if tv != nil {
+				index.Add(db, tv, v)
+				pathResolved = true
+			}
+			if !ok {
+				typeCacheMutex.Lock()
+				typeCache[typ] = pathResolved
+				typeCacheMutex.Unlock()
+			}
+		}
+	}, 4)
+
+	status.Done()
+	indexMap := writeToStreamingMap(db, index.m)
+	outDs, err = outDs.Commit(indexMap, dataset.CommitOptions{})
+	d.Chk.NoError(err)
+	fmt.Printf("Committed index with %d entries to dataset: %s\n", indexMap.Len(), outDsArg)
+	return 0
+}
+
+var cnt = int64(0)
+
+func (idx *Index) Add(db datas.Database, k, v types.Value) {
+	idx.mutex.Lock()
+	defer idx.mutex.Unlock()
+
+	cnt++
+	se, ok := idx.m[k]
+	if !ok {
+		valChan := make(chan types.Value)
+		setChan := types.NewStreamingSet(db, valChan)
+		se = StreamingSetEntry{valChan: valChan, setChan: setChan}
+		idx.m[k] = se
+	}
+	se.valChan <- v
+	status.Printf("Indexed %s objects", humanize.Comma(cnt))
+}
+
+func writeToStreamingMap(db datas.Database, indexMap IndexMap) types.Map {
+	itemCnt := len(indexMap)
+	writtenCnt := int64(0)
+	indexedCnt := int64(0)
+	kvChan := make(chan types.Value)
+	mapChan := types.NewStreamingMap(db, kvChan)
+	for k, v := range indexMap {
+		close(v.valChan)
+		s := <-v.setChan
+		kvChan <- k
+		kvChan <- s
+		indexedCnt += int64(s.Len())
+		delete(indexMap, k)
+		writtenCnt++
+		status.Printf("Wrote %s/%d keys, %s indexedObjects", humanize.Comma(writtenCnt), itemCnt, humanize.Comma(indexedCnt))
+	}
+	close(kvChan)
+	status.Done()
+	return <-mapChan
+}

samples/go/nomdex/parser.go

@@ -0,0 +1,225 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"text/scanner"
+	"unicode"
+
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/types"
+)
+
+/**** Query language BNF
+  query := expr
+  expr := expr boolop relExpr | group
+  relExpr := index relOp value
+  group := '(' expr ')' | relExpr
+  boolOp := 'and' | 'or'
+  relOp := '=' | '<' | '<=' | '>' | '>='
+  value := "<string>" | int | float
+*/
+
+type compOp string
+type boolOp string
+
+const (
+	equals compOp = "="
+	gt     compOp = ">"
+	gte    compOp = ">="
+	lt     compOp = "<"
+	lte    compOp = "<="
+	openP         = "("
+	closeP        = ")"
+	and    boolOp = "and"
+	or     boolOp = "or"
+)
+
+var (
+	reloperators  = []compOp{equals, gt, gte, lt, lte}
+	booloperators = []boolOp{and, or}
+	indexPath     = ""
+)
+
+type qScanner struct {
+	s           scanner.Scanner
+	peekedToken rune
+	peekedText  string
+	peeked      bool
+}
+
+func (qs *qScanner) Scan() rune {
+	var r rune
+	if qs.peeked {
+		r = qs.peekedToken
+		qs.peeked = false
+	} else {
+		r = qs.s.Scan()
+	}
+	return r
+}
+
+func (qs *qScanner) Peek() rune {
+	var r rune
+
+	if !qs.peeked {
+		qs.peekedToken = qs.s.Scan()
+		qs.peekedText = qs.s.TokenText()
+		qs.peeked = true
+	}
+	r = qs.peekedToken
+	return r
+}
+
+func (qs *qScanner) TokenText() string {
+	var text string
+	if qs.peeked {
+		text = qs.peekedText
+	} else {
+		text = qs.s.TokenText()
+	}
+	return text
+}
+
+func (qs *qScanner) Pos() scanner.Position {
+	return qs.s.Pos()
+}
+
+func parseQuery(q string) (expr, error) {
+	s := NewQueryScanner(q)
+	var expr expr
+	err := d.Try(func() {
+		expr = s.parseExpr(0)
+	})
+	return expr, err
+}
+
+func NewQueryScanner(query string) *qScanner {
+	isIdentRune := func(r rune, i int) bool {
+		identChars := ":/.>=-"
+		startIdentChars := "><"
+		if i == 0 {
+			return unicode.IsLetter(r) || strings.ContainsRune(startIdentChars, r)
+		}
+		return unicode.IsLetter(r) || unicode.IsDigit(r) || strings.ContainsRune(identChars, r)
+	}
+
+	errorFunc := func(s *scanner.Scanner, msg string) {
+		d.PanicIfError(fmt.Errorf("%s, pos: %s\n", msg, s.Pos()))
+	}
+
+	var s scanner.Scanner
+	s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanStrings | scanner.SkipComments
+	s.Init(strings.NewReader(query))
+	s.IsIdentRune = isIdentRune
+	s.Error = errorFunc
+	qs := qScanner{s: s}
+	return &qs
+}
+
+func (s *qScanner) parseExpr(level int) expr {
+	tok := s.Scan()
+	switch tok {
+	case '(':
+		expr := s.parseExpr(level + 1)
+		tok := s.Scan()
+		if tok != ')' {
+			d.PanicIfError(fmt.Errorf("missing ending paren for expr"))
+		} else {
+			tok = s.Peek()
+			if tok == ')' {
+				return expr
+			}
+			tok = s.Scan()
+			text := s.TokenText()
+			switch {
+			case tok == scanner.Ident && isBoolOp(text):
+				op := boolOp(text)
+				expr2 := s.parseExpr(level + 1)
+				return logExpr{op, expr, expr2}
+			case tok == scanner.EOF:
+				return expr
+			default:
+				d.PanicIfError(fmt.Errorf("extra text found at end of expr, tok: %d, text: %s", int(tok), s.TokenText()))
+			}
+		}
+	case '_':
+		rexpr := s.parseCompExpr(level+1, s.TokenText())
+		tok := s.Peek()
+		switch tok {
+		case ')':
+			return rexpr
+		case rune(scanner.Ident):
+			tok = s.Scan()
+			text := s.TokenText()
+			if isBoolOp(text) {
+				op := boolOp(text)
+				expr2 := s.parseExpr(level + 1)
+				return logExpr{op, rexpr, expr2}
+			} else {
+				d.PanicIfError(fmt.Errorf("expected boolean op, found: %s, level: %d", text, level))
+			}
+		case rune(scanner.EOF):
+			return rexpr
+		default:
+			tok = s.Scan()
+		}
+	default:
+		d.PanicIfError(fmt.Errorf("unexpected token in expr: %s, %d", s.TokenText(), tok))
+	}
+	return logExpr{}
+}
+
+func (s *qScanner) parseCompExpr(level int, indexPath string) compExpr {
+	tok := s.Scan()
+	text := s.TokenText()
+	if !isCompOp(text) {
+		d.PanicIfError(fmt.Errorf("expected relop token but found: '%s'", text))
+	}
+	op := compOp(text)
+	tok = s.Scan()
+	text = s.TokenText()
+	switch tok {
+	case scanner.String:
+		return compExpr{indexPath, op, valueFromString(text)}
+	case scanner.Float:
+		f, _ := strconv.ParseFloat(text, 64)
+		return compExpr{indexPath, op, types.Number(f)}
+	case scanner.Int:
+		f, _ := strconv.ParseInt(text, 10, 64)
+		return compExpr{indexPath, op, types.Number(f)}
+	}
+	d.PanicIfError(fmt.Errorf("expected value token, found: '%s'", text))
+	return compExpr{}
+}
+
+func valueFromString(t string) types.Value {
+	l := len(t)
+	if l < 2 && t[0] == '"' && t[l-1] == '"' {
+		d.PanicIfError(fmt.Errorf("Unable to get value from token: %s", t))
+	}
+	return types.String(t[1 : l-1])
+}
+
+func isCompOp(s string) bool {
+	for _, op := range reloperators {
+		if s == string(op) {
+			return true
+		}
+	}
+	return false
+}
+
+func isBoolOp(s string) bool {
+	for _, op := range booloperators {
+		if s == string(op) {
+			return true
+		}
+	}
+	return false
+}

samples/go/nomdex/parser_test.go

@@ -0,0 +1,119 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"testing"
+	"text/scanner"
+
+	"github.com/attic-labs/noms/go/types"
+	"github.com/attic-labs/testify/assert"
+)
+
+type scannerResult struct {
+	tok  int
+	text string
+}
+
+type parseResult struct {
+	query string
+	ex    expr
+}
+
+func TestQueryScanner(t *testing.T) {
+	assert := assert.New(t)
+
+	s := NewQueryScanner(`9 (99.9) "99.9" and or http://localhost:8000/cli-tour::yo <= >= < > = _`)
+
+	scannerResults := []scannerResult{
+		scannerResult{tok: scanner.Int, text: "9"},
+		scannerResult{tok: int('('), text: "("},
+		scannerResult{tok: scanner.Float, text: "99.9"},
+		scannerResult{tok: int(')'), text: ")"},
+		scannerResult{tok: scanner.String, text: `"99.9"`},
+		scannerResult{tok: scanner.Ident, text: "and"},
+		scannerResult{tok: scanner.Ident, text: "or"},
+		scannerResult{tok: scanner.Ident, text: "http://localhost:8000/cli-tour::yo"},
+		scannerResult{tok: scanner.Ident, text: "<="},
+		scannerResult{tok: scanner.Ident, text: ">="},
+		scannerResult{tok: scanner.Ident, text: "<"},
+		scannerResult{tok: scanner.Ident, text: ">"},
+		scannerResult{tok: int('='), text: "="},
+		scannerResult{tok: int('_'), text: "_"},
+	}
+
+	for _, sr := range scannerResults {
+		tok := s.Scan()
+		assert.Equal(sr.tok, int(tok), "expected text: %s, found: %s, pos: %s", sr.text, s.TokenText(), s.Pos())
+		assert.Equal(sr.text, s.TokenText())
+	}
+	tok := s.Scan()
+	assert.Equal(scanner.EOF, int(tok))
+}
+
+func TestPeek(t *testing.T) {
+	assert := assert.New(t)
+
+	s := NewQueryScanner(`_ < "one"`)
+	scannerResults := []scannerResult{
+		scannerResult{tok: int('_'), text: "_"},
+		scannerResult{tok: scanner.Ident, text: "<"},
+		scannerResult{tok: scanner.String, text: `"one"`},
+		scannerResult{tok: scanner.EOF, text: ""},
+	}
+
+	for _, sr := range scannerResults {
+		assert.Equal(sr.tok, int(s.Peek()))
+		assert.Equal(sr.text, s.TokenText())
+		assert.Equal(sr.tok, int(s.Scan()))
+		assert.Equal(sr.text, s.TokenText())
+	}
+}
+
+func TestParsing(t *testing.T) {
+	assert := assert.New(t)
+
+	re1 := compExpr{"_", equals, types.Number(2015)}
+	re2 := compExpr{"_", gte, types.Number(2020)}
+	re3 := compExpr{"_", lte, types.Number(2022)}
+	re4 := compExpr{"_", lt, types.Number(2030)}
+
+	queries := []parseResult{
+		parseResult{`_ = 2015`, re1},
+		parseResult{`(_ = 2015 )`, re1},
+		parseResult{`(((_ = 2015 ) ))`, re1},
+		parseResult{`_ = 2015 or _ >= 2020`, logExpr{or, re1, re2}},
+		parseResult{`(_ = 2015) or _ >= 2020`, logExpr{or, re1, re2}},
+		parseResult{`_ = 2015 or (_ >= 2020)`, logExpr{or, re1, re2}},
+		parseResult{`(_ = 2015 or _ >= 2020)`, logExpr{or, re1, re2}},
+		parseResult{`(_ = 2015 or _ >= 2020) and _ <= 2022`, logExpr{and, logExpr{or, re1, re2}, re3}},
+		parseResult{`_ = 2015 or _ >= 2020 and _ <= 2022`, logExpr{or, re1, logExpr{and, re2, re3}}},
+		parseResult{`_ = 2015 or _ >= 2020 and _ <= 2022 or _ < 2030`, logExpr{or, re1, logExpr{and, re2, logExpr{or, re3, re4}}}},
+		parseResult{`(_ = 2015 or _ >= 2020) and (_ <= 2022 or _ < 2030)`, logExpr{and, logExpr{or, re1, re2}, logExpr{or, re3, re4}}},
+	}
+
+	for _, pr := range queries {
+		expr, err := parseQuery(pr.query)
+		assert.NoError(err)
+		assert.Equal(pr.ex, expr, "bad query: %s", pr.query)
+	}
+
+	badQueries := []string{
+		`sdfsd = 2015`,
+		`_ = "unfinished string`,
+		`_ and 2015`,
+		`_ < `,
+		`_ < 2015 and ()`,
+		`_ < 2015 an _ > 2016`,
+		`(_ < 2015) what`,
+		`(_< 2015`,
+	}
+
+	for _, q := range badQueries {
+		expr, err := parseQuery(q)
+		assert.Error(err)
+		assert.Nil(expr)
+	}
+}

samples/go/nomdex/query_range.go

@@ -0,0 +1,159 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"sort"
+
+	"github.com/attic-labs/noms/go/types"
+)
+
+type bound struct {
+	value    types.Value
+	include  bool
+	infinity int8
+}
+
+func (b bound) isLessThanOrEqual(o bound) (res bool) {
+	return b.equals(o) || b.isLessThan(o)
+}
+
+//
+func (b bound) isLessThan(o bound) (res bool) {
+	if b.infinity < o.infinity {
+		return true
+	}
+
+	if b.infinity > o.infinity {
+		return false
+	}
+
+	if b.value.Less(o.value) {
+		return true
+	}
+
+	if b.value.Equals(o.value) {
+		if !b.include && o.include {
+			return true
+		}
+	}
+	return false
+}
+
+func (b bound) isGreaterThanOrEqual(o bound) (res bool) {
+	return !b.isLessThan(o)
+}
+
+func (b bound) isGreaterThan(o bound) (res bool) {
+	return !b.equals(o) || !b.isLessThan(o)
+}
+
+func (b bound) equals(o bound) bool {
+	return b.infinity == o.infinity && b.include == o.include &&
+		(b.value == nil && o.value == nil || (b.value != nil && o.value != nil && b.value.Equals(o.value)))
+}
+
+func (b bound) String() string {
+	var s1 string
+	if b.value == nil {
+		s1 = "<nil>"
+	} else {
+		buf := bytes.Buffer{}
+		types.WriteEncodedValue(&buf, b.value)
+		s1 = buf.String()
+	}
+	return fmt.Sprintf("bound{v: %s, include: %t, infinity: %d}", s1, b.include, b.infinity)
+}
+
+func (v bound) minValue(o bound) (res bound) {
+	if v.isLessThan(o) {
+		return v
+	}
+	return o
+}
+
+func (v bound) maxValue(o bound) (res bound) {
+	if v.isLessThan(o) {
+		return o
+	}
+	return v
+}
+
+type queryRange struct {
+	lower bound
+	upper bound
+}
+
+func (r queryRange) and(o queryRange) (rangeDescs queryRangeSlice) {
+	if !r.intersects(o) {
+		return []queryRange{}
+	}
+
+	lower := r.lower.maxValue(o.lower)
+	upper := r.upper.minValue(o.upper)
+	return []queryRange{queryRange{lower, upper}}
+}
+
+func (r queryRange) or(o queryRange) (rSlice queryRangeSlice) {
+	if r.intersects(o) {
+		v1 := r.lower.minValue(o.lower)
+		v2 := r.upper.maxValue(o.upper)
+		return queryRangeSlice{queryRange{v1, v2}}
+	}
+	rSlice = queryRangeSlice{r, o}
+	sort.Sort(rSlice)
+	return rSlice
+}
+
+func (r queryRange) intersects(o queryRange) (res bool) {
+	if r.lower.isGreaterThanOrEqual(o.lower) && r.lower.isLessThanOrEqual(o.upper) {
+		return true
+	}
+	if r.upper.isGreaterThanOrEqual(o.lower) && r.upper.isLessThanOrEqual(o.upper) {
+		return true
+	}
+	if o.lower.isGreaterThanOrEqual(r.lower) && o.lower.isLessThanOrEqual(r.upper) {
+		return true
+	}
+	if o.upper.isGreaterThanOrEqual(r.lower) && o.upper.isLessThanOrEqual(r.upper) {
+		return true
+	}
+	return false
+}
+
+func (r queryRange) String() string {
+	return fmt.Sprintf("queryRange{lower: %s, upper: %s", r.lower, r.upper)
+}
+
+// queryRangeSlice defines the sort.Interface. This implementation sorts queryRanges by
+// the lower bound in ascending order.
+type queryRangeSlice []queryRange
+
+func (rSlice queryRangeSlice) Len() int {
+	return len(rSlice)
+}
+
+func (rSlice queryRangeSlice) Swap(i, j int) {
+	rSlice[i], rSlice[j] = rSlice[j], rSlice[i]
+}
+
+func (rSlice queryRangeSlice) Less(i, j int) bool {
+	return !rSlice[i].lower.equals(rSlice[j].lower) && rSlice[i].lower.isLessThanOrEqual(rSlice[j].lower)
+}
+
+func (rSlice queryRangeSlice) dbgPrint(w io.Writer) {
+	for i, rd := range rSlice {
+		if i == 0 {
+			fmt.Fprintf(w, "\n#################\n")
+		}
+		fmt.Fprintf(w, "queryRange %d: %s\n", i, rd)
+	}
+	if len(rSlice) > 0 {
+		fmt.Fprintf(w, "\n")
+	}
+}

samples/go/nomdex/query_range_test.go

@@ -0,0 +1,150 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"testing"
+
+	"github.com/attic-labs/noms/go/types"
+	"github.com/attic-labs/testify/assert"
+)
+
+const nilHolder = -1000000
+
+var (
+	r1  = qr(2, true, 5, true)
+	r2  = qr(0, true, 8, true)
+	r3  = qr(0, true, 3, true)
+	r4  = qr(3, true, 8, true)
+	r5  = qr(0, true, 1, true)
+	r6  = qr(6, true, 10, true)
+	r7  = qr(nilHolder, true, 10, true)
+	r8  = qr(3, true, nilHolder, true)
+	r10 = qr(2, true, 5, false)
+	r11 = qr(5, true, 10, true)
+)
+
+func newBound(i int, include bool, infinity int) bound {
+	var v types.Value
+	if i != nilHolder {
+		v = types.Number(i)
+	}
+	return bound{value: v, include: include, infinity: int8(infinity)}
+}
+
+func qr(lower int, lowerIncl bool, upper int, upperIncl bool) queryRange {
+	lowerInf := 0
+	if lower == nilHolder {
+		lowerInf = -1
+	}
+	upperInf := 0
+	if upper == nilHolder {
+		upperInf = 1
+	}
+	return queryRange{newBound(lower, lowerIncl, lowerInf), newBound(upper, upperIncl, upperInf)}
+}
+
+func TestRangeIntersects(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.True(r1.intersects(r2))
+	assert.True(r1.intersects(r3))
+	assert.True(r1.intersects(r4))
+	assert.True(r2.intersects(r1))
+	assert.True(r1.intersects(r7))
+	assert.True(r1.intersects(r8))
+	assert.True(r3.intersects(r4))
+	assert.True(r3.intersects(r4))
+
+	assert.False(r1.intersects(r5))
+	assert.False(r1.intersects(r6))
+	assert.False(r10.intersects(r11))
+}
+
+func TestRangeAnd(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.Empty(r1.and(r5))
+	assert.Empty(r1.and(r6))
+
+	assert.Equal(r1, r1.and(r2)[0])
+	assert.Equal(r1, r2.and(r1)[0])
+
+	expected := qr(3, true, 5, true)
+	assert.Equal(expected, r1.and(r4)[0])
+}
+
+func TestRangeOr(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.Equal(r2, r1.or(r2)[0])
+
+	expected := qr(0, true, 5, true)
+	assert.Equal(expected, r1.or(r3)[0])
+
+	expectedSlice := queryRangeSlice{r5, r1}
+	assert.Equal(expectedSlice, r1.or(r5))
+	assert.Equal(expectedSlice, r5.or(r1))
+}
+
+func TestIsLessThan(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.True(newBound(1, true, 0).isLessThanOrEqual(newBound(2, true, 0)))
+	assert.False(newBound(2, true, 0).isLessThanOrEqual(newBound(1, true, 0)))
+	assert.True(newBound(1, true, 0).isLessThanOrEqual(newBound(1, true, 0)))
+
+	assert.True(newBound(1, false, 0).isLessThanOrEqual(newBound(2, false, 0)))
+	assert.False(newBound(2, false, 0).isLessThanOrEqual(newBound(1, false, 0)))
+	assert.True(newBound(1, false, 0).isLessThanOrEqual(newBound(1, false, 0)))
+
+	assert.False(newBound(1, true, 0).isLessThanOrEqual(newBound(1, false, 0)))
+	assert.True(newBound(1, false, 0).isLessThanOrEqual(newBound(1, true, 0)))
+
+	assert.True(newBound(nilHolder, true, -1).isLessThanOrEqual(newBound(1, true, 0)))
+	assert.False(newBound(1, false, 0).isLessThanOrEqual(newBound(nilHolder, true, -1)))
+}
+
+func TestIsGreaterThan(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.True(newBound(2, true, 0).isGreaterThanOrEqual(newBound(1, true, 0)))
+	assert.False(newBound(1, true, 0).isGreaterThanOrEqual(newBound(2, true, 0)))
+	assert.True(newBound(1, true, 0).isGreaterThanOrEqual(newBound(1, true, 0)))
+
+	assert.True(newBound(2, false, 0).isGreaterThanOrEqual(newBound(1, false, 0)))
+	assert.False(newBound(1, false, 0).isGreaterThanOrEqual(newBound(2, false, 0)))
+	assert.True(newBound(1, false, 0).isGreaterThanOrEqual(newBound(1, false, 0)))
+
+	assert.True(newBound(1, true, 0).isGreaterThanOrEqual(newBound(1, false, 0)))
+	assert.False(newBound(1, false, 0).isGreaterThanOrEqual(newBound(2, true, 0)))
+
+	assert.True(newBound(nilHolder, true, 1).isGreaterThanOrEqual(newBound(1, true, 0)))
+	assert.False(newBound(1, true, 0).isGreaterThanOrEqual(newBound(nilHolder, true, 1)))
+}
+
+func TestMinValue(t *testing.T) {
+	assert := assert.New(t)
+	ve1 := newBound(5, false, 0)
+	ve2 := newBound(5, true, 0)
+	ve3 := newBound(nilHolder, true, -1)
+	ve4 := newBound(nilHolder, true, 1)
+
+	assert.Equal(ve1, ve1.minValue(ve2))
+	assert.Equal(ve3, ve1.minValue(ve3))
+	assert.Equal(ve1, ve1.minValue(ve4))
+}
+
+func TestMaxValue(t *testing.T) {
+	assert := assert.New(t)
+	ve1 := newBound(5, false, 0)
+	ve2 := newBound(5, true, 0)
+	ve3 := newBound(nilHolder, true, -1)
+	ve4 := newBound(nilHolder, true, 1)
+
+	assert.Equal(ve2, ve1.maxValue(ve2))
+	assert.Equal(ve1, ve1.maxValue(ve3))
+	assert.Equal(ve4, ve1.maxValue(ve4))
+}