Implement experimental block store (#2870)

* Move NBS into Noms

* vendor in deps

go/nbs/README.md

@@ -0,0 +1,8 @@
+# Noms Block Store
+An experimental storage layer for [noms](https://github.com/attic-labs/noms).
+
+- Provides storage for a content-addressed DAG of nodes (with exactly one root), where each node is encoded as a sequence of bytes and addressed by a 20-byte hash of the byte-sequence.
+- There is no `update` or `delete`, only `insert`, `update root` and `garbage collect`.
+- Insertion of any novel byte-sequence is durable only upon updating the root.
+- File-level multiprocess concurrency is supported, with optimistic locking for multiple writers.
+- Writers need not worry about re-writing duplicate chunks. NBS will efficiently detect and drop (most) duplicates.

go/nbs/benchmarks/.gitignore

@@ -0,0 +1 @@
+benchmarks

go/nbs/benchmarks/block_store_benchmarks.go

@@ -0,0 +1,122 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"fmt"
+
+	"sync"
+
+	"github.com/attic-labs/noms/go/chunks"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/attic-labs/noms/go/types"
+	"github.com/attic-labs/testify/assert"
+)
+
+type blockStore interface {
+	types.BatchStore
+	GetMany(hashes []hash.Hash) []chunks.Chunk
+}
+
+type storeOpenFn func() blockStore
+
+func benchmarkNovelWrite(refreshStore storeOpenFn, src *dataSource, t assert.TestingT) bool {
+	store := refreshStore()
+	writeToEmptyStore(store, src, t)
+	assert.NoError(t, store.Close())
+	return true
+}
+
+func writeToEmptyStore(store blockStore, src *dataSource, t assert.TestingT) {
+	root := store.Root()
+	assert.Equal(t, hash.Hash{}, root)
+
+	chunx := goReadChunks(src)
+	for c := range chunx {
+		store.SchedulePut(*c, 1, types.Hints{})
+	}
+	newRoot := chunks.NewChunk([]byte("root"))
+	store.SchedulePut(newRoot, 1, types.Hints{})
+	assert.True(t, store.UpdateRoot(newRoot.Hash(), root))
+}
+
+func goReadChunks(src *dataSource) <-chan *chunks.Chunk {
+	chunx := make(chan *chunks.Chunk, 1024)
+	go func() {
+		src.ReadChunks(chunx)
+		close(chunx)
+	}()
+	return chunx
+}
+
+func benchmarkNoRefreshWrite(openStore storeOpenFn, src *dataSource, t assert.TestingT) {
+	store := openStore()
+	chunx := goReadChunks(src)
+	for c := range chunx {
+		store.SchedulePut(*c, 1, types.Hints{})
+	}
+	assert.NoError(t, store.Close())
+}
+
+func verifyChunk(h hash.Hash, c chunks.Chunk) {
+	if len(c.Data()) == 0 {
+		panic(fmt.Sprintf("Failed to fetch %s\n", h.String()))
+	}
+}
+
+func benchmarkRead(openStore storeOpenFn, hashes hashSlice, src *dataSource, t assert.TestingT) {
+	store := openStore()
+	for _, h := range hashes {
+		verifyChunk(h, store.Get(h))
+	}
+	assert.NoError(t, store.Close())
+}
+
+func verifyChunks(hashes []hash.Hash, batch []chunks.Chunk) {
+	for i, c := range batch {
+		verifyChunk(hashes[i], c)
+	}
+}
+
+func benchmarkReadMany(openStore storeOpenFn, hashes hashSlice, src *dataSource, batchSize, concurrency int, t assert.TestingT) {
+	store := openStore()
+	batch := make([]hash.Hash, 0, batchSize)
+
+	wg := sync.WaitGroup{}
+	limit := make(chan struct{}, concurrency)
+
+	for _, h := range hashes {
+		batch = append(batch, h)
+
+		if len(batch) == batchSize {
+			limit <- struct{}{}
+			wg.Add(1)
+			go func(hashes []hash.Hash) {
+				verifyChunks(hashes, store.GetMany(hashes))
+				wg.Done()
+				<-limit
+			}(batch)
+
+			batch = make([]hash.Hash, 0, batchSize)
+		}
+	}
+
+	if len(batch) > 0 {
+		verifyChunks(batch, store.GetMany(batch))
+	}
+
+	wg.Wait()
+
+	assert.NoError(t, store.Close())
+}
+
+func ensureNovelWrite(wrote bool, openStore storeOpenFn, src *dataSource, t assert.TestingT) bool {
+	if !wrote {
+		store := openStore()
+		defer store.Close()
+		writeToEmptyStore(store, src, t)
+	}
+	return true
+}

go/nbs/benchmarks/cachedrop/drop_cache.go

@@ -0,0 +1,23 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"log"
+	"os"
+)
+
+const dropCaches = "/proc/sys/vm/drop_caches"
+
+func main() {
+	f, err := os.OpenFile(dropCaches, os.O_WRONLY, 0666)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	if _, err := f.WriteString("1"); err != nil {
+		log.Fatalln(err)
+	}
+}

go/nbs/benchmarks/chunker/.gitignore

@@ -0,0 +1 @@
+chunker

go/nbs/benchmarks/chunker/main.go

@@ -0,0 +1,58 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/nbs/benchmarks/gen"
+	"github.com/dustin/go-humanize"
+	flag "github.com/juju/gnuflag"
+)
+
+const (
+	KB               = uint64(1 << 10)
+	MB               = uint64(1 << 20)
+	averageChunkSize = 4 * KB
+)
+
+var (
+	genSize    = flag.Uint64("gen", 1024, "MiB of data to generate and chunk")
+	chunkInput = flag.Bool("chunk", false, "Treat arg as data file to chunk")
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "%s [--gen=<data in MiB>|--chunk] /path/to/file\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+	flag.Parse(true)
+	if flag.NArg() != 1 {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	fileName := flag.Arg(0)
+
+	var fd *os.File
+	var err error
+	if *chunkInput {
+		fd, err = os.Open(fileName)
+		d.Chk.NoError(err)
+		defer fd.Close()
+	} else {
+		fd, err = gen.OpenOrGenerateDataFile(fileName, (*genSize)*humanize.MiByte)
+		d.Chk.NoError(err)
+		defer fd.Close()
+	}
+
+	cm := gen.OpenOrBuildChunkMap(fileName+".chunks", fd)
+	defer cm.Close()
+
+	return
+}

go/nbs/benchmarks/data_source.go

@@ -0,0 +1,147 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/attic-labs/noms/go/chunks"
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/attic-labs/noms/go/nbs/benchmarks/gen"
+	"github.com/dustin/go-humanize"
+	flag "github.com/juju/gnuflag"
+)
+
+var readFile = flag.String("input-file", "", "A file full of test data. Creates and saves associated .chunks file at runtime if it doesn't yet exist. If none is specified, data and .chunks files will be generated and saved.")
+
+const averageChunkSize = 4 * 1024
+
+type hashSlice []hash.Hash
+
+func (s hashSlice) Len() int {
+	return len(s)
+}
+
+func (s hashSlice) Less(i, j int) bool {
+	return s[i].Less(s[j])
+}
+
+func (s hashSlice) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+type dataSource struct {
+	data, cm            *os.File
+	totalData, dataRead uint64
+	hashes              hashSlice
+}
+
+func getInput(dataSize uint64) (src *dataSource, err error) {
+	filename := *readFile
+	if filename == "" {
+		filename = humanize.IBytes(dataSize) + ".bin"
+	}
+	data, err := gen.OpenOrGenerateDataFile(filename, dataSize)
+	if err != nil {
+		return nil, err
+	}
+	chunkFile := data.Name() + ".chunks"
+	cm := gen.OpenOrBuildChunkMap(chunkFile, data)
+	fmt.Println("Reading from", filename, "with chunks", chunkFile)
+	src = &dataSource{data: data, cm: cm, totalData: dataSize}
+	tuples := make(chan offsetTuple, 1024)
+	go func() {
+		src.readTuples(tuples)
+		close(tuples)
+	}()
+	for ot := range tuples {
+		src.hashes = append(src.hashes, ot.h)
+	}
+	return src, err
+}
+
+type offsetTuple struct {
+	h hash.Hash
+	l uint64
+}
+
+func (src *dataSource) PrimeFilesystemCache() {
+	bufData := bufio.NewReaderSize(src.data, 10*humanize.MiByte)
+	tuples := make(chan offsetTuple, 16)
+	go func() {
+		src.readTuples(tuples)
+		close(tuples)
+	}()
+
+	for ot := range tuples {
+		buff := make([]byte, ot.l)
+		n, err := io.ReadFull(bufData, buff)
+		d.Chk.NoError(err)
+		d.Chk.True(uint64(n) == ot.l)
+	}
+}
+
+func (src *dataSource) ReadChunks(chunkChan chan<- *chunks.Chunk) {
+	bufData := bufio.NewReaderSize(src.data, humanize.MiByte)
+	tuples := make(chan offsetTuple, 1024)
+	go func() {
+		src.readTuples(tuples)
+		close(tuples)
+	}()
+
+	for ot := range tuples {
+		buff := make([]byte, ot.l)
+		n, err := io.ReadFull(bufData, buff)
+		d.Chk.NoError(err)
+		d.Chk.True(uint64(n) == ot.l)
+		c := chunks.NewChunkWithHash(ot.h, buff)
+		chunkChan <- &c
+	}
+}
+
+func (src *dataSource) GetHashes() hashSlice {
+	out := make(hashSlice, len(src.hashes))
+	copy(out, src.hashes)
+	return out
+}
+
+func (src *dataSource) readTuples(tuples chan<- offsetTuple) {
+	src.reset()
+
+	otBuf := [gen.OffsetTupleLen]byte{}
+	cm := bufio.NewReaderSize(src.cm, humanize.MiByte)
+	ot := offsetTuple{}
+
+	for src.dataRead < src.totalData {
+		n, err := io.ReadFull(cm, otBuf[:])
+		if err != nil {
+			d.Chk.True(err == io.EOF)
+			return
+		}
+		d.Chk.True(n == gen.OffsetTupleLen)
+		ot.h = hash.FromSlice(otBuf[:20])
+		ot.l = uint64(binary.BigEndian.Uint32(otBuf[20:]))
+		src.dataRead += ot.l
+		tuples <- ot
+	}
+}
+
+func (src *dataSource) reset() {
+	_, err := src.data.Seek(0, os.SEEK_SET)
+	d.Chk.NoError(err)
+	_, err = src.cm.Seek(0, os.SEEK_SET)
+	d.Chk.NoError(err)
+	src.dataRead = 0
+}
+
+func (src *dataSource) Close() {
+	d.Chk.NoError(src.data.Close())
+	d.Chk.NoError(src.cm.Close())
+}

go/nbs/benchmarks/drop_cache_linux.go

@@ -0,0 +1,13 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// +build linux
+
+package main
+
+import "os/exec"
+
+func dropCache() error {
+	return exec.Command("./drop_cache").Run()
+}

go/nbs/benchmarks/drop_cache_other.go

@@ -0,0 +1,11 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// +build !linux
+
+package main
+
+func dropCache() error {
+	return nil
+}

go/nbs/benchmarks/file_block_store.go

@@ -0,0 +1,56 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+
+	"github.com/attic-labs/noms/go/chunks"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/attic-labs/noms/go/types"
+	"github.com/dustin/go-humanize"
+)
+
+type fileBlockStore struct {
+	bw *bufio.Writer
+	w  io.WriteCloser
+}
+
+func newFileBlockStore(w io.WriteCloser) blockStore {
+	return fileBlockStore{bufio.NewWriterSize(w, humanize.MiByte), w}
+}
+
+func (fb fileBlockStore) Get(h hash.Hash) chunks.Chunk {
+	panic("not impl")
+}
+
+func (fb fileBlockStore) GetMany(batch []hash.Hash) (result []chunks.Chunk) {
+	panic("not impl")
+}
+
+func (fb fileBlockStore) SchedulePut(c chunks.Chunk, refHeight uint64, hints types.Hints) {
+	io.Copy(fb.bw, bytes.NewReader(c.Data()))
+}
+
+func (fb fileBlockStore) AddHints(hints types.Hints) {
+}
+
+func (fb fileBlockStore) Flush() {}
+
+func (fb fileBlockStore) Close() error {
+	fb.w.Close()
+	return nil
+}
+
+func (fb fileBlockStore) Root() hash.Hash {
+	return hash.Hash{}
+}
+
+func (fb fileBlockStore) UpdateRoot(current, last hash.Hash) bool {
+	fb.bw.Flush()
+	return true
+}

go/nbs/benchmarks/gen/gen.go

@@ -0,0 +1,154 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package gen
+
+import (
+	"bytes"
+	"crypto/sha512"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/dustin/go-humanize"
+)
+
+const (
+	OffsetTupleLen   = 24
+	averageChunkSize = 4 * uint64(1<<10) // 4KB
+)
+
+func OpenOrGenerateDataFile(name string, totalData uint64) (data *os.File, err error) {
+	data, err = os.Open(name)
+	if os.IsNotExist(err) {
+		data, err = os.Create(name)
+		fmt.Printf("Creating data file with %s\n", humanize.IBytes(totalData))
+		generateData(data, totalData)
+		_, err = data.Seek(0, os.SEEK_SET)
+		d.Chk.NoError(err)
+		return data, nil
+	}
+	d.Chk.NoError(err)
+
+	info, err := data.Stat()
+	d.Chk.NoError(err)
+	if uint64(info.Size()) < totalData {
+		data.Close()
+		return nil, fmt.Errorf("%s is too small to benchmark with %s", name, humanize.IBytes(totalData))
+	}
+	return data, nil
+}
+
+func OpenOrBuildChunkMap(name string, data *os.File) *os.File {
+	cm, err := os.Open(name)
+	if os.IsNotExist(err) {
+		cm, err = os.Create(name)
+		fmt.Printf("Chunking %s into chunk-map: %s ...", data.Name(), name)
+		cc := chunk(cm, data)
+		fmt.Println(cc, " chunks")
+
+		_, err = cm.Seek(0, os.SEEK_SET)
+		d.Chk.NoError(err)
+		return cm
+	}
+	d.Chk.NoError(err)
+	return cm
+}
+
+func generateData(w io.Writer, totalData uint64) {
+	r := &randomByteReader{}
+
+	buff := [humanize.MiByte]byte{}
+	bs := buff[:]
+	buffIdx := 0
+
+	for bc := uint64(0); bc < totalData; bc++ {
+		b, _ := r.ReadByte()
+		bs[buffIdx] = b
+		buffIdx++
+		if buffIdx == int(humanize.MiByte) {
+			io.Copy(w, bytes.NewReader(bs))
+			buffIdx = 0
+		}
+	}
+}
+
+type randomByteReader struct {
+	rand    *rand.Rand
+	scratch [2 * averageChunkSize]byte
+	pos     int
+}
+
+func (rbr *randomByteReader) ReadByte() (byte, error) {
+	if rbr.rand == nil {
+		rbr.rand = rand.New(rand.NewSource(0))
+		rbr.pos = cap(rbr.scratch)
+	}
+	if rbr.pos >= cap(rbr.scratch) {
+		rbr.rand.Read(rbr.scratch[:])
+		rbr.pos = 0
+	}
+	b := rbr.scratch[rbr.pos]
+	rbr.pos++
+	return b, nil
+}
+
+func (rbr *randomByteReader) Close() error {
+	return nil
+}
+
+type offsetTuple [OffsetTupleLen]byte
+
+func chunk(w io.Writer, r io.Reader) (chunkCount int) {
+	buff := [humanize.MiByte]byte{}
+	bs := buff[:]
+	buffIdx := uint64(0)
+	rv := newRollingValueHasher()
+	sha := sha512.New()
+	ot := offsetTuple{}
+	lastOffset := uint64(0)
+
+	var err error
+	var n int
+
+	writeChunk := func() {
+		chunkCount++
+		var d []byte
+		d = sha.Sum(d)
+		copy(ot[:hash.ByteLen], d)
+
+		chunkLength := uint32(buffIdx - lastOffset)
+
+		binary.BigEndian.PutUint32(ot[hash.ByteLen:], chunkLength)
+
+		io.Copy(w, bytes.NewReader(ot[:]))
+
+		lastOffset = buffIdx
+		sha.Reset()
+	}
+
+	for err == nil {
+		n, err = io.ReadFull(r, bs)
+
+		for i := uint64(0); i < uint64(n); i++ {
+			b := bs[i]
+			buffIdx++
+			sha.Write(bs[i : i+1])
+
+			if rv.HashByte(b) {
+				writeChunk()
+			}
+		}
+	}
+
+	if lastOffset < buffIdx {
+		writeChunk()
+	}
+
+	return
+}

go/nbs/benchmarks/gen/rolling_value_hasher.go

@@ -0,0 +1,27 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package gen
+
+import "github.com/kch42/buzhash"
+
+const (
+	chunkPattern = uint32(1<<12 - 1) // Avg Chunk Size of 4k
+
+	// The window size to use for computing the rolling hash. This is way more than necessary assuming random data (two bytes would be sufficient with a target chunk size of 4k). The benefit of a larger window is it allows for better distribution on input with lower entropy. At a target chunk size of 4k, any given byte changing has roughly a 1.5% chance of affecting an existing boundary, which seems like an acceptable trade-off.
+	chunkWindow = uint32(64)
+)
+
+type rollingValueHasher struct {
+	bz *buzhash.BuzHash
+}
+
+func newRollingValueHasher() *rollingValueHasher {
+	return &rollingValueHasher{buzhash.NewBuzHash(chunkWindow)}
+}
+
+func (rv *rollingValueHasher) HashByte(b byte) bool {
+	rv.bz.HashByte(b)
+	return rv.bz.Sum32()&chunkPattern == chunkPattern
+}

go/nbs/benchmarks/main.go

@@ -0,0 +1,155 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"sort"
+	"time"
+
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/nbs"
+	"github.com/attic-labs/noms/go/util/profile"
+	"github.com/attic-labs/testify/assert"
+	"github.com/dustin/go-humanize"
+	flag "github.com/juju/gnuflag"
+)
+
+var (
+	count    = flag.Int("c", 10, "Number of iterations to run")
+	dataSize = flag.Uint64("data", 4096, "MiB of data to test with")
+	mtMiB    = flag.Uint64("mem", 64, "Size in MiB of memTable")
+	useNBS   = flag.String("useNBS", "", "Existing Database to use for not-WriteNovel benchmarks")
+	toNBS    = flag.String("toNBS", "", "Write to an NBS store in the given directory")
+	toFile   = flag.String("toFile", "", "Write to a file in the given directory")
+)
+
+type panickingBencher struct {
+	n int
+}
+
+func (pb panickingBencher) Errorf(format string, args ...interface{}) {
+	panic(fmt.Sprintf(format, args...))
+}
+
+func (pb panickingBencher) N() int {
+	return pb.n
+}
+
+func (pb panickingBencher) ResetTimer() {}
+func (pb panickingBencher) StartTimer() {}
+func (pb panickingBencher) StopTimer()  {}
+
+func main() {
+	profile.RegisterProfileFlags(flag.CommandLine)
+	flag.Parse(true)
+
+	if flag.NArg() < 1 {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	pb := panickingBencher{*count}
+
+	src, err := getInput((*dataSize) * humanize.MiByte)
+	d.PanicIfError(err)
+	defer src.Close()
+
+	bufSize := (*mtMiB) * humanize.MiByte
+
+	open := newNullBlockStore
+	wrote := false
+	var writeDB func()
+	var refresh func() blockStore
+	if *toNBS != "" || *toFile != "" {
+		var dir string
+		if *toNBS != "" {
+			dir = makeTempDir(*toNBS, pb)
+			open = func() blockStore {
+				return nbs.NewBlockStore(dir, bufSize)
+			}
+		} else if *toFile != "" {
+			dir = makeTempDir(*toFile, pb)
+			open = func() blockStore {
+				f, err := ioutil.TempFile(dir, "")
+				d.Chk.NoError(err)
+				return newFileBlockStore(f)
+			}
+		}
+		defer os.RemoveAll(dir)
+		writeDB = func() { wrote = ensureNovelWrite(wrote, open, src, pb) }
+		refresh = func() blockStore {
+			os.RemoveAll(dir)
+			os.MkdirAll(dir, 0777)
+			return open()
+		}
+	} else {
+		if *useNBS != "" {
+			open = func() blockStore {
+				return nbs.NewBlockStore(*useNBS, bufSize)
+			}
+		}
+		writeDB = func() {}
+		refresh = func() blockStore { panic("WriteNovel unsupported with --useLDB and --useNBS") }
+	}
+
+	benchmarks := []struct {
+		name  string
+		setup func()
+		run   func()
+	}{
+		{"WriteNovel", func() {}, func() { wrote = benchmarkNovelWrite(refresh, src, pb) }},
+		{"WriteDuplicate", writeDB, func() { benchmarkNoRefreshWrite(open, src, pb) }},
+		{"ReadSequential", writeDB, func() { benchmarkRead(open, src.GetHashes(), src, pb) }},
+		{"ReadManySequential", writeDB, func() { benchmarkReadMany(open, src.GetHashes(), src, 1<<8, 6, pb) }},
+		{"ReadHashOrder", writeDB, func() {
+			ordered := src.GetHashes()
+			sort.Sort(ordered)
+			benchmarkRead(open, ordered, src, pb)
+		}},
+	}
+	w := 0
+	for _, bm := range benchmarks {
+		if len(bm.name) > w {
+			w = len(bm.name)
+		}
+	}
+	defer profile.MaybeStartProfile().Stop()
+	for _, bm := range benchmarks {
+		if matched, _ := regexp.MatchString(flag.Arg(0), bm.name); matched {
+			trialName := fmt.Sprintf("%dMiB/%sbuffer/%-[3]*s", *dataSize, humanize.IBytes(bufSize), w, bm.name)
+			bm.setup()
+			dur := time.Duration(0)
+			var trials []time.Duration
+			for i := 0; i < *count; i++ {
+				d.Chk.NoError(dropCache())
+				src.PrimeFilesystemCache()
+
+				t := time.Now()
+				bm.run()
+				trialDur := time.Since(t)
+				trials = append(trials, trialDur)
+				dur += trialDur
+			}
+			fmt.Printf("%s\t%d\t%ss/iter %v\n", trialName, *count, humanize.FormatFloat("", (dur/time.Duration(*count)).Seconds()), formatTrials(trials))
+		}
+	}
+}
+
+func makeTempDir(tmpdir string, t assert.TestingT) (dir string) {
+	dir, err := ioutil.TempDir(tmpdir, "")
+	assert.NoError(t, err)
+	return
+}
+
+func formatTrials(trials []time.Duration) (formatted []string) {
+	for _, trial := range trials {
+		formatted = append(formatted, humanize.FormatFloat("", trial.Seconds()))
+	}
+	return
+}

go/nbs/benchmarks/null_block_store.go

@@ -0,0 +1,47 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"github.com/attic-labs/noms/go/chunks"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/attic-labs/noms/go/types"
+)
+
+type nullBlockStore struct {
+	bogus int32
+}
+
+func newNullBlockStore() blockStore {
+	return nullBlockStore{}
+}
+
+func (nb nullBlockStore) Get(h hash.Hash) chunks.Chunk {
+	panic("not impl")
+}
+
+func (nb nullBlockStore) GetMany(batch []hash.Hash) (result []chunks.Chunk) {
+	panic("not impl")
+}
+
+func (nb nullBlockStore) SchedulePut(c chunks.Chunk, refHeight uint64, hints types.Hints) {
+}
+
+func (nb nullBlockStore) AddHints(hints types.Hints) {
+}
+
+func (nb nullBlockStore) Flush() {}
+
+func (nb nullBlockStore) Close() error {
+	return nil
+}
+
+func (nb nullBlockStore) Root() hash.Hash {
+	return hash.Hash{}
+}
+
+func (nb nullBlockStore) UpdateRoot(current, last hash.Hash) bool {
+	return true
+}

go/nbs/block_store_test.go

@@ -0,0 +1,98 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/attic-labs/noms/go/chunks"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/attic-labs/testify/assert"
+	"github.com/attic-labs/testify/suite"
+)
+
+func TestBlockStoreSuite(t *testing.T) {
+	suite.Run(t, &BlockStoreSuite{})
+}
+
+type BlockStoreSuite struct {
+	suite.Suite
+	dir        string
+	store      *NomsBlockStore
+	putCountFn func() int
+}
+
+func (suite *BlockStoreSuite) SetupTest() {
+	var err error
+	suite.dir, err = ioutil.TempDir("", "")
+	suite.NoError(err)
+	suite.store = NewBlockStore(suite.dir, defaultMemTableSize)
+	suite.putCountFn = func() int {
+		return int(suite.store.putCount)
+	}
+}
+
+func (suite *BlockStoreSuite) TearDownTest() {
+	suite.store.Close()
+	os.Remove(suite.dir)
+}
+
+func (suite *BlockStoreSuite) TestChunkStorePut() {
+	input := "abc"
+	c := chunks.NewChunk([]byte(input))
+	suite.store.Put(c)
+	h := c.Hash()
+
+	// See http://www.di-mgt.com.au/sha_testvectors.html
+	suite.Equal("rmnjb8cjc5tblj21ed4qs821649eduie", h.String())
+
+	suite.store.UpdateRoot(h, suite.store.Root()) // Commit writes
+
+	// And reading it via the API should work...
+	assertInputInStore(input, h, suite.store, suite.Assert())
+	if suite.putCountFn != nil {
+		suite.Equal(1, suite.putCountFn())
+	}
+
+	// Re-writing the same data should cause a second put
+	c = chunks.NewChunk([]byte(input))
+	suite.store.Put(c)
+	suite.Equal(h, c.Hash())
+	assertInputInStore(input, h, suite.store, suite.Assert())
+	suite.store.UpdateRoot(h, suite.store.Root()) // Commit writes
+
+	if suite.putCountFn != nil {
+		suite.Equal(2, suite.putCountFn())
+	}
+}
+
+func (suite *BlockStoreSuite) TestChunkStorePutMany() {
+	input1, input2 := "abc", "def"
+	c1, c2 := chunks.NewChunk([]byte(input1)), chunks.NewChunk([]byte(input2))
+	suite.store.PutMany([]chunks.Chunk{c1, c2})
+
+	suite.store.UpdateRoot(c1.Hash(), suite.store.Root()) // Commit writes
+
+	// And reading it via the API should work...
+	assertInputInStore(input1, c1.Hash(), suite.store, suite.Assert())
+	assertInputInStore(input2, c2.Hash(), suite.store, suite.Assert())
+	if suite.putCountFn != nil {
+		suite.Equal(2, suite.putCountFn())
+	}
+}
+
+func assertInputInStore(input string, h hash.Hash, s chunks.ChunkStore, assert *assert.Assertions) {
+	chunk := s.Get(h)
+	assert.False(chunk.IsEmpty(), "Shouldn't get empty chunk for %s", h.String())
+	assert.Equal(input, string(chunk.Data()))
+}
+
+func (suite *BlockStoreSuite) TestChunkStoreGetNonExisting() {
+	h := hash.Parse("11111111111111111111111111111111")
+	c := suite.store.Get(h)
+	suite.True(c.IsEmpty())
+}

go/nbs/compact.go

@@ -0,0 +1,50 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"bytes"
+	"io/ioutil"
+	"math"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"io"
+
+	"github.com/attic-labs/noms/go/d"
+)
+
+var maxInt = int64(math.MaxInt64)
+
+func init() {
+	if strconv.IntSize == 32 {
+		maxInt = math.MaxInt32
+	}
+}
+
+func compact(dir string, mt *memTable, haver chunkReader) (name addr, wrote bool) {
+	tempName, h, wrote := func() (string, addr, bool) {
+		temp, err := ioutil.TempFile(dir, "nbs_table_")
+		d.PanicIfError(err)
+		defer checkClose(temp)
+
+		maxSize := maxTableSize(uint64(len(mt.order)), mt.totalData)
+		buff := make([]byte, maxSize)
+		tw := newTableWriter(buff)
+		mt.write(tw, haver)
+		tableSize, h := tw.finish()
+		io.Copy(temp, bytes.NewReader(buff[:tableSize]))
+
+		return temp.Name(), h, tw.totalPhysicalData > 0
+	}()
+	if wrote {
+		err := os.Rename(tempName, filepath.Join(dir, h.String()))
+		d.PanicIfError(err)
+	} else {
+		os.Remove(tempName)
+	}
+	return h, wrote
+}

go/nbs/compact_test.go

@@ -0,0 +1,70 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/attic-labs/testify/assert"
+)
+
+func TestCompactMemTable(t *testing.T) {
+	assert := assert.New(t)
+	mt := newMemTable(1024)
+
+	chunks := [][]byte{
+		[]byte("hello2"),
+		[]byte("goodbye2"),
+		[]byte("badbye2"),
+	}
+
+	for _, c := range chunks {
+		assert.True(mt.addChunk(computeAddr(c), c))
+	}
+
+	dir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(dir)
+
+	tableAddr, wrote := compact(dir, mt, nil)
+	if assert.True(wrote) {
+		buff, err := ioutil.ReadFile(filepath.Join(dir, tableAddr.String()))
+		assert.NoError(err)
+		tr := newTableReader(buff, memReaderAt(buff))
+		for _, c := range chunks {
+			assert.True(tr.has(computeAddr(c)))
+		}
+	}
+}
+
+func TestCompactMemTableNoData(t *testing.T) {
+	assert := assert.New(t)
+	mt := newMemTable(1024)
+	existingTable := newMemTable(1024)
+
+	chunks := [][]byte{
+		[]byte("hello2"),
+		[]byte("goodbye2"),
+		[]byte("badbye2"),
+	}
+
+	for _, c := range chunks {
+		assert.True(mt.addChunk(computeAddr(c), c))
+		assert.True(existingTable.addChunk(computeAddr(c), c))
+	}
+
+	dir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(dir)
+
+	tableAddr, wrote := compact(dir, mt, existingTable)
+	assert.False(wrote)
+
+	_, err = os.Stat(filepath.Join(dir, tableAddr.String()))
+	assert.True(os.IsNotExist(err), "%v", err)
+}

go/nbs/file_manifest.go

@@ -0,0 +1,174 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/attic-labs/noms/go/constants"
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/hash"
+)
+
+const (
+	manifestFileName = "manifest"
+	lockFileName     = "LOCK"
+)
+
+// fileManifest provides access to a NomsBlockStore manifest stored on disk in |dir|. The format
+// is currently human readable:
+//
+// |-- String --|-- String --|-------- String --------|-- String --|- String --|...|-- String --|- String --|
+// | nbs version:Noms version:Base32-encoded root hash:table 1 hash:table 1 cnt:...:table N hash:table N cnt|
+type fileManifest struct {
+	dir string
+}
+
+type tableSpec struct {
+	name       addr
+	chunkCount uint64
+}
+
+// ParseIfExists looks for a LOCK and manifest file in fm.dir. If it finds
+// them, it takes the lock, parses the manifest and returns its contents,
+// setting |exists| to true. If not, it sets |exists| to false and returns. In
+// that case, the other return values are undefined. If |readHook| is non-nil,
+// it will be executed while ParseIfExists() holds the manfiest file lock.
+// This is to allow for race condition testing.
+func (fm fileManifest) ParseIfExists(readHook func()) (exists bool, vers string, root hash.Hash, tableSpecs []tableSpec) {
+	// !exists(lockFileName) => unitialized store
+	if l := openIfExists(filepath.Join(fm.dir, lockFileName)); l != nil {
+		var f io.ReadCloser
+		func() {
+			d.PanicIfError(unix.Flock(int(l.Fd()), unix.LOCK_EX))
+			defer checkClose(l) // releases the flock()
+
+			if readHook != nil {
+				readHook()
+			}
+			f = openIfExists(filepath.Join(fm.dir, manifestFileName))
+		}()
+
+		if f != nil {
+			defer checkClose(f)
+			exists = true
+			vers, root, tableSpecs = parseManifest(f)
+		}
+	}
+	return
+}
+
+// Returns nil if path does not exist
+func openIfExists(path string) *os.File {
+	f, err := os.Open(path)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	d.PanicIfError(err)
+	return f
+}
+
+func parseManifest(r io.Reader) (string, hash.Hash, []tableSpec) {
+	manifest, err := ioutil.ReadAll(r)
+	d.PanicIfError(err)
+
+	slices := bytes.Split(manifest, []byte(":"))
+	if len(slices) < 3 || len(slices)%2 == 0 {
+		d.Chk.Fail("Malformed manifest: " + string(manifest))
+	}
+	d.PanicIfFalse(StorageVersion == string(slices[0]))
+
+	tableInfo := slices[3:]
+	specs := make([]tableSpec, len(tableInfo)/2)
+	for i := range specs {
+		specs[i].name = ParseAddr(tableInfo[2*i])
+		specs[i].chunkCount, err = strconv.ParseUint(string(tableInfo[2*i+1]), 10, 64)
+		d.PanicIfError(err)
+	}
+
+	return string(slices[1]), hash.Parse(string(slices[2])), specs
+}
+
+func writeManifest(temp io.Writer, root hash.Hash, tables chunkSources) {
+	strs := make([]string, 2*len(tables)+3)
+	strs[0], strs[1], strs[2] = StorageVersion, constants.NomsVersion, root.String()
+	tableInfo := strs[3:]
+	for i, t := range tables {
+		tableInfo[2*i] = t.hash().String()
+		tableInfo[2*i+1] = strconv.FormatUint(t.count(), 10)
+	}
+	_, err := io.WriteString(temp, strings.Join(strs, ":"))
+	d.PanicIfError(err)
+}
+
+// Update optimistically tries to write a new manifest, containing |newRoot|
+// and the elements of |tables|. If the existing manifest on disk doesn't
+// contain |root|, Update fails and returns the parsed contents of the
+// manifest on disk. Callers should check that |actual| == |newRoot| upon
+// return and, if not, merge any desired new table information with the
+// contents of |tableSpecs| before trying again.
+// If writeHook is non-nil, it will be invoked wile the manifest file lock is
+// held. This is to allow for testing of race conditions.
+func (fm fileManifest) Update(tables chunkSources, root, newRoot hash.Hash, writeHook func()) (actual hash.Hash, tableSpecs []tableSpec) {
+
+	// Write a temporary manifest file, to be renamed over manifestFileName upon success.
+	// The closure here ensures this file is closed before moving on.
+	tempManifestPath := func() string {
+		temp, err := ioutil.TempFile(fm.dir, "nbs_manifest_")
+		d.PanicIfError(err)
+		defer checkClose(temp)
+		writeManifest(temp, newRoot, tables)
+		return temp.Name()
+	}()
+	defer os.Remove(tempManifestPath) // If we rename below, this will be a no-op
+
+	// Take manifest file lock
+	defer checkClose(flock(filepath.Join(fm.dir, lockFileName))) // closing releases the lock
+
+	// writeHook is for testing, allowing other code to slip in and try to do stuff while we hold the lock.
+	if writeHook != nil {
+		writeHook()
+	}
+
+	// Read current manifest (if it exists). The closure ensures that the file is closed before moving on, so we can rename over it later if need be.
+	manifestPath := filepath.Join(fm.dir, manifestFileName)
+	func() {
+		if f := openIfExists(manifestPath); f != nil {
+			defer checkClose(f)
+
+			var mVers string
+			mVers, actual, tableSpecs = parseManifest(f)
+			d.PanicIfFalse(constants.NomsVersion == mVers)
+		} else {
+			d.Chk.True(root == hash.Hash{})
+		}
+	}()
+
+	if root != actual {
+		return actual, tableSpecs
+	}
+	err := os.Rename(tempManifestPath, manifestPath)
+	d.PanicIfError(err)
+	return newRoot, nil
+}
+
+func checkClose(c io.Closer) {
+	d.PanicIfError(c.Close())
+}
+
+func flock(lockFilePath string) io.Closer {
+	l, err := os.Create(lockFilePath)
+	d.PanicIfError(err)
+	d.PanicIfError(unix.Flock(int(l.Fd()), unix.LOCK_EX))
+	return l
+}

go/nbs/file_manifest_test.go

@@ -0,0 +1,152 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/attic-labs/noms/go/constants"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/attic-labs/testify/assert"
+)
+
+func makeFileManifestTempDir(t *testing.T) fileManifest {
+	dir, err := ioutil.TempDir("", "")
+	assert.NoError(t, err)
+	return fileManifest{dir}
+}
+
+func TestFileManifestParseIfExists(t *testing.T) {
+	assert := assert.New(t)
+	fm := makeFileManifestTempDir(t)
+	defer os.RemoveAll(fm.dir)
+
+	exists, vers, root, tableSpecs := fm.ParseIfExists(nil)
+	assert.False(exists)
+
+	// Simulate another process writing a manifest (with an old Noms version).
+	newRoot := hash.FromData([]byte("new root"))
+	tableName := hash.FromData([]byte("table1"))
+	b, err := clobberManifest(fm.dir, strings.Join([]string{StorageVersion, "0", newRoot.String(), tableName.String(), "0"}, ":"))
+	assert.NoError(err, string(b))
+
+	// ParseIfExists should now reflect the manifest written above.
+	exists, vers, root, tableSpecs = fm.ParseIfExists(nil)
+	assert.True(exists)
+	assert.Equal("0", vers)
+	assert.Equal(newRoot, root)
+	if assert.Len(tableSpecs, 1) {
+		assert.Equal(tableName.String(), tableSpecs[0].name.String())
+		assert.Equal(uint64(0), tableSpecs[0].chunkCount)
+	}
+}
+
+func TestFileManifestParseIfExistsHoldsLock(t *testing.T) {
+	assert := assert.New(t)
+	fm := makeFileManifestTempDir(t)
+	defer os.RemoveAll(fm.dir)
+
+	// Simulate another process writing a manifest.
+	newRoot := hash.FromData([]byte("new root"))
+	tableName := hash.FromData([]byte("table1"))
+	b, err := clobberManifest(fm.dir, strings.Join([]string{StorageVersion, constants.NomsVersion, newRoot.String(), tableName.String(), "0"}, ":"))
+	assert.NoError(err, string(b))
+
+	// ParseIfExists should now reflect the manifest written above.
+	exists, vers, root, tableSpecs := fm.ParseIfExists(func() {
+		// This should fail to get the lock, and therefore _not_ clobber the manifest.
+		badRoot := hash.FromData([]byte("bad root"))
+		b, err := tryClobberManifest(fm.dir, strings.Join([]string{StorageVersion, "0", badRoot.String(), tableName.String(), "0"}, ":"))
+		assert.NoError(err, string(b))
+	})
+
+	assert.True(exists)
+	assert.Equal(constants.NomsVersion, vers)
+	assert.Equal(newRoot, root)
+	if assert.Len(tableSpecs, 1) {
+		assert.Equal(tableName.String(), tableSpecs[0].name.String())
+		assert.Equal(uint64(0), tableSpecs[0].chunkCount)
+	}
+}
+
+func TestFileManifestUpdate(t *testing.T) {
+	assert := assert.New(t)
+	fm := makeFileManifestTempDir(t)
+	defer os.RemoveAll(fm.dir)
+
+	// Simulate another process having already put old Noms data in dir/.
+	b, err := clobberManifest(fm.dir, strings.Join([]string{StorageVersion, "0", hash.Hash{}.String()}, ":"))
+	assert.NoError(err, string(b))
+
+	assert.Panics(func() { fm.Update(nil, hash.Hash{}, hash.Hash{}, nil) })
+}
+
+func TestFileManifestUpdateWinRace(t *testing.T) {
+	assert := assert.New(t)
+	fm := makeFileManifestTempDir(t)
+	defer os.RemoveAll(fm.dir)
+
+	newRoot2 := hash.FromData([]byte("new root 2"))
+	actual, tableSpecs := fm.Update(nil, hash.Hash{}, newRoot2, func() {
+		// This should fail to get the lock, and therefore _not_ clobber the manifest. So the Update should succeed.
+		newRoot := hash.FromData([]byte("new root"))
+		b, err := tryClobberManifest(fm.dir, strings.Join([]string{StorageVersion, constants.NomsVersion, newRoot.String()}, ":"))
+		assert.NoError(err, string(b))
+	})
+	assert.Equal(newRoot2, actual)
+	assert.Nil(tableSpecs)
+}
+
+func TestFileManifestUpdateRootOptimisticLockFail(t *testing.T) {
+	assert := assert.New(t)
+	fm := makeFileManifestTempDir(t)
+	defer os.RemoveAll(fm.dir)
+
+	tableName := hash.FromData([]byte("table1"))
+	newRoot := hash.FromData([]byte("new root"))
+	b, err := tryClobberManifest(fm.dir, strings.Join([]string{StorageVersion, constants.NomsVersion, newRoot.String(), tableName.String(), "3"}, ":"))
+	assert.NoError(err, string(b))
+
+	newRoot2 := hash.FromData([]byte("new root 2"))
+	actual, tableSpecs := fm.Update(nil, hash.Hash{}, newRoot2, nil)
+	assert.Equal(newRoot, actual)
+	if assert.Len(tableSpecs, 1) {
+		assert.Equal(tableName.String(), tableSpecs[0].name.String())
+		assert.Equal(uint64(3), tableSpecs[0].chunkCount)
+	}
+	actual, tableSpecs = fm.Update(nil, actual, newRoot2, nil)
+}
+
+// tryClobberManifest simulates another process trying to access dir/manifestFileName concurrently. To avoid deadlock, it does a non-blocking lock of dir/lockFileName. If it can get the lock, it clobbers the manifest.
+func tryClobberManifest(dir, contents string) ([]byte, error) {
+	return runClobber(dir, contents, false)
+}
+
+// clobberManifest simulates another process writing dir/manifestFileName concurrently. It takes the lock, so it's up to the caller to avoid deadlock.
+func clobberManifest(dir, contents string) ([]byte, error) {
+	return runClobber(dir, contents, true)
+}
+
+func runClobber(dir, contents string, takeLock bool) ([]byte, error) {
+	_, filename, _, _ := runtime.Caller(1)
+	clobber := filepath.Join(filepath.Dir(filename), "test/manifest_clobber.go")
+	mkPath := func(f string) string {
+		return filepath.Join(dir, f)
+	}
+	args := []string{"run", clobber}
+	if takeLock {
+		args = append(args, "--take-lock")
+	}
+	args = append(args, mkPath(lockFileName), mkPath(manifestFileName), contents)
+
+	c := exec.Command("go", args...)
+	return c.CombinedOutput()
+}

go/nbs/mem_table.go

@@ -0,0 +1,94 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import "sort"
+
+type memTable struct {
+	chunks             map[addr][]byte
+	order              []hasRecord
+	maxData, totalData uint64
+}
+
+func newMemTable(memTableSize uint64) *memTable {
+	return &memTable{chunks: map[addr][]byte{}, maxData: memTableSize}
+}
+
+func (mt *memTable) addChunk(h addr, data []byte) bool {
+	if len(data) == 0 {
+		panic("NBS blocks cannont be zero length")
+	}
+	if _, ok := mt.chunks[h]; ok {
+		return true
+	}
+	dataLen := uint64(len(data))
+	if mt.totalData+dataLen > mt.maxData {
+		return false
+	}
+	mt.totalData += dataLen
+	mt.chunks[h] = data
+	mt.order = append(mt.order, hasRecord{
+		&h,
+		h.Prefix(),
+		len(mt.order),
+		false,
+	})
+	return true
+}
+
+func (mt *memTable) count() uint64 {
+	return uint64(len(mt.order))
+}
+
+func (mt *memTable) has(h addr) (has bool) {
+	_, has = mt.chunks[h]
+	return
+}
+
+func (mt *memTable) hasMany(addrs []hasRecord) (remaining bool) {
+	for i, addr := range addrs {
+		if addr.has {
+			continue
+		}
+
+		if mt.has(*addr.a) {
+			addrs[i].has = true
+		} else {
+			remaining = true
+		}
+	}
+	return
+}
+
+func (mt *memTable) get(h addr) []byte {
+	return mt.chunks[h]
+}
+
+func (mt *memTable) getMany(reqs []getRecord) (remaining bool) {
+	for i, r := range reqs {
+		data := mt.chunks[*r.a]
+		if data != nil {
+			reqs[i].data = data
+		} else {
+			remaining = true
+		}
+	}
+	return
+}
+
+func (mt *memTable) write(tw chunkWriter, haver chunkReader) {
+	if haver != nil {
+		sort.Sort(hasRecordByPrefix(mt.order)) // hasMany() requires addresses to be sorted.
+		haver.hasMany(mt.order)
+		sort.Sort(hasRecordByOrder(mt.order)) // restore "insertion" order for write
+	}
+
+	for _, addr := range mt.order {
+		if !addr.has {
+			h := addr.a
+			tw.addChunk(*h, mt.chunks[*h])
+		}
+	}
+}

go/nbs/mem_table_test.go

@@ -0,0 +1,152 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/attic-labs/testify/assert"
+)
+
+type memReaderAt []byte
+
+func (mr memReaderAt) ReadAt(buff []byte, offset int64) (n int, err error) {
+	copy(buff, mr[offset:])
+	n = len(buff)
+	if int64(len(mr))-offset < int64(n) {
+		n = int(int64(len(mr)) - offset)
+	}
+	return
+}
+
+func TestMemTableAddHasGetChunk(t *testing.T) {
+	assert := assert.New(t)
+	mt := newMemTable(1024)
+
+	chunks := [][]byte{
+		[]byte("hello2"),
+		[]byte("goodbye2"),
+		[]byte("badbye2"),
+	}
+
+	for _, c := range chunks {
+		assert.True(mt.addChunk(computeAddr(c), c))
+	}
+
+	assertChunksInReader(chunks, mt, assert)
+
+	for _, c := range chunks {
+		assert.Equal(bytes.Compare(c, mt.get(computeAddr(c))), 0)
+	}
+
+	notPresent := []byte("nope")
+	assert.False(mt.has(computeAddr(notPresent)))
+	assert.Nil(mt.get(computeAddr(notPresent)))
+}
+
+func TestMemTableAddOverflowChunk(t *testing.T) {
+	memTableSize := uint64(1024)
+
+	assert := assert.New(t)
+	big := make([]byte, memTableSize)
+	little := []byte{0x01}
+	{
+		bigAddr := computeAddr(big)
+		mt := newMemTable(memTableSize)
+		assert.True(mt.addChunk(bigAddr, big))
+		assert.True(mt.has(bigAddr))
+		assert.False(mt.addChunk(computeAddr(little), little))
+		assert.False(mt.has(computeAddr(little)))
+	}
+
+	{
+		big := big[:memTableSize-1]
+		bigAddr := computeAddr(big)
+		mt := newMemTable(memTableSize)
+		assert.True(mt.addChunk(bigAddr, big))
+		assert.True(mt.has(bigAddr))
+		assert.True(mt.addChunk(computeAddr(little), little))
+		assert.True(mt.has(computeAddr(little)))
+		other := []byte("o")
+		assert.False(mt.addChunk(computeAddr(other), other))
+		assert.False(mt.has(computeAddr(other)))
+	}
+}
+
+func TestMemTableWrite(t *testing.T) {
+	assert := assert.New(t)
+	mt := newMemTable(1024)
+
+	chunks := [][]byte{
+		[]byte("hello2"),
+		[]byte("goodbye2"),
+		[]byte("badbye2"),
+	}
+
+	for _, c := range chunks {
+		assert.True(mt.addChunk(computeAddr(c), c))
+	}
+
+	td1, _ := buildTable(chunks[1:2])
+	td2, _ := buildTable(chunks[2:])
+	tr1, tr2 := newTableReader(td1, memReaderAt(td1)), newTableReader(td2, memReaderAt(td2))
+	assert.True(tr1.has(computeAddr(chunks[1])))
+	assert.True(tr2.has(computeAddr(chunks[2])))
+
+	writeSize := maxTableSize(1, uint64(len(chunks[0])))
+
+	buff := make([]byte, writeSize)
+	tw := newTableWriter(buff)
+
+	mt.write(tw, chunkReaderGroup{tr1, tr2})
+
+	ll, _ := tw.finish()
+
+	outReader := newTableReader(buff[:ll], memReaderAt(buff[:ll]))
+	assert.True(outReader.has(computeAddr(chunks[0])))
+	assert.False(outReader.has(computeAddr(chunks[1])))
+	assert.False(outReader.has(computeAddr(chunks[2])))
+}
+
+type chunkReaderGroup []chunkReader
+
+func (crg chunkReaderGroup) has(h addr) bool {
+	for _, haver := range crg {
+		if haver.has(h) {
+			return true
+		}
+	}
+	return false
+}
+
+func (crg chunkReaderGroup) get(h addr) []byte {
+	for _, haver := range crg {
+		if data := haver.get(h); data != nil {
+			return data
+		}
+	}
+	return nil
+}
+
+func (crg chunkReaderGroup) hasMany(addrs []hasRecord) (remaining bool) {
+	for _, haver := range crg {
+		if !haver.hasMany(addrs) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (crg chunkReaderGroup) getMany(reqs []getRecord) (remaining bool) {
+	for _, haver := range crg {
+		if !haver.getMany(reqs) {
+			return false
+		}
+	}
+
+	return true
+}

go/nbs/mmap_table_reader.go

@@ -0,0 +1,59 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/attic-labs/noms/go/d"
+)
+
+type mmapTableReader struct {
+	tableReader
+	f    *os.File
+	buff []byte
+	h    addr
+}
+
+var pageSize = int64(os.Getpagesize())
+
+func newMmapTableReader(dir string, h addr, chunkCount uint64) chunkSource {
+	success := false
+	f, err := os.Open(filepath.Join(dir, h.String()))
+	d.PanicIfError(err)
+	defer func() {
+		if !success {
+			d.PanicIfError(f.Close())
+		}
+	}()
+
+	fi, err := f.Stat()
+	d.PanicIfError(err)
+	d.PanicIfTrue(fi.Size() < 0)
+
+	// index. Mmap won't take an offset that's not page-aligned, so find the nearest page boundary preceding the index.
+	indexOffset := fi.Size() - int64(footerSize) - int64(indexSize(chunkCount))
+	aligned := indexOffset / pageSize * pageSize // Thanks, integer arithmetic!
+	d.PanicIfTrue(fi.Size()-aligned > maxInt)
+	buff, err := unix.Mmap(int(f.Fd()), aligned, int(fi.Size()-aligned), unix.PROT_READ, unix.MAP_SHARED)
+	d.PanicIfError(err)
+	success = true
+
+	source := &mmapTableReader{newTableReader(buff[indexOffset-aligned:], f), f, buff, h}
+	d.PanicIfFalse(chunkCount == source.count())
+	return source
+}
+
+func (mmtr *mmapTableReader) close() error {
+	mmtr.f.Close()
+	return unix.Munmap(mmtr.buff)
+}
+
+func (mmtr *mmapTableReader) hash() addr {
+	return mmtr.h
+}

go/nbs/mmap_table_reader_test.go

@@ -0,0 +1,35 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/attic-labs/testify/assert"
+)
+
+func TestMmapTableReader(t *testing.T) {
+	assert := assert.New(t)
+	dir, err := ioutil.TempDir("", "")
+	assert.NoError(err)
+	defer os.RemoveAll(dir)
+
+	chunks := [][]byte{
+		[]byte("hello2"),
+		[]byte("goodbye2"),
+		[]byte("badbye2"),
+	}
+
+	tableData, h := buildTable(chunks)
+	err = ioutil.WriteFile(filepath.Join(dir, h.String()), tableData, 0666)
+	assert.NoError(err)
+
+	trc := newMmapTableReader(dir, h, uint64(len(chunks)))
+	defer trc.close()
+	assertChunksInReader(chunks, trc, assert)
+}

go/nbs/root_tracker_test.go

@@ -0,0 +1,139 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/attic-labs/noms/go/chunks"
+	"github.com/attic-labs/noms/go/constants"
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/attic-labs/testify/assert"
+)
+
+func makeStoreInTempDir(t *testing.T) (dir string, store *NomsBlockStore) {
+	dir, err := ioutil.TempDir("", "")
+	assert.NoError(t, err)
+	store = NewBlockStore(dir, defaultMemTableSize)
+	return
+}
+
+func TestChunkStoreZeroValue(t *testing.T) {
+	assert := assert.New(t)
+	dir, store := makeStoreInTempDir(t)
+	defer os.RemoveAll(dir)
+	defer store.Close()
+
+	// No manifest file gets written until the first call to UpdateRoot(). Prior to that, Root() will simply return hash.Hash{}.
+	assert.Equal(hash.Hash{}, store.Root())
+	assert.Equal(constants.NomsVersion, store.Version())
+}
+
+func TestChunkStoreVersion(t *testing.T) {
+	assert := assert.New(t)
+	dir, store := makeStoreInTempDir(t)
+	defer os.RemoveAll(dir)
+	defer store.Close()
+
+	assert.Equal(constants.NomsVersion, store.Version())
+	newRoot := hash.FromData([]byte("new root"))
+	if assert.True(store.UpdateRoot(newRoot, hash.Hash{})) {
+		assert.Equal(constants.NomsVersion, store.Version())
+	}
+}
+
+func TestChunkStoreUpdateRoot(t *testing.T) {
+	assert := assert.New(t)
+	dir, store := makeStoreInTempDir(t)
+	defer os.RemoveAll(dir)
+	defer store.Close()
+
+	assert.Equal(hash.Hash{}, store.Root())
+
+	newRootChunk := chunks.NewChunk([]byte("new root"))
+	newRoot := newRootChunk.Hash()
+	store.Put(newRootChunk)
+	if assert.True(store.UpdateRoot(newRoot, hash.Hash{})) {
+		assert.True(store.Has(newRoot))
+		assert.Equal(newRoot, store.Root())
+	}
+
+	secondRootChunk := chunks.NewChunk([]byte("newer root"))
+	secondRoot := secondRootChunk.Hash()
+	store.Put(secondRootChunk)
+	if assert.True(store.UpdateRoot(secondRoot, newRoot)) {
+		assert.Equal(secondRoot, store.Root())
+		assert.True(store.Has(newRoot))
+		assert.True(store.Has(secondRoot))
+	}
+}
+
+func TestChunkStoreManifestAppearsAfterConstruction(t *testing.T) {
+	assert := assert.New(t)
+	dir, store := makeStoreInTempDir(t)
+	defer os.RemoveAll(dir)
+	defer store.Close()
+
+	assert.Equal(hash.Hash{}, store.Root())
+	assert.Equal(constants.NomsVersion, store.Version())
+
+	// Simulate another process writing a manifest (with an old Noms version) after construction.
+	chunks := [][]byte{[]byte("hello2"), []byte("goodbye2"), []byte("badbye2")}
+	newRoot := hash.FromData([]byte("new root"))
+	h := createOnDiskTable(dir, chunks)
+	b, err := clobberManifest(dir, strings.Join([]string{StorageVersion, "0", newRoot.String(), h.String(), "3"}, ":"))
+	assert.NoError(err, string(b))
+
+	// Creating another store should reflect the manifest written above.
+	store2 := NewBlockStore(dir, defaultMemTableSize)
+	defer store2.Close()
+
+	assert.Equal(newRoot, store2.Root())
+	assert.Equal("0", store2.Version())
+	assertDataInStore(chunks, store2, assert)
+}
+
+func createOnDiskTable(dir string, chunks [][]byte) addr {
+	tableData, h := buildTable(chunks)
+	d.PanicIfError(ioutil.WriteFile(filepath.Join(dir, h.String()), tableData, 0666))
+	return h
+}
+
+func assertDataInStore(slices [][]byte, store chunks.ChunkSource, assert *assert.Assertions) {
+	for _, data := range slices {
+		assert.True(store.Has(chunks.NewChunk(data).Hash()))
+	}
+}
+
+func TestChunkStoreManifestFirstWriteByOtherProcess(t *testing.T) {
+	assert := assert.New(t)
+	dir, err := ioutil.TempDir(os.TempDir(), "")
+	assert.NoError(err)
+	defer os.RemoveAll(dir)
+
+	// Simulate another process having already written a manifest (with an old Noms version).
+	chunks := [][]byte{[]byte("hello2"), []byte("goodbye2"), []byte("badbye2")}
+	h := createOnDiskTable(dir, chunks)
+	newRoot := hash.FromData([]byte("new root"))
+	b, err := tryClobberManifest(dir, strings.Join([]string{StorageVersion, "0", newRoot.String(), h.String(), "3"}, ":"))
+	assert.NoError(err, string(b))
+
+	store := hookedNewNomsBlockStore(dir, defaultMemTableSize, func() {
+		// This should fail to get the lock, and therefore _not_ clobber the manifest.
+		badRoot := hash.FromData([]byte("bad root"))
+		b, err := tryClobberManifest(dir, strings.Join([]string{StorageVersion, "0", badRoot.String(), h.String(), "3"}, ":"))
+		assert.NoError(err, string(b))
+	})
+	defer store.Close()
+
+	assert.Equal(newRoot, store.Root())
+	assert.Equal("0", store.Version())
+	assertDataInStore(chunks, store, assert)
+}

go/nbs/store.go

@@ -0,0 +1,293 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"sort"
+	"sync"
+
+	"github.com/attic-labs/noms/go/chunks"
+	"github.com/attic-labs/noms/go/constants"
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/attic-labs/noms/go/types"
+)
+
+// The root of a Noms Chunk Store is stored in a 'manifest', along with the
+// names of the tables that hold all the chunks in the store. The number of
+// chunks in each table is also stored in the manifest.
+
+const (
+	// StorageVersion is the version of the on-disk Noms Chunks Store data format.
+	StorageVersion = "0"
+
+	defaultMemTableSize uint64 = 512 * 1 << 20 // 512MB
+)
+
+type NomsBlockStore struct {
+	mm          fileManifest
+	tm          tableManager
+	nomsVersion string
+
+	mu        sync.RWMutex // protects the following state
+	mt        *memTable
+	mtSize    uint64
+	immTables chunkSources // slice is never mutated. on change, new slice is constructed and assigned
+	root      hash.Hash
+
+	putCount uint64
+}
+
+type chunkSources []chunkSource
+
+func (css chunkSources) has(h addr) bool {
+	for _, haver := range css {
+		if haver.has(h) {
+			return true
+		}
+	}
+	return false
+}
+
+func (css chunkSources) hasMany(addrs []hasRecord) (remaining bool) {
+	for _, haver := range css {
+		if !haver.hasMany(addrs) {
+			return false
+		}
+	}
+	return true
+}
+
+func (css chunkSources) get(h addr) []byte {
+	for _, haver := range css {
+		if data := haver.get(h); data != nil {
+			return data
+		}
+	}
+	return nil
+}
+
+func (css chunkSources) getMany(reqs []getRecord) (remaining bool) {
+	for _, haver := range css {
+		if !haver.getMany(reqs) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func NewBlockStore(dir string, memTableSize uint64) *NomsBlockStore {
+	return hookedNewNomsBlockStore(dir, memTableSize, nil)
+}
+
+func hookedNewNomsBlockStore(dir string, memTableSize uint64, readHook func()) *NomsBlockStore {
+	if memTableSize == 0 {
+		memTableSize = defaultMemTableSize
+	}
+	nbs := &NomsBlockStore{
+		mm:          fileManifest{dir},
+		tm:          &fileTableManager{dir},
+		nomsVersion: constants.NomsVersion,
+		mtSize:      memTableSize,
+	}
+
+	if exists, vers, root, tableSpecs := nbs.mm.ParseIfExists(readHook); exists {
+		nbs.nomsVersion, nbs.root = vers, root
+		nbs.immTables = unionTables(nil, nbs.tm, tableSpecs)
+	}
+
+	return nbs
+}
+
+// unionTables returns a new chunkSources that contains all of curTables as well as a chunkSource for each as-yet-unknown table named by tableSpecs.
+func unionTables(curTables chunkSources, tm tableManager, tableSpecs []tableSpec) chunkSources {
+	newTables := make(chunkSources, len(curTables))
+	known := map[addr]struct{}{}
+	for i, t := range curTables {
+		known[t.hash()] = struct{}{}
+		newTables[i] = curTables[i]
+	}
+
+	for _, t := range tableSpecs {
+		if _, present := known[t.name]; !present {
+			newTables = append(newTables, tm.open(t.name, t.chunkCount))
+		}
+	}
+	return newTables
+}
+
+func (nbs *NomsBlockStore) Put(c chunks.Chunk) {
+	a := addr(c.Hash().Digest())
+	d.PanicIfFalse(nbs.addChunk(a, c.Data()))
+	nbs.putCount++
+}
+
+func (nbs *NomsBlockStore) SchedulePut(c chunks.Chunk, refHeight uint64, hints types.Hints) {
+	nbs.Put(c)
+}
+
+func (nbs *NomsBlockStore) PutMany(chunx []chunks.Chunk) (err chunks.BackpressureError) {
+	for ; len(chunx) > 0; chunx = chunx[1:] {
+		c := chunx[0]
+		a := addr(c.Hash().Digest())
+		if !nbs.addChunk(a, c.Data()) {
+			break
+		}
+		nbs.putCount++
+	}
+	for _, c := range chunx {
+		err = append(err, c.Hash())
+	}
+	return err
+}
+
+func (nbs *NomsBlockStore) addChunk(h addr, data []byte) bool {
+	nbs.mu.Lock()
+	defer nbs.mu.Unlock()
+	if nbs.mt == nil {
+		nbs.mt = newMemTable(nbs.mtSize)
+	}
+	if !nbs.mt.addChunk(h, data) {
+		if tableHash, wrote := nbs.tm.compact(nbs.mt, nbs.immTables); wrote {
+			nbs.immTables = prependTable(nbs.immTables, nbs.tm.open(tableHash, nbs.mt.count()))
+		}
+		nbs.mt = newMemTable(nbs.mtSize)
+		return nbs.mt.addChunk(h, data)
+	}
+	return true
+}
+
+func prependTable(curTables chunkSources, crc chunkSource) chunkSources {
+	newTables := make(chunkSources, len(curTables)+1)
+	newTables[0] = crc
+	copy(newTables[1:], curTables)
+	return newTables
+}
+
+func (nbs *NomsBlockStore) Get(h hash.Hash) chunks.Chunk {
+	a := addr(h.Digest())
+	data, tables := func() (data []byte, tables chunkSources) {
+		nbs.mu.RLock()
+		defer nbs.mu.RUnlock()
+		if nbs.mt != nil {
+			data = nbs.mt.get(a)
+		}
+		return data, nbs.immTables
+	}()
+	if data != nil {
+		return chunks.NewChunkWithHash(h, data)
+	}
+	if data := tables.get(a); data != nil {
+		return chunks.NewChunkWithHash(h, data)
+	}
+	return chunks.EmptyChunk
+}
+
+func (nbs *NomsBlockStore) GetMany(hashes []hash.Hash) []chunks.Chunk {
+	reqs := make([]getRecord, len(hashes))
+	for i, h := range hashes {
+		a := addr(h.Digest())
+		reqs[i] = getRecord{
+			a:      &a,
+			prefix: a.Prefix(),
+			order:  i,
+		}
+	}
+
+	tables, remaining := func() (tables chunkSources, remaining bool) {
+		nbs.mu.RLock()
+		defer nbs.mu.RUnlock()
+		tables = nbs.immTables
+
+		if nbs.mt != nil {
+			remaining = nbs.mt.getMany(reqs)
+		} else {
+			remaining = true
+		}
+
+		return
+	}()
+
+	sort.Sort(getRecordByPrefix(reqs))
+
+	if remaining {
+		tables.getMany(reqs)
+	}
+
+	sort.Sort(getRecordByOrder(reqs))
+
+	resp := make([]chunks.Chunk, len(hashes))
+	for i, req := range reqs {
+		if req.data == nil {
+			resp[i] = chunks.EmptyChunk
+		} else {
+			resp[i] = chunks.NewChunkWithHash(hashes[i], req.data)
+		}
+	}
+
+	return resp
+}
+
+func (nbs *NomsBlockStore) Has(h hash.Hash) bool {
+	a := addr(h.Digest())
+	has, tables := func() (bool, chunkSources) {
+		nbs.mu.RLock()
+		defer nbs.mu.RUnlock()
+		return nbs.mt != nil && nbs.mt.has(a), nbs.immTables
+	}()
+	return has || tables.has(a)
+}
+
+func (nbs *NomsBlockStore) Root() hash.Hash {
+	nbs.mu.RLock()
+	defer nbs.mu.RUnlock()
+	return nbs.root
+}
+
+func (nbs *NomsBlockStore) UpdateRoot(current, last hash.Hash) bool {
+	nbs.mu.Lock()
+	defer nbs.mu.Unlock()
+	d.Chk.True(nbs.root == last, "UpdateRoot: last != nbs.Root(); %s != %s", last, nbs.root)
+
+	if nbs.mt != nil && nbs.mt.count() > 0 {
+		if tableHash, wrote := nbs.tm.compact(nbs.mt, nbs.immTables); wrote {
+			nbs.immTables = prependTable(nbs.immTables, nbs.tm.open(tableHash, nbs.mt.count()))
+		}
+		nbs.mt = nil
+	}
+
+	actual, tableNames := nbs.mm.Update(nbs.immTables, nbs.root, current, nil)
+
+	if current != actual {
+		nbs.root = actual
+		nbs.immTables = unionTables(nbs.immTables, nbs.tm, tableNames)
+		return false
+	}
+	nbs.nomsVersion, nbs.root = constants.NomsVersion, current
+	return true
+}
+
+func (nbs *NomsBlockStore) Version() string {
+	return nbs.nomsVersion
+}
+
+func (nbs *NomsBlockStore) Close() (err error) {
+	nbs.mu.Lock()
+	defer nbs.mu.Unlock()
+	for _, t := range nbs.immTables {
+		err = t.close() // TODO: somehow coalesce these errors??
+	}
+	return
+}
+
+// types.BatchStore
+func (nbs *NomsBlockStore) AddHints(hints types.Hints) {
+	// noop
+}
+
+func (nbs *NomsBlockStore) Flush() {
+	// noop
+}

go/nbs/table.go

@@ -0,0 +1,210 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"bytes"
+	"crypto/sha512"
+	"encoding/base32"
+	"encoding/binary"
+)
+
+/*
+   An NBS Table stores N byte slices ("chunks") which are addressed by a 20-byte hash of their
+   contents. The footer encodes N as well as the total bytes consumed by all contained chunks.
+   An Index maps each address to the position of its corresponding chunk. Addresses are logically sorted within the Index, but the corresponding chunks need not be.
+
+   Table:
+   +----------------+----------------+-----+----------------+-------+--------+
+   | Chunk Record 0 | Chunk Record 1 | ... | Chunk Record N | Index | Footer |
+   +----------------+----------------+-----+----------------+-------+--------+
+
+   Chunk Record:
+   +--------------------+---------------------------+
+   | (4) Address suffix | (Chunk Length) Chunk Data |
+   +--------------------+---------------------------+
+
+     -Address suffix is the 4 least-significant bytes of the Chunk's address. Used (e.g. in place
+      of CRC32) as a checksum and a filter against false positive reads costing more than one IOP.
+
+   Index:
+   +------------+-------+----------+
+   | Prefix Map | Sizes | Suffixes |
+   +------------+-------+----------+
+
+   Prefix Map:
+   +--------------+--------------+-----+----------------+
+   | Prefix Tuple | Prefix Tuple | ... | Prefix Tuple N |
+   +--------------+--------------+-----+----------------+
+
+     -The Prefix Map contains N Prefix Tuples.
+     -Each Prefix Tuple corresponds to a unique Chunk Record in the Table.
+     -The Prefix Tuples are sorted in increasing lexicographic order within the Prefix Map.
+     -NB: THE SAME PREFIX MAY APPEAR MULTIPLE TIMES, as distinct Hashes (referring to distinct Chunks) may share the same Prefix.
+
+   Prefix Tuple:
+   +-----------------+------------------+
+   | (8) Hash Prefix | (Uint32) Ordinal |
+   +-----------------+------------------+
+
+     -First 8 bytes of a Chunk's Hash
+     -Ordinal is the 0-based ordinal position of the associated record within the sequence of chunk records, the associated Length within Lengths, and the associated Hash Suffix within Suffixes.
+
+   Lengths:
+   +-----------------+-----------------+-----+-------------------+
+   | (Uint32) Length | (Uint32) Length | ... | (Uint32) Length N |
+   +-----------------+-----------------+-----+-------------------+
+
+     - Each Length is the length of a Chunk Record in this Table.
+     - Length M must correspond to Chunk Record M for 0 <= M <= N
+
+   Suffixes:
+   +------------------+------------------+-----+--------------------+
+   | (12) Hash Suffix | (12) Hash Suffix | ... | (12) Hash Suffix N |
+   +------------------+------------------+-----+--------------------+
+
+     - Each Hash Suffix is the last 12 bytes of a Chunk in this Table.
+     - Hash Suffix M must correspond to Chunk Record M for 0 <= M <= N
+
+   Footer:
+   +----------------------+---------------------------+------------------+
+   | (Uint64) Chunk Count | (Uint64) Total Chunk Data | (8) Magic Number |
+   +----------------------+---------------------------+------------------+
+
+     -Total Chunk Data is the sum of the logical byte lengths of all contained chunk byte slices.
+     -Magic Number is the first 8 bytes of the SHA256 hash of "https://github.com/attic-labs/nbs".
+
+    NOTE: Unsigned integer quanities, hashes and hash suffix are all encoded big-endian
+
+
+  Looking up Chunks in an NBS Table
+  There are two phases to loading chunk data for a given Hash from an NBS Table: Checking for the chunk's presence, and fetching the chunk's bytes. When performing a has-check, only the first phase is necessary.
+
+  Phase one: Chunk presence
+  - Slice off the first 8 bytes of your Hash to create a Prefix
+  - Since the Prefix Tuples in the Prefix Map are in lexicographic order, binary search the Prefix Map for the desired Prefix.
+  - For all Prefix Tuples with a matching Prefix:
+    - Load the Ordinal
+    - Use the Ordinal to index into Suffixes
+    - Check the Suffix of your Hash against the loaded Suffix
+    - If they match, your chunk is in this Table in the Chunk Record indicated by Ordinal
+    - If they don't match, continue to the next matching Prefix Tuple
+  - If not found, your chunk is not in this Table.
+
+  Phase two: Loading Chunk data
+  - Take the Ordinal discovered in Phase one
+  - Calculate the Offset of your desired Chunk Record: Sum(Lengths[0]...Lengths[Ordinal-1])
+  - Load Lengths[Ordinal] bytes from Table[Offset]
+  - Check the first 4 bytes of the loaded data against the last 4 bytes of your desired Hash. They should match, and the rest of the data is your Chunk data.
+*/
+
+const (
+	addrSize           = uint64(20)
+	addrPrefixSize     = uint64(8)
+	addrSuffixSize     = addrSize - addrPrefixSize
+	ordinalSize        = uint64(4)
+	lengthSize         = uint64(4)
+	uint64Size         = uint64(8)
+	magicNumber        = "\xff\xb5\xd8\xc2\x24\x63\xee\x50"
+	magicNumberSize    = uint64(len(magicNumber))
+	footerSize         = uint64Size + uint64Size + magicNumberSize
+	prefixTupleSize    = addrPrefixSize + ordinalSize
+	checksumSize       = uint64(4)
+	maxChunkLengthSize = uint64(binary.MaxVarintLen64)
+	maxChunkSize       = uint64(0xffffffff) // Snappy won't compress slices bigger than this
+)
+
+func computeAddrDefault(data []byte) addr {
+	r := sha512.Sum512(data)
+	h := addr{}
+	copy(h[:], r[:addrSize])
+	return h
+}
+
+var computeAddr = computeAddrDefault
+
+type addr [addrSize]byte
+
+var encoding = base32.NewEncoding("0123456789abcdefghijklmnopqrstuv")
+
+func (a addr) String() string {
+	return encoding.EncodeToString(a[:])
+}
+
+func (a addr) Prefix() uint64 {
+	return binary.BigEndian.Uint64(a[:])
+}
+
+func (a addr) Checksum() uint32 {
+	return binary.BigEndian.Uint32(a[addrSize-checksumSize:])
+}
+
+func ParseAddr(b []byte) (h addr) {
+	encoding.Decode(h[:], b)
+	return
+}
+
+type addrSlice []addr
+
+func (hs addrSlice) Len() int           { return len(hs) }
+func (hs addrSlice) Less(i, j int) bool { return bytes.Compare(hs[i][:], hs[j][:]) < 0 }
+func (hs addrSlice) Swap(i, j int)      { hs[i], hs[j] = hs[j], hs[i] }
+
+type hasRecord struct {
+	a      *addr
+	prefix uint64
+	order  int
+	has    bool
+}
+
+type hasRecordByPrefix []hasRecord
+
+func (hs hasRecordByPrefix) Len() int           { return len(hs) }
+func (hs hasRecordByPrefix) Less(i, j int) bool { return hs[i].prefix < hs[j].prefix }
+func (hs hasRecordByPrefix) Swap(i, j int)      { hs[i], hs[j] = hs[j], hs[i] }
+
+type hasRecordByOrder []hasRecord
+
+func (hs hasRecordByOrder) Len() int           { return len(hs) }
+func (hs hasRecordByOrder) Less(i, j int) bool { return hs[i].order < hs[j].order }
+func (hs hasRecordByOrder) Swap(i, j int)      { hs[i], hs[j] = hs[j], hs[i] }
+
+type getRecord struct {
+	a      *addr
+	prefix uint64
+	order  int
+	data   []byte
+}
+
+type getRecordByPrefix []getRecord
+
+func (hs getRecordByPrefix) Len() int           { return len(hs) }
+func (hs getRecordByPrefix) Less(i, j int) bool { return hs[i].prefix < hs[j].prefix }
+func (hs getRecordByPrefix) Swap(i, j int)      { hs[i], hs[j] = hs[j], hs[i] }
+
+type getRecordByOrder []getRecord
+
+func (hs getRecordByOrder) Len() int           { return len(hs) }
+func (hs getRecordByOrder) Less(i, j int) bool { return hs[i].order < hs[j].order }
+func (hs getRecordByOrder) Swap(i, j int)      { hs[i], hs[j] = hs[j], hs[i] }
+
+type chunkReader interface {
+	has(h addr) bool
+	hasMany(addrs []hasRecord) bool
+	get(h addr) []byte
+	getMany(reqs []getRecord) bool
+}
+
+type chunkSource interface {
+	chunkReader
+	close() error
+	count() uint64
+	hash() addr
+}
+
+type chunkWriter interface {
+	// addChunk writes chunk data with the address h. Returns true if the data was written, false if not.
+	addChunk(h addr, data []byte) bool
+}

go/nbs/table_manager.go

@@ -0,0 +1,22 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+type tableManager interface {
+	compact(mt *memTable, haver chunkReader) (name addr, wrote bool)
+	open(name addr, chunkCount uint64) chunkSource
+}
+
+type fileTableManager struct {
+	dir string
+}
+
+func (ftm *fileTableManager) compact(mt *memTable, haver chunkReader) (name addr, wrote bool) {
+	return compact(ftm.dir, mt, haver)
+}
+
+func (ftm *fileTableManager) open(name addr, chunkCount uint64) chunkSource {
+	return newMmapTableReader(ftm.dir, name, chunkCount)
+}

go/nbs/table_reader.go

@@ -0,0 +1,348 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"sort"
+
+	"github.com/attic-labs/noms/go/d"
+	"github.com/golang/snappy"
+)
+
+// tableReader implements get & has queries against a single nbs table. goroutine safe.
+type tableReader struct {
+	r                 io.ReaderAt
+	suffixes          []byte
+	prefixes, offsets []uint64
+	lengths, ordinals []uint32
+	chunkCount        uint64
+}
+
+// newTableReader parses a valid nbs table byte stream and returns a reader. buff must end with an NBS index and footer, though it may contain an unspecified number of bytes before that data. r should allow retrieving any desired range of bytes from the table.
+func newTableReader(buff []byte, r io.ReaderAt) tableReader {
+	tr := tableReader{r: r}
+
+	pos := uint64(len(buff))
+
+	// footer
+	pos -= magicNumberSize
+	d.Chk.True(string(buff[pos:]) == magicNumber)
+
+	// skip total chunk data
+	pos -= uint64Size
+
+	pos -= uint64Size
+	tr.chunkCount = binary.BigEndian.Uint64(buff[pos : pos+uint64Size])
+
+	// index
+	suffixesSize := tr.chunkCount * addrSuffixSize
+	pos -= suffixesSize
+	tr.suffixes = buff[pos : pos+suffixesSize]
+
+	lengthsSize := tr.chunkCount * lengthSize
+	pos -= lengthsSize
+	tr.lengths, tr.offsets = computeOffsets(tr.chunkCount, buff[pos:pos+lengthsSize])
+
+	tuplesSize := tr.chunkCount * prefixTupleSize
+	pos -= tuplesSize
+	tr.prefixes, tr.ordinals = computePrefixes(tr.chunkCount, buff[pos:pos+tuplesSize])
+	return tr
+}
+
+func computeOffsets(count uint64, buff []byte) (lengths []uint32, offsets []uint64) {
+	lengths = make([]uint32, count)
+	offsets = make([]uint64, count)
+
+	lengths[0] = binary.BigEndian.Uint32(buff)
+
+	for i := uint64(1); i < count; i++ {
+		lengths[i] = binary.BigEndian.Uint32(buff[i*lengthSize:])
+		offsets[i] = offsets[i-1] + uint64(lengths[i-1])
+	}
+	return
+}
+
+func computePrefixes(count uint64, buff []byte) (prefixes []uint64, ordinals []uint32) {
+	prefixes = make([]uint64, count)
+	ordinals = make([]uint32, count)
+
+	for i := uint64(0); i < count; i++ {
+		idx := i * prefixTupleSize
+		prefixes[i] = binary.BigEndian.Uint64(buff[idx:])
+		ordinals[i] = binary.BigEndian.Uint32(buff[idx+addrPrefixSize:])
+	}
+	return
+}
+
+// Scan across (logically) two ordered slices of address prefixes.
+func (tr tableReader) hasMany(addrs []hasRecord) (remaining bool) {
+	// TODO: Use findInIndex if (tr.chunkCount - len(addrs)*Log2(tr.chunkCount)) > (tr.chunkCount - len(addrs))
+
+	filterIdx := uint64(0)
+	filterLen := uint64(len(tr.prefixes))
+
+	for i, addr := range addrs {
+		if addr.has {
+			continue
+		}
+
+		for filterIdx < filterLen && addr.prefix > tr.prefixes[filterIdx] {
+			filterIdx++
+		}
+
+		if filterIdx >= filterLen {
+			remaining = true
+			return
+		}
+
+		if addr.prefix != tr.prefixes[filterIdx] {
+			remaining = true
+			continue
+		}
+
+		// prefixes are equal, so locate and compare against the corresponding suffix
+		for j := filterIdx; j < filterLen && addr.prefix == tr.prefixes[j]; j++ {
+			li := tr.prefixIdxToOrdinal(j) * addrSuffixSize
+			if bytes.Compare(addr.a[addrPrefixSize:], tr.suffixes[li:li+addrSuffixSize]) == 0 {
+				addrs[i].has = true
+				break
+			}
+		}
+
+		if !addrs[i].has {
+			remaining = true
+		}
+	}
+
+	return
+}
+
+func (tr tableReader) prefixIdxToOrdinal(idx uint64) uint64 {
+	return uint64(tr.ordinals[idx])
+}
+
+// returns the first position in |tr.prefixes| whose value == |prefix|. Returns |tr.chunkCount|
+// if absent
+func (tr tableReader) prefixIdx(prefix uint64) (idx uint64) {
+	// NOTE: The golang impl of sort.Search is basically inlined here. This method can be called in
+	// an extremely tight loop and inlining the code was a significant perf improvement.
+	idx, j := 0, tr.chunkCount
+	for idx < j {
+		h := idx + (j-idx)/2 // avoid overflow when computing h
+		// i ≤ h < j
+		if tr.prefixes[h] < prefix {
+			idx = h + 1 // preserves f(i-1) == false
+		} else {
+			j = h // preserves f(j) == true
+		}
+	}
+
+	return
+}
+
+func (tr tableReader) count() uint64 {
+	return tr.chunkCount
+}
+
+// returns true iff |h| can be found in this table.
+func (tr tableReader) has(h addr) bool {
+	prefix := h.Prefix()
+	idx := tr.prefixIdx(prefix)
+
+	for ; idx < tr.chunkCount && tr.prefixes[idx] == prefix; idx++ {
+		ordinal := tr.prefixIdxToOrdinal(idx)
+		suffixOffset := ordinal * addrSuffixSize
+
+		if bytes.Compare(tr.suffixes[suffixOffset:suffixOffset+addrSuffixSize], h[addrPrefixSize:]) == 0 {
+			return true
+		}
+	}
+
+	return false
+}
+
+// returns the storage associated with |h|, iff present. Returns nil if absent. On success,
+// the returned byte slice directly references the underlying storage.
+func (tr tableReader) get(h addr) (data []byte) {
+	prefix := h.Prefix()
+	idx := tr.prefixIdx(prefix)
+
+	for ; idx < tr.chunkCount && tr.prefixes[idx] == prefix; idx++ {
+		ordinal := tr.prefixIdxToOrdinal(idx)
+		offset := tr.offsets[ordinal]
+		length := uint64(tr.lengths[ordinal])
+		buff := make([]byte, length) // TODO: Avoid this allocation for every get
+		n, err := tr.r.ReadAt(buff, int64(offset))
+		d.Chk.NoError(err)
+		d.Chk.True(n == int(length))
+		data = tr.parseChunk(h, buff)
+		if data != nil {
+			break
+		}
+	}
+
+	return
+}
+
+type offsetRec struct {
+	reqIdx, ordinal uint32
+	offset          uint64
+}
+
+type offsetRecSlice []offsetRec
+
+func (hs offsetRecSlice) Len() int           { return len(hs) }
+func (hs offsetRecSlice) Less(i, j int) bool { return hs[i].offset < hs[j].offset }
+func (hs offsetRecSlice) Swap(i, j int)      { hs[i], hs[j] = hs[j], hs[i] }
+
+const blockSize = 1 << 12
+const readAmpThresh = 1 << 1
+
+// getMany retrieves multiple stored blocks and optimizes by attempting to read in larger physical
+// blocks which contain multiple stored blocks. |reqs| must be sorted by address prefix.
+func (tr tableReader) getMany(reqs []getRecord) (remaining bool) {
+	filterIdx := uint64(0)
+	filterLen := uint64(len(tr.prefixes))
+	offsetRecords := make(offsetRecSlice, 0, len(reqs))
+
+	// Pass #1: Iterate over |reqs| and |tr.prefixes| (both sorted by address) and build the set
+	// of table locations which must be read in order to satisfy the getMany operation.
+	for i, req := range reqs {
+		// advance within the prefixes until we reach one which is >= req.prefix
+		for filterIdx < filterLen && tr.prefixes[filterIdx] < req.prefix {
+			filterIdx++
+		}
+
+		if filterIdx >= filterLen {
+			remaining = true // last prefix visited.
+			break
+		}
+
+		if req.prefix != tr.prefixes[filterIdx] {
+			remaining = true
+			continue
+		}
+
+		// record all offsets within the table which *may* contain the address we are searching for.
+		for j := filterIdx; j < filterLen && req.prefix == tr.prefixes[j]; j++ {
+			offsetRecords = append(offsetRecords, offsetRec{uint32(i), tr.ordinals[j], tr.offsets[tr.ordinals[j]]})
+		}
+	}
+
+	// Now |offsets| contains all locations within the table which must be search (note that there
+	// may be duplicates of a particular location). Sort by offset and scan forward, grouping
+	// sequences of reads into large physical reads.
+	sort.Sort(offsetRecords)
+
+	scratch := []byte{} // raw byte area into which reads occur
+	// slice within |scratch| which contains a contiguous sequence of bytes read from the table
+	buff := scratch[:]
+	baseOffset := uint64(0) // the offset within the table which corresponds to byte 0 of |buff|
+
+	for i, rec := range offsetRecords {
+		if reqs[rec.reqIdx].data != nil {
+			continue // already satisfied
+		}
+
+		// offset within |buff| which corresponds to the logical location of the chunkRecord
+		localOffset := rec.offset - baseOffset
+		length := tr.lengths[rec.ordinal]
+
+		if uint64(len(buff)) < localOffset+uint64(length) {
+			// |buff| doesn't contain sufficient bytes to read the current chunk record. scan forward
+			// and read in a new sequence of bytes
+
+			readStart := rec.offset
+			readEnd := rec.offset + uint64(length) // implicitly include the first chunk
+
+			// As we scan forward, for each location/length, we'll include it in the current read if
+			// the total number of bytes we'll read contains fewer than X bytes we don't care about.
+			readAmp := uint64(0)
+
+			// scan ahead in offsets
+			for j := i + 1; j < len(offsetRecords); j++ {
+				fRec := offsetRecords[j]
+
+				if reqs[fRec.reqIdx].data != nil {
+					continue // already satisfied
+				}
+
+				if fRec.offset < readEnd {
+					// |offsetRecords| will contain an offsetRecord for *every* chunkRecord whose address
+					// prefix matches the prefix of a requested address. If the set of requests contains
+					// addresses which share a common prefix, then it's possible for multiple offsetRecords
+					// to reference the same table offset position. In that case, we'll see sequential
+					// offsetRecords with the same fRec.offset.
+					continue
+				}
+
+				// only consider "wasted" bytes ABOVE block_size to be read amplification.
+				fReadAmp := fRec.offset - readEnd
+				if fReadAmp < blockSize {
+					fReadAmp = 0
+				} else {
+					fReadAmp -= blockSize
+				}
+
+				fLength := tr.lengths[fRec.ordinal]
+				if fRec.offset+uint64(fLength)-readStart < readAmpThresh*(readAmp+fReadAmp) {
+					break // including the next block will read too many unneeded bytes
+				}
+
+				readEnd = fRec.offset + uint64(fLength)
+				readAmp += fReadAmp
+			}
+
+			// Ensure our memory buffer is large enough
+			if readEnd-readStart > uint64(len(scratch)) {
+				scratch = make([]byte, readEnd-readStart)
+			}
+
+			buff = scratch[:readEnd-readStart]
+			n, err := tr.r.ReadAt(buff, int64(readStart))
+			d.Chk.NoError(err)
+			d.Chk.True(uint64(n) == readEnd-readStart)
+
+			baseOffset = readStart
+			localOffset = 0
+		}
+
+		chunkRecord := buff[localOffset : localOffset+uint64(length)]
+		data := tr.parseChunk(*reqs[rec.reqIdx].a, chunkRecord)
+		if data != nil {
+			reqs[rec.reqIdx].data = data
+		} else {
+			remaining = true
+		}
+	}
+
+	return
+}
+
+// Fetches the byte stream of data logically encoded within the table starting at |pos|.
+func (tr tableReader) parseChunk(h addr, buff []byte) []byte {
+	// chksum (4 LSBytes, big-endian)
+	chksum := binary.BigEndian.Uint32(buff)
+	if chksum != h.Checksum() {
+		return nil // false positive
+	}
+	buff = buff[checksumSize:]
+
+	// data
+	data, err := snappy.Decode(nil, buff)
+	d.Chk.NoError(err)
+
+	computedAddr := computeAddr(data)
+	d.Chk.True(chksum == computedAddr.Checksum()) // integrity check
+
+	if computedAddr != h {
+		return nil // false positive
+	}
+
+	return data
+}

go/nbs/table_test.go

@@ -0,0 +1,279 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"encoding/binary"
+	"fmt"
+	"sort"
+	"testing"
+
+	"bytes"
+
+	"github.com/attic-labs/noms/go/d"
+	"github.com/attic-labs/noms/go/hash"
+	"github.com/attic-labs/testify/assert"
+)
+
+func buildTable(chunks [][]byte) ([]byte, addr) {
+	totalData := uint64(0)
+	for _, chunk := range chunks {
+		totalData += uint64(len(chunk))
+	}
+	capacity := maxTableSize(uint64(len(chunks)), totalData)
+
+	buff := make([]byte, capacity)
+
+	tw := newTableWriter(buff)
+
+	for _, chunk := range chunks {
+		tw.addChunk(computeAddr(chunk), chunk)
+	}
+
+	length, blockHash := tw.finish()
+	return buff[:length], blockHash
+}
+
+func TestSimple(t *testing.T) {
+	assert := assert.New(t)
+
+	chunks := [][]byte{
+		[]byte("hello2"),
+		[]byte("goodbye2"),
+		[]byte("badbye2"),
+	}
+
+	tableData, _ := buildTable(chunks)
+	tr := newTableReader(tableData, memReaderAt(tableData))
+
+	assertChunksInReader(chunks, tr, assert)
+
+	assert.Equal(string(chunks[0]), string(tr.get(computeAddr(chunks[0]))))
+	assert.Equal(string(chunks[1]), string(tr.get(computeAddr(chunks[1]))))
+	assert.Equal(string(chunks[2]), string(tr.get(computeAddr(chunks[2]))))
+
+	notPresent := [][]byte{
+		[]byte("yo"),
+		[]byte("do"),
+		[]byte("so much to do"),
+	}
+
+	assertChunksNotInReader(notPresent, tr, assert)
+
+	assert.NotEqual(string(notPresent[0]), string(tr.get(computeAddr(notPresent[0]))))
+	assert.NotEqual(string(notPresent[1]), string(tr.get(computeAddr(notPresent[1]))))
+	assert.NotEqual(string(notPresent[2]), string(tr.get(computeAddr(notPresent[2]))))
+}
+
+func assertChunksInReader(chunks [][]byte, r chunkReader, assert *assert.Assertions) {
+	for _, c := range chunks {
+		assert.True(r.has(computeAddr(c)))
+	}
+}
+
+func assertChunksNotInReader(chunks [][]byte, r chunkReader, assert *assert.Assertions) {
+	for _, c := range chunks {
+		assert.False(r.has(computeAddr(c)))
+	}
+}
+
+func TestHasMany(t *testing.T) {
+	assert := assert.New(t)
+
+	chunks := [][]byte{
+		[]byte("hello2"),
+		[]byte("goodbye2"),
+		[]byte("badbye2"),
+	}
+
+	tableData, _ := buildTable(chunks)
+	tr := newTableReader(tableData, memReaderAt(tableData))
+
+	addrs := addrSlice{computeAddr(chunks[0]), computeAddr(chunks[1]), computeAddr(chunks[2])}
+	hasAddrs := []hasRecord{
+		{&addrs[0], binary.BigEndian.Uint64(addrs[0][:addrPrefixSize]), 0, false},
+		{&addrs[1], binary.BigEndian.Uint64(addrs[1][:addrPrefixSize]), 1, false},
+		{&addrs[2], binary.BigEndian.Uint64(addrs[2][:addrPrefixSize]), 2, false},
+	}
+	sort.Sort(hasRecordByPrefix(hasAddrs))
+
+	tr.hasMany(hasAddrs)
+	for _, ha := range hasAddrs {
+		assert.True(ha.has, "Nothing for prefix %d", ha.prefix)
+	}
+}
+
+func TestHasManySequentialPrefix(t *testing.T) {
+	assert := assert.New(t)
+
+	// Use bogus addrs so we can generate the case of sequentially non-unique prefixes in the index
+	// Note that these are already sorted
+	addrStrings := []string{
+		"0rfgadopg6h3fk7d253ivbjsij4qo3nv",
+		"0rfgadopg6h3fk7d253ivbjsij4qo4nv",
+		"0rfgadopg6h3fk7d253ivbjsij4qo9nv",
+	}
+
+	addrs := make([]addr, len(addrStrings))
+	for i, s := range addrStrings {
+		addrs[i] = addr(hash.Parse(s).Digest())
+	}
+
+	bogusData := []byte("bogus") // doesn't matter what this is. hasMany() won't check chunkRecords
+	totalData := uint64(len(bogusData) * len(addrs))
+
+	capacity := maxTableSize(uint64(len(addrs)), totalData)
+	buff := make([]byte, capacity)
+	tw := newTableWriter(buff)
+
+	for _, a := range addrs {
+		tw.addChunk(a, bogusData)
+	}
+
+	length, _ := tw.finish()
+	buff = buff[:length]
+
+	tr := newTableReader(buff, memReaderAt(buff))
+
+	hasAddrs := make([]hasRecord, 2)
+	// Leave out the first address
+	hasAddrs[0] = hasRecord{&addrs[1], addrs[1].Prefix(), 1, false}
+	hasAddrs[1] = hasRecord{&addrs[2], addrs[2].Prefix(), 2, false}
+
+	tr.hasMany(hasAddrs)
+
+	for _, ha := range hasAddrs {
+		assert.True(ha.has, fmt.Sprintf("Nothing for prefix %x\n", ha.prefix))
+	}
+}
+
+func TestGetMany(t *testing.T) {
+	assert := assert.New(t)
+
+	chunks := [][]byte{
+		[]byte("hello2"),
+		[]byte("goodbye2"),
+		[]byte("badbye2"),
+	}
+
+	tableData, _ := buildTable(chunks)
+	tr := newTableReader(tableData, memReaderAt(tableData))
+
+	addrs := addrSlice{computeAddr(chunks[0]), computeAddr(chunks[1]), computeAddr(chunks[2])}
+	getBatch := []getRecord{
+		{&addrs[0], binary.BigEndian.Uint64(addrs[0][:addrPrefixSize]), 0, nil},
+		{&addrs[1], binary.BigEndian.Uint64(addrs[1][:addrPrefixSize]), 1, nil},
+		{&addrs[2], binary.BigEndian.Uint64(addrs[2][:addrPrefixSize]), 2, nil},
+	}
+	sort.Sort(getRecordByPrefix(getBatch))
+
+	tr.getMany(getBatch)
+	for _, rec := range getBatch {
+		assert.NotNil(rec.data, "Nothing for prefix %d", rec.prefix)
+	}
+}
+
+func Test65k(t *testing.T) {
+	assert := assert.New(t)
+
+	count := 1 << 16
+	chunks := make([][]byte, count)
+
+	dataFn := func(i int) []byte {
+		return []byte(fmt.Sprintf("data%d", i*2))
+	}
+
+	for i := 0; i < count; i++ {
+		chunks[i] = dataFn(i)
+	}
+
+	tableData, _ := buildTable(chunks)
+	tr := newTableReader(tableData, memReaderAt(tableData))
+
+	for i := 0; i < count; i++ {
+		data := dataFn(i)
+		h := computeAddr(data)
+		assert.True(tr.has(computeAddr(data)))
+		assert.Equal(string(data), string(tr.get(h)))
+	}
+
+	for i := count; i < count*2; i++ {
+		data := dataFn(i)
+		h := computeAddr(data)
+		assert.False(tr.has(computeAddr(data)))
+		assert.NotEqual(string(data), string(tr.get(h)))
+	}
+}
+
+// Ensure all addresses share the first 7 bytes. Useful for easily generating tests which have
+// "prefix" collisions.
+func computeAddrCommonPrefix(data []byte) addr {
+	a := computeAddrDefault(data)
+	a[0] = 0x01
+	a[1] = 0x23
+	a[2] = 0x45
+	a[3] = 0x67
+	a[4] = 0x89
+	a[5] = 0xab
+	a[6] = 0xcd
+	return a
+}
+
+func doTestNGetMany(t *testing.T, count int) {
+	assert := assert.New(t)
+
+	chunks := make([][]byte, count)
+
+	dataFn := func(i int) []byte {
+		return []byte(fmt.Sprintf("data%d", i*2))
+	}
+
+	for i := 0; i < count; i++ {
+		chunks[i] = dataFn(i)
+	}
+
+	tableData, _ := buildTable(chunks)
+	tr := newTableReader(tableData, memReaderAt(tableData))
+
+	getBatch := make([]getRecord, len(chunks))
+	for i := 0; i < count; i++ {
+		a := computeAddr(dataFn(i))
+		getBatch[i] = getRecord{&a, a.Prefix(), i, nil}
+	}
+
+	sort.Sort(getRecordByPrefix(getBatch))
+
+	tr.getMany(getBatch)
+
+	sort.Sort(getRecordByOrder(getBatch))
+
+	for i := 0; i < count; i++ {
+		assert.True(bytes.Compare(getBatch[i].data, dataFn(i)) == 0)
+	}
+}
+
+func Test65kGetMany(t *testing.T) {
+	doTestNGetMany(t, 1<<16)
+}
+
+func Test2kGetManyCommonPrefix(t *testing.T) {
+	computeAddr = computeAddrCommonPrefix
+	defer func() {
+		computeAddr = computeAddrDefault
+	}()
+
+	doTestNGetMany(t, 1<<11)
+}
+
+func TestEmpty(t *testing.T) {
+	assert := assert.New(t)
+
+	buff := make([]byte, footerSize)
+	tw := newTableWriter(buff)
+	length, _ := tw.finish()
+	assert.Equal(length, footerSize)
+
+	d.PanicIfError(nil)
+}

go/nbs/table_writer.go

@@ -0,0 +1,141 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package nbs
+
+import (
+	"crypto/sha512"
+	"encoding/binary"
+	"hash"
+	"sort"
+
+	"github.com/attic-labs/noms/go/d"
+	"github.com/golang/snappy"
+)
+
+// tableWriter encodes a collection of byte stream chunks into a nbs table. NOT goroutine safe.
+type tableWriter struct {
+	buff              []byte
+	pos               uint64
+	totalPhysicalData uint64
+	prefixes          prefixIndexSlice // TODO: This is in danger of exploding memory
+	blockHash         hash.Hash
+}
+
+func maxTableSize(numChunks, totalData uint64) uint64 {
+	avgChunkSize := totalData / numChunks
+	d.Chk.True(avgChunkSize < maxChunkSize)
+	maxSnappySize := snappy.MaxEncodedLen(int(avgChunkSize))
+	d.Chk.True(maxSnappySize > 0)
+	return numChunks*(prefixTupleSize+lengthSize+addrSuffixSize+checksumSize+uint64(maxSnappySize)) + footerSize
+}
+
+func indexSize(numChunks uint64) uint64 {
+	return numChunks * (addrSuffixSize + lengthSize + prefixTupleSize)
+}
+
+// len(buff) must be >= maxTableSize(numChunks, totalData)
+func newTableWriter(buff []byte) *tableWriter {
+	return &tableWriter{
+		buff:      buff,
+		blockHash: sha512.New(),
+	}
+}
+
+func (tw *tableWriter) addChunk(h addr, data []byte) bool {
+	if len(data) == 0 {
+		panic("NBS blocks cannont be zero length")
+	}
+
+	// checksum (4 LSBytes, big-endian)
+	copy(tw.buff[tw.pos:tw.pos+checksumSize], h[addrSize-checksumSize:])
+	tw.pos += checksumSize
+
+	// Compress data straight into tw.buff
+	compressed := snappy.Encode(tw.buff[tw.pos:], data)
+	dataLength := uint64(len(compressed))
+	tw.pos += dataLength
+	tw.totalPhysicalData += dataLength
+
+	// Stored in insertion order
+	tw.prefixes = append(tw.prefixes, prefixIndexRec{
+		h.Prefix(),
+		h[addrPrefixSize:],
+		uint32(len(tw.prefixes)),
+		uint32(checksumSize + dataLength),
+	})
+
+	return true
+}
+
+func (tw *tableWriter) finish() (byteLength uint64, blockAddr addr) {
+	tw.writeIndex()
+	tw.writeFooter()
+	byteLength = tw.pos
+
+	var h []byte
+	h = tw.blockHash.Sum(h) // Appends hash to h
+	copy(blockAddr[:], h)
+	return
+}
+
+type prefixIndexRec struct {
+	prefix      uint64
+	suffix      []byte
+	order, size uint32
+}
+
+type prefixIndexSlice []prefixIndexRec
+
+func (hs prefixIndexSlice) Len() int           { return len(hs) }
+func (hs prefixIndexSlice) Less(i, j int) bool { return hs[i].prefix < hs[j].prefix }
+func (hs prefixIndexSlice) Swap(i, j int)      { hs[i], hs[j] = hs[j], hs[i] }
+
+func (tw *tableWriter) writeIndex() {
+	sort.Sort(tw.prefixes)
+
+	pfxScratch := [addrPrefixSize]byte{}
+
+	numRecords := uint64(len(tw.prefixes))
+	lengthsOffset := tw.pos + numRecords*prefixTupleSize    // skip prefix and ordinal for each record
+	suffixesOffset := lengthsOffset + numRecords*lengthSize // skip size for each record
+	for _, pi := range tw.prefixes {
+		binary.BigEndian.PutUint64(pfxScratch[:], pi.prefix)
+		tw.blockHash.Write(pfxScratch[:])
+		tw.blockHash.Write(pi.suffix)
+
+		// hash prefix
+		n := uint64(copy(tw.buff[tw.pos:], pfxScratch[:]))
+		d.Chk.True(n == addrPrefixSize)
+		tw.pos += n
+
+		// order
+		binary.BigEndian.PutUint32(tw.buff[tw.pos:], pi.order)
+		tw.pos += ordinalSize
+
+		// length
+		offset := lengthsOffset + uint64(pi.order)*lengthSize
+		binary.BigEndian.PutUint32(tw.buff[offset:], pi.size)
+
+		// hash suffix
+		offset = suffixesOffset + uint64(pi.order)*addrSuffixSize
+		n = uint64(copy(tw.buff[offset:], pi.suffix))
+		d.Chk.True(n == addrSuffixSize)
+	}
+	tw.pos = suffixesOffset + numRecords*addrSuffixSize
+}
+
+func (tw *tableWriter) writeFooter() {
+	// chunk count
+	binary.BigEndian.PutUint64(tw.buff[tw.pos:], uint64(len(tw.prefixes)))
+	tw.pos += uint64Size
+
+	// total chunk data
+	binary.BigEndian.PutUint64(tw.buff[tw.pos:], tw.totalPhysicalData)
+	tw.pos += uint64Size
+
+	// magic number
+	copy(tw.buff[tw.pos:], magicNumber)
+	tw.pos += magicNumberSize
+}

go/nbs/test/manifest_clobber.go

@@ -0,0 +1,47 @@
+// Copyright 2016 Attic Labs, Inc. All rights reserved.
+// Licensed under the Apache License, version 2.0:
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+var takeLock = flag.Bool("take-lock", false, "Expect to be able to lock the lock file.")
+
+func main() {
+	flag.Parse()
+
+	if flag.NArg() < 3 {
+		log.Fatalln("Not enough arguments")
+	}
+
+	l, err := os.Create(flag.Arg(0))
+	if err != nil {
+		log.Fatalln(err)
+	}
+	defer l.Close()
+	// lock released by closing l.
+	err = unix.Flock(int(l.Fd()), unix.LOCK_EX|unix.LOCK_NB)
+	if err != nil {
+		if !*takeLock && err == unix.EWOULDBLOCK {
+			return
+		}
+		log.Fatalln(err)
+	}
+
+	// Clobber manifest file at flag.Arg(1) with contents at flag.Arg(2)
+	m, err := os.Create(flag.Arg(1))
+	if err != nil {
+		log.Fatalln(err)
+	}
+	defer m.Close()
+	if _, err = m.WriteString(flag.Arg(2)); err != nil {
+		log.Fatalln(err)
+	}
+}

go/spec/spec.go

@@ -15,6 +15,7 @@ import (
 	"github.com/attic-labs/noms/go/chunks"
 	"github.com/attic-labs/noms/go/d"
 	"github.com/attic-labs/noms/go/datas"
+	"github.com/attic-labs/noms/go/nbs"
 	"github.com/attic-labs/noms/go/types"
 )
 
@@ -155,6 +156,8 @@ func (sp Spec) NewChunkStore() chunks.ChunkStore {
 	switch sp.Protocol {
 	case "http", "https":
 		return nil
+	case "nbs":
+		return nbs.NewBlockStore(sp.DatabaseName, 1<<28)
 	case "ldb":
 		return getLdbStore(sp.DatabaseName)
 	case "mem":
@@ -246,6 +249,8 @@ func (sp Spec) createDatabase() datas.Database {
 		return datas.NewRemoteDatabase(sp.Href(), sp.Options.Authorization)
 	case "ldb":
 		return datas.NewDatabase(getLdbStore(sp.DatabaseName))
+	case "nbs":
+		return datas.NewDatabase(nbs.NewBlockStore(sp.DatabaseName, 1<<28))
 	case "mem":
 		return datas.NewDatabase(chunks.NewMemoryStore())
 	}
@@ -273,7 +278,7 @@ func parseDatabaseSpec(spec string) (protocol, name string, err error) {
 	}
 
 	switch parts[0] {
-	case "ldb":
+	case "ldb", "nbs":
 		protocol, name = parts[0], parts[1]
 
 	case "http", "https":

vendor/golang.org/x/sys/AUTHORS

@@ -0,0 +1,3 @@
+# This source code refers to The Go Authors for copyright purposes.
+# The master list of authors is in the main Go distribution,
+# visible at http://tip.golang.org/AUTHORS.

vendor/golang.org/x/sys/CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# Contributing to Go
+
+Go is an open source project.
+
+It is the work of hundreds of contributors. We appreciate your help!
+
+
+## Filing issues
+
+When [filing an issue](https://golang.org/issue/new), make sure to answer these five questions:
+
+1. What version of Go are you using (`go version`)?
+2. What operating system and processor architecture are you using?
+3. What did you do?
+4. What did you expect to see?
+5. What did you see instead?
+
+General questions should go to the [golang-nuts mailing list](https://groups.google.com/group/golang-nuts) instead of the issue tracker.
+The gophers there will answer or ask you to file an issue if you've tripped over a bug.
+
+## Contributing code
+
+Please read the [Contribution Guidelines](https://golang.org/doc/contribute.html)
+before sending patches.
+
+**We do not accept GitHub pull requests**
+(we use [Gerrit](https://code.google.com/p/gerrit/) instead for code review).
+
+Unless otherwise noted, the Go source files are distributed under
+the BSD-style license found in the LICENSE file.
+

vendor/golang.org/x/sys/CONTRIBUTORS

@@ -0,0 +1,3 @@
+# This source code was written by the Go contributors.
+# The master list of contributors is in the main Go distribution,
+# visible at http://tip.golang.org/CONTRIBUTORS.

vendor/golang.org/x/sys/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/golang.org/x/sys/PATENTS

@@ -0,0 +1,22 @@
+Additional IP Rights Grant (Patents)
+
+"This implementation" means the copyrightable works distributed by
+Google as part of the Go project.
+
+Google hereby grants to You a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable (except as stated in this section)
+patent license to make, have made, use, offer to sell, sell, import,
+transfer and otherwise run, modify and propagate the contents of this
+implementation of Go, where such license applies only to those patent
+claims, both currently owned or controlled by Google and acquired in
+the future, licensable by Google that are necessarily infringed by this
+implementation of Go.  This grant does not include claims that would be
+infringed only as a consequence of further modification of this
+implementation.  If you or your agent or exclusive licensee institute or
+order or agree to the institution of patent litigation against any
+entity (including a cross-claim or counterclaim in a lawsuit) alleging
+that this implementation of Go or any code incorporated within this
+implementation of Go constitutes direct or contributory patent
+infringement, or inducement of patent infringement, then any patent
+rights granted to you under this License for this implementation of Go
+shall terminate as of the date such litigation is filed.

vendor/golang.org/x/sys/README

@@ -0,0 +1,3 @@
+This repository holds supplemental Go packages for low-level interactions with the operating system.
+
+To submit changes to this repository, see http://golang.org/doc/contribute.html.

vendor/golang.org/x/sys/codereview.cfg

@@ -0,0 +1 @@
+issuerepo: golang/go

vendor/golang.org/x/sys/unix/.gitignore

@@ -0,0 +1 @@
+_obj/

vendor/golang.org/x/sys/unix/asm.s

@@ -0,0 +1,10 @@
+// Copyright 2014 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+TEXT ·use(SB),NOSPLIT,$0
+	RET

vendor/golang.org/x/sys/unix/asm_darwin_386.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for 386, Darwin
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-28
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-52
+	JMP	syscall·Syscall9(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_darwin_amd64.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for AMD64, Darwin
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-56
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-104
+	JMP	syscall·Syscall9(SB)
+
+TEXT	·RawSyscall(SB),NOSPLIT,$0-56
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_darwin_arm.s

@@ -0,0 +1,30 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+// +build arm,darwin
+
+#include "textflag.h"
+
+//
+// System call support for ARM, Darwin
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-28
+	B	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-40
+	B	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-52
+	B	syscall·Syscall9(SB)
+
+TEXT	·RawSyscall(SB),NOSPLIT,$0-28
+	B	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-40
+	B	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_darwin_arm64.s

@@ -0,0 +1,30 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+// +build arm64,darwin
+
+#include "textflag.h"
+
+//
+// System call support for AMD64, Darwin
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-56
+	B	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-80
+	B	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-104
+	B	syscall·Syscall9(SB)
+
+TEXT	·RawSyscall(SB),NOSPLIT,$0-56
+	B	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-80
+	B	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_dragonfly_amd64.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for AMD64, DragonFly
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-64
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-88
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-112
+	JMP	syscall·Syscall9(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-64
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-88
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_freebsd_386.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for 386, FreeBSD
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-28
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-52
+	JMP	syscall·Syscall9(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_freebsd_amd64.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for AMD64, FreeBSD
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-56
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-104
+	JMP	syscall·Syscall9(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-56
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_freebsd_arm.s

@@ -0,0 +1,29 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for ARM, FreeBSD
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-28
+	B	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-40
+	B	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-52
+	B	syscall·Syscall9(SB)
+
+TEXT	·RawSyscall(SB),NOSPLIT,$0-28
+	B	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-40
+	B	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_linux_386.s

@@ -0,0 +1,35 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System calls for 386, Linux
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-28
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·RawSyscall6(SB)
+
+TEXT ·socketcall(SB),NOSPLIT,$0-36
+	JMP	syscall·socketcall(SB)
+
+TEXT ·rawsocketcall(SB),NOSPLIT,$0-36
+	JMP	syscall·rawsocketcall(SB)
+
+TEXT ·seek(SB),NOSPLIT,$0-28
+	JMP	syscall·seek(SB)

vendor/golang.org/x/sys/unix/asm_linux_amd64.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System calls for AMD64, Linux
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-56
+	JMP	syscall·Syscall(SB)
+
+TEXT ·Syscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-56
+	JMP	syscall·RawSyscall(SB)
+
+TEXT ·RawSyscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·RawSyscall6(SB)
+
+TEXT ·gettimeofday(SB),NOSPLIT,$0-16
+	JMP	syscall·gettimeofday(SB)

vendor/golang.org/x/sys/unix/asm_linux_arm.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System calls for arm, Linux
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-28
+	B	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-40
+	B	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	B	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-40
+	B	syscall·RawSyscall6(SB)
+
+TEXT ·seek(SB),NOSPLIT,$0-32
+	B	syscall·seek(SB)

vendor/golang.org/x/sys/unix/asm_linux_arm64.s

@@ -0,0 +1,24 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+// +build arm64
+// +build !gccgo
+
+#include "textflag.h"
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-56
+	B	syscall·Syscall(SB)
+
+TEXT ·Syscall6(SB),NOSPLIT,$0-80
+	B	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-56
+	B	syscall·RawSyscall(SB)
+
+TEXT ·RawSyscall6(SB),NOSPLIT,$0-80
+	B	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_linux_mips64x.s

@@ -0,0 +1,28 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+// +build mips64 mips64le
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System calls for mips64, Linux
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-56
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·RawSyscall(SB),NOSPLIT,$0-56
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_linux_ppc64x.s

@@ -0,0 +1,28 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+// +build ppc64 ppc64le
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System calls for ppc64, Linux
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-56
+	BR	syscall·Syscall(SB)
+
+TEXT ·Syscall6(SB),NOSPLIT,$0-80
+	BR	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-56
+	BR	syscall·RawSyscall(SB)
+
+TEXT ·RawSyscall6(SB),NOSPLIT,$0-80
+	BR	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_linux_s390x.s

@@ -0,0 +1,28 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build s390x
+// +build linux
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System calls for s390x, Linux
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT ·Syscall(SB),NOSPLIT,$0-56
+	BR	syscall·Syscall(SB)
+
+TEXT ·Syscall6(SB),NOSPLIT,$0-80
+	BR	syscall·Syscall6(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-56
+	BR	syscall·RawSyscall(SB)
+
+TEXT ·RawSyscall6(SB),NOSPLIT,$0-80
+	BR	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_netbsd_386.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for 386, NetBSD
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-28
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-52
+	JMP	syscall·Syscall9(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_netbsd_amd64.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for AMD64, NetBSD
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-56
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-104
+	JMP	syscall·Syscall9(SB)
+
+TEXT	·RawSyscall(SB),NOSPLIT,$0-56
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_netbsd_arm.s

@@ -0,0 +1,29 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for ARM, NetBSD
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-28
+	B	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-40
+	B	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-52
+	B	syscall·Syscall9(SB)
+
+TEXT	·RawSyscall(SB),NOSPLIT,$0-28
+	B	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-40
+	B	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_openbsd_386.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for 386, OpenBSD
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-28
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-52
+	JMP	syscall·Syscall9(SB)
+
+TEXT ·RawSyscall(SB),NOSPLIT,$0-28
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-40
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_openbsd_amd64.s

@@ -0,0 +1,29 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System call support for AMD64, OpenBSD
+//
+
+// Just jump to package syscall's implementation for all these functions.
+// The runtime may know about them.
+
+TEXT	·Syscall(SB),NOSPLIT,$0-56
+	JMP	syscall·Syscall(SB)
+
+TEXT	·Syscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·Syscall6(SB)
+
+TEXT	·Syscall9(SB),NOSPLIT,$0-104
+	JMP	syscall·Syscall9(SB)
+
+TEXT	·RawSyscall(SB),NOSPLIT,$0-56
+	JMP	syscall·RawSyscall(SB)
+
+TEXT	·RawSyscall6(SB),NOSPLIT,$0-80
+	JMP	syscall·RawSyscall6(SB)

vendor/golang.org/x/sys/unix/asm_solaris_amd64.s

@@ -0,0 +1,17 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+//
+// System calls for amd64, Solaris are implemented in runtime/syscall_solaris.go
+//
+
+TEXT ·sysvicall6(SB),NOSPLIT,$0-64
+	JMP	syscall·sysvicall6(SB)
+
+TEXT ·rawSysvicall6(SB),NOSPLIT,$0-64
+	JMP	syscall·rawSysvicall6(SB)

vendor/golang.org/x/sys/unix/bluetooth_linux.go

@@ -0,0 +1,35 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Bluetooth sockets and messages
+
+package unix
+
+// Bluetooth Protocols
+const (
+	BTPROTO_L2CAP  = 0
+	BTPROTO_HCI    = 1
+	BTPROTO_SCO    = 2
+	BTPROTO_RFCOMM = 3
+	BTPROTO_BNEP   = 4
+	BTPROTO_CMTP   = 5
+	BTPROTO_HIDP   = 6
+	BTPROTO_AVDTP  = 7
+)
+
+const (
+	HCI_CHANNEL_RAW     = 0
+	HCI_CHANNEL_USER    = 1
+	HCI_CHANNEL_MONITOR = 2
+	HCI_CHANNEL_CONTROL = 3
+)
+
+// Socketoption Level
+const (
+	SOL_BLUETOOTH = 0x112
+	SOL_HCI       = 0x0
+	SOL_L2CAP     = 0x6
+	SOL_RFCOMM    = 0x12
+	SOL_SCO       = 0x11
+)

vendor/golang.org/x/sys/unix/constants.go

@@ -0,0 +1,13 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package unix
+
+const (
+	R_OK = 0x4
+	W_OK = 0x2
+	X_OK = 0x1
+)

vendor/golang.org/x/sys/unix/creds_test.go

@@ -0,0 +1,121 @@
+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package unix_test
+
+import (
+	"bytes"
+	"net"
+	"os"
+	"syscall"
+	"testing"
+
+	"golang.org/x/sys/unix"
+)
+
+// TestSCMCredentials tests the sending and receiving of credentials
+// (PID, UID, GID) in an ancillary message between two UNIX
+// sockets. The SO_PASSCRED socket option is enabled on the sending
+// socket for this to work.
+func TestSCMCredentials(t *testing.T) {
+	fds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_STREAM, 0)
+	if err != nil {
+		t.Fatalf("Socketpair: %v", err)
+	}
+	defer unix.Close(fds[0])
+	defer unix.Close(fds[1])
+
+	err = unix.SetsockoptInt(fds[0], unix.SOL_SOCKET, unix.SO_PASSCRED, 1)
+	if err != nil {
+		t.Fatalf("SetsockoptInt: %v", err)
+	}
+
+	srvFile := os.NewFile(uintptr(fds[0]), "server")
+	defer srvFile.Close()
+	srv, err := net.FileConn(srvFile)
+	if err != nil {
+		t.Errorf("FileConn: %v", err)
+		return
+	}
+	defer srv.Close()
+
+	cliFile := os.NewFile(uintptr(fds[1]), "client")
+	defer cliFile.Close()
+	cli, err := net.FileConn(cliFile)
+	if err != nil {
+		t.Errorf("FileConn: %v", err)
+		return
+	}
+	defer cli.Close()
+
+	var ucred unix.Ucred
+	if os.Getuid() != 0 {
+		ucred.Pid = int32(os.Getpid())
+		ucred.Uid = 0
+		ucred.Gid = 0
+		oob := unix.UnixCredentials(&ucred)
+		_, _, err := cli.(*net.UnixConn).WriteMsgUnix(nil, oob, nil)
+		if op, ok := err.(*net.OpError); ok {
+			err = op.Err
+		}
+		if sys, ok := err.(*os.SyscallError); ok {
+			err = sys.Err
+		}
+		if err != syscall.EPERM {
+			t.Fatalf("WriteMsgUnix failed with %v, want EPERM", err)
+		}
+	}
+
+	ucred.Pid = int32(os.Getpid())
+	ucred.Uid = uint32(os.Getuid())
+	ucred.Gid = uint32(os.Getgid())
+	oob := unix.UnixCredentials(&ucred)
+
+	// this is going to send a dummy byte
+	n, oobn, err := cli.(*net.UnixConn).WriteMsgUnix(nil, oob, nil)
+	if err != nil {
+		t.Fatalf("WriteMsgUnix: %v", err)
+	}
+	if n != 0 {
+		t.Fatalf("WriteMsgUnix n = %d, want 0", n)
+	}
+	if oobn != len(oob) {
+		t.Fatalf("WriteMsgUnix oobn = %d, want %d", oobn, len(oob))
+	}
+
+	oob2 := make([]byte, 10*len(oob))
+	n, oobn2, flags, _, err := srv.(*net.UnixConn).ReadMsgUnix(nil, oob2)
+	if err != nil {
+		t.Fatalf("ReadMsgUnix: %v", err)
+	}
+	if flags != 0 {
+		t.Fatalf("ReadMsgUnix flags = 0x%x, want 0", flags)
+	}
+	if n != 1 {
+		t.Fatalf("ReadMsgUnix n = %d, want 1 (dummy byte)", n)
+	}
+	if oobn2 != oobn {
+		// without SO_PASSCRED set on the socket, ReadMsgUnix will
+		// return zero oob bytes
+		t.Fatalf("ReadMsgUnix oobn = %d, want %d", oobn2, oobn)
+	}
+	oob2 = oob2[:oobn2]
+	if !bytes.Equal(oob, oob2) {
+		t.Fatal("ReadMsgUnix oob bytes don't match")
+	}
+
+	scm, err := unix.ParseSocketControlMessage(oob2)
+	if err != nil {
+		t.Fatalf("ParseSocketControlMessage: %v", err)
+	}
+	newUcred, err := unix.ParseUnixCredentials(&scm[0])
+	if err != nil {
+		t.Fatalf("ParseUnixCredentials: %v", err)
+	}
+	if *newUcred != ucred {
+		t.Fatalf("ParseUnixCredentials = %+v, want %+v", newUcred, ucred)
+	}
+}

vendor/golang.org/x/sys/unix/env_unix.go

@@ -0,0 +1,27 @@
+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+// Unix environment variables.
+
+package unix
+
+import "syscall"
+
+func Getenv(key string) (value string, found bool) {
+	return syscall.Getenv(key)
+}
+
+func Setenv(key, value string) error {
+	return syscall.Setenv(key, value)
+}
+
+func Clearenv() {
+	syscall.Clearenv()
+}
+
+func Environ() []string {
+	return syscall.Environ()
+}

vendor/golang.org/x/sys/unix/env_unset.go

@@ -0,0 +1,14 @@
+// Copyright 2014 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.4
+
+package unix
+
+import "syscall"
+
+func Unsetenv(key string) error {
+	// This was added in Go 1.4.
+	return syscall.Unsetenv(key)
+}

vendor/golang.org/x/sys/unix/export_test.go

@@ -0,0 +1,9 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package unix
+
+var Itoa = itoa

vendor/golang.org/x/sys/unix/flock.go

@@ -0,0 +1,24 @@
+// +build linux darwin freebsd openbsd netbsd dragonfly
+
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd
+
+package unix
+
+import "unsafe"
+
+// fcntl64Syscall is usually SYS_FCNTL, but is overridden on 32-bit Linux
+// systems by flock_linux_32bit.go to be SYS_FCNTL64.
+var fcntl64Syscall uintptr = SYS_FCNTL
+
+// FcntlFlock performs a fcntl syscall for the F_GETLK, F_SETLK or F_SETLKW command.
+func FcntlFlock(fd uintptr, cmd int, lk *Flock_t) error {
+	_, _, errno := Syscall(fcntl64Syscall, fd, uintptr(cmd), uintptr(unsafe.Pointer(lk)))
+	if errno == 0 {
+		return nil
+	}
+	return errno
+}

vendor/golang.org/x/sys/unix/flock_linux_32bit.go

@@ -0,0 +1,13 @@
+// +build linux,386 linux,arm
+
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unix
+
+func init() {
+	// On 32-bit Linux systems, the fcntl syscall that matches Go's
+	// Flock_t type is SYS_FCNTL64, not SYS_FCNTL.
+	fcntl64Syscall = SYS_FCNTL64
+}

vendor/golang.org/x/sys/unix/gccgo.go

@@ -0,0 +1,46 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build gccgo
+
+package unix
+
+import "syscall"
+
+// We can't use the gc-syntax .s files for gccgo.  On the plus side
+// much of the functionality can be written directly in Go.
+
+//extern gccgoRealSyscall
+func realSyscall(trap, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r, errno uintptr)
+
+func Syscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) {
+	syscall.Entersyscall()
+	r, errno := realSyscall(trap, a1, a2, a3, 0, 0, 0, 0, 0, 0)
+	syscall.Exitsyscall()
+	return r, 0, syscall.Errno(errno)
+}
+
+func Syscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) {
+	syscall.Entersyscall()
+	r, errno := realSyscall(trap, a1, a2, a3, a4, a5, a6, 0, 0, 0)
+	syscall.Exitsyscall()
+	return r, 0, syscall.Errno(errno)
+}
+
+func Syscall9(trap, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) {
+	syscall.Entersyscall()
+	r, errno := realSyscall(trap, a1, a2, a3, a4, a5, a6, a7, a8, a9)
+	syscall.Exitsyscall()
+	return r, 0, syscall.Errno(errno)
+}
+
+func RawSyscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) {
+	r, errno := realSyscall(trap, a1, a2, a3, 0, 0, 0, 0, 0, 0)
+	return r, 0, syscall.Errno(errno)
+}
+
+func RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) {
+	r, errno := realSyscall(trap, a1, a2, a3, a4, a5, a6, 0, 0, 0)
+	return r, 0, syscall.Errno(errno)
+}

vendor/golang.org/x/sys/unix/gccgo_c.c

@@ -0,0 +1,41 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build gccgo
+
+#include <errno.h>
+#include <stdint.h>
+#include <unistd.h>
+
+#define _STRINGIFY2_(x) #x
+#define _STRINGIFY_(x) _STRINGIFY2_(x)
+#define GOSYM_PREFIX _STRINGIFY_(__USER_LABEL_PREFIX__)
+
+// Call syscall from C code because the gccgo support for calling from
+// Go to C does not support varargs functions.
+
+struct ret {
+	uintptr_t r;
+	uintptr_t err;
+};
+
+struct ret
+gccgoRealSyscall(uintptr_t trap, uintptr_t a1, uintptr_t a2, uintptr_t a3, uintptr_t a4, uintptr_t a5, uintptr_t a6, uintptr_t a7, uintptr_t a8, uintptr_t a9)
+{
+	struct ret r;
+
+	errno = 0;
+	r.r = syscall(trap, a1, a2, a3, a4, a5, a6, a7, a8, a9);
+	r.err = errno;
+	return r;
+}
+
+// Define the use function in C so that it is not inlined.
+
+extern void use(void *) __asm__ (GOSYM_PREFIX GOPKGPATH ".use") __attribute__((noinline));
+
+void
+use(void *p __attribute__ ((unused)))
+{
+}

vendor/golang.org/x/sys/unix/gccgo_linux_amd64.go

@@ -0,0 +1,20 @@
+// Copyright 2015 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build gccgo,linux,amd64
+
+package unix
+
+import "syscall"
+
+//extern gettimeofday
+func realGettimeofday(*Timeval, *byte) int32
+
+func gettimeofday(tv *Timeval) (err syscall.Errno) {
+	r := realGettimeofday(tv, nil)
+	if r < 0 {
+		return syscall.GetErrno()
+	}
+	return 0
+}

vendor/golang.org/x/sys/unix/gccgo_linux_sparc64.go

@@ -0,0 +1,20 @@
+// Copyright 2016 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build gccgo,linux,sparc64
+
+package unix
+
+import "syscall"
+
+//extern sysconf
+func realSysconf(name int) int64
+
+func sysconf(name int) (n int64, err syscall.Errno) {
+	r := realSysconf(name)
+	if r < 0 {
+		return 0, syscall.GetErrno()
+	}
+	return r, 0
+}

vendor/golang.org/x/sys/unix/mkall.sh

@@ -0,0 +1,292 @@
+#!/usr/bin/env bash
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# The unix package provides access to the raw system call
+# interface of the underlying operating system.  Porting Go to
+# a new architecture/operating system combination requires
+# some manual effort, though there are tools that automate
+# much of the process.  The auto-generated files have names
+# beginning with z.
+#
+# This script runs or (given -n) prints suggested commands to generate z files
+# for the current system.  Running those commands is not automatic.
+# This script is documentation more than anything else.
+#
+# * asm_${GOOS}_${GOARCH}.s
+#
+# This hand-written assembly file implements system call dispatch.
+# There are three entry points:
+#
+# 	func Syscall(trap, a1, a2, a3 uintptr) (r1, r2, err uintptr);
+# 	func Syscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr);
+# 	func RawSyscall(trap, a1, a2, a3 uintptr) (r1, r2, err uintptr);
+#
+# The first and second are the standard ones; they differ only in
+# how many arguments can be passed to the kernel.
+# The third is for low-level use by the ForkExec wrapper;
+# unlike the first two, it does not call into the scheduler to
+# let it know that a system call is running.
+#
+# * syscall_${GOOS}.go
+#
+# This hand-written Go file implements system calls that need
+# special handling and lists "//sys" comments giving prototypes
+# for ones that can be auto-generated.  Mksyscall reads those
+# comments to generate the stubs.
+#
+# * syscall_${GOOS}_${GOARCH}.go
+#
+# Same as syscall_${GOOS}.go except that it contains code specific
+# to ${GOOS} on one particular architecture.
+#
+# * types_${GOOS}.c
+#
+# This hand-written C file includes standard C headers and then
+# creates typedef or enum names beginning with a dollar sign
+# (use of $ in variable names is a gcc extension).  The hardest
+# part about preparing this file is figuring out which headers to
+# include and which symbols need to be #defined to get the
+# actual data structures that pass through to the kernel system calls.
+# Some C libraries present alternate versions for binary compatibility
+# and translate them on the way in and out of system calls, but
+# there is almost always a #define that can get the real ones.
+# See types_darwin.c and types_linux.c for examples.
+#
+# * zerror_${GOOS}_${GOARCH}.go
+#
+# This machine-generated file defines the system's error numbers,
+# error strings, and signal numbers.  The generator is "mkerrors.sh".
+# Usually no arguments are needed, but mkerrors.sh will pass its
+# arguments on to godefs.
+#
+# * zsyscall_${GOOS}_${GOARCH}.go
+#
+# Generated by mksyscall.pl; see syscall_${GOOS}.go above.
+#
+# * zsysnum_${GOOS}_${GOARCH}.go
+#
+# Generated by mksysnum_${GOOS}.
+#
+# * ztypes_${GOOS}_${GOARCH}.go
+#
+# Generated by godefs; see types_${GOOS}.c above.
+
+GOOSARCH="${GOOS}_${GOARCH}"
+
+# defaults
+mksyscall="./mksyscall.pl"
+mkerrors="./mkerrors.sh"
+zerrors="zerrors_$GOOSARCH.go"
+mksysctl=""
+zsysctl="zsysctl_$GOOSARCH.go"
+mksysnum=
+mktypes=
+run="sh"
+
+case "$1" in
+-syscalls)
+	for i in zsyscall*go
+	do
+		sed 1q $i | sed 's;^// ;;' | sh > _$i && gofmt < _$i > $i
+		rm _$i
+	done
+	exit 0
+	;;
+-n)
+	run="cat"
+	shift
+esac
+
+case "$#" in
+0)
+	;;
+*)
+	echo 'usage: mkall.sh [-n]' 1>&2
+	exit 2
+esac
+
+GOOSARCH_in=syscall_$GOOSARCH.go
+case "$GOOSARCH" in
+_* | *_ | _)
+	echo 'undefined $GOOS_$GOARCH:' "$GOOSARCH" 1>&2
+	exit 1
+	;;
+darwin_386)
+	mkerrors="$mkerrors -m32"
+	mksyscall="./mksyscall.pl -l32"
+	mksysnum="./mksysnum_darwin.pl $(xcrun --show-sdk-path --sdk macosx)/usr/include/sys/syscall.h"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+darwin_amd64)
+	mkerrors="$mkerrors -m64"
+	mksysnum="./mksysnum_darwin.pl $(xcrun --show-sdk-path --sdk macosx)/usr/include/sys/syscall.h"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+darwin_arm)
+	mkerrors="$mkerrors"
+	mksysnum="./mksysnum_darwin.pl /usr/include/sys/syscall.h"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+darwin_arm64)
+	mkerrors="$mkerrors -m64"
+	mksysnum="./mksysnum_darwin.pl $(xcrun --show-sdk-path --sdk iphoneos)/usr/include/sys/syscall.h"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+dragonfly_386)
+	mkerrors="$mkerrors -m32"
+	mksyscall="./mksyscall.pl -l32 -dragonfly"
+	mksysnum="curl -s 'http://gitweb.dragonflybsd.org/dragonfly.git/blob_plain/HEAD:/sys/kern/syscalls.master' | ./mksysnum_dragonfly.pl"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+dragonfly_amd64)
+	mkerrors="$mkerrors -m64"
+	mksyscall="./mksyscall.pl -dragonfly"
+	mksysnum="curl -s 'http://gitweb.dragonflybsd.org/dragonfly.git/blob_plain/HEAD:/sys/kern/syscalls.master' | ./mksysnum_dragonfly.pl"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+freebsd_386)
+	mkerrors="$mkerrors -m32"
+	mksyscall="./mksyscall.pl -l32"
+	mksysnum="curl -s 'http://svn.freebsd.org/base/stable/10/sys/kern/syscalls.master' | ./mksysnum_freebsd.pl"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+freebsd_amd64)
+	mkerrors="$mkerrors -m64"
+	mksysnum="curl -s 'http://svn.freebsd.org/base/stable/10/sys/kern/syscalls.master' | ./mksysnum_freebsd.pl"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+freebsd_arm)
+	mkerrors="$mkerrors"
+	mksyscall="./mksyscall.pl -l32 -arm"
+	mksysnum="curl -s 'http://svn.freebsd.org/base/stable/10/sys/kern/syscalls.master' | ./mksysnum_freebsd.pl"
+	# Let the type of C char be signed for making the bare syscall
+	# API consistent across over platforms.
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs -- -fsigned-char"
+	;;
+linux_386)
+	mkerrors="$mkerrors -m32"
+	mksyscall="./mksyscall.pl -l32"
+	mksysnum="./mksysnum_linux.pl /usr/include/asm/unistd_32.h"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+linux_amd64)
+	unistd_h=$(ls -1 /usr/include/asm/unistd_64.h /usr/include/x86_64-linux-gnu/asm/unistd_64.h 2>/dev/null | head -1)
+	if [ "$unistd_h" = "" ]; then
+		echo >&2 cannot find unistd_64.h
+		exit 1
+	fi
+	mkerrors="$mkerrors -m64"
+	mksysnum="./mksysnum_linux.pl $unistd_h"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+linux_arm)
+	mkerrors="$mkerrors"
+	mksyscall="./mksyscall.pl -l32 -arm"
+	mksysnum="curl -s 'http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/plain/arch/arm/include/uapi/asm/unistd.h' | ./mksysnum_linux.pl -"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+linux_arm64)
+	unistd_h=$(ls -1 /usr/include/asm/unistd.h /usr/include/asm-generic/unistd.h 2>/dev/null | head -1)
+	if [ "$unistd_h" = "" ]; then
+		echo >&2 cannot find unistd_64.h
+		exit 1
+	fi
+	mksysnum="./mksysnum_linux.pl $unistd_h"
+	# Let the type of C char be signed for making the bare syscall
+	# API consistent across over platforms.
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs -- -fsigned-char"
+	;;
+linux_ppc64)
+	GOOSARCH_in=syscall_linux_ppc64x.go
+	unistd_h=/usr/include/asm/unistd.h
+	mkerrors="$mkerrors -m64"
+	mksysnum="./mksysnum_linux.pl $unistd_h"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+linux_ppc64le)
+	GOOSARCH_in=syscall_linux_ppc64x.go
+	unistd_h=/usr/include/powerpc64le-linux-gnu/asm/unistd.h
+	mkerrors="$mkerrors -m64"
+	mksysnum="./mksysnum_linux.pl $unistd_h"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+linux_s390x)
+	GOOSARCH_in=syscall_linux_s390x.go
+	unistd_h=/usr/include/asm/unistd.h
+	mkerrors="$mkerrors -m64"
+	mksysnum="./mksysnum_linux.pl $unistd_h"
+	# Let the type of C char be signed to make the bare sys
+	# API more consistent between platforms.
+	# This is a deliberate departure from the way the syscall
+	# package generates its version of the types file.
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs -- -fsigned-char"
+	;;
+linux_sparc64)
+	GOOSARCH_in=syscall_linux_sparc64.go
+	unistd_h=/usr/include/sparc64-linux-gnu/asm/unistd.h
+	mkerrors="$mkerrors -m64"
+	mksysnum="./mksysnum_linux.pl $unistd_h"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+netbsd_386)
+	mkerrors="$mkerrors -m32"
+	mksyscall="./mksyscall.pl -l32 -netbsd"
+	mksysnum="curl -s 'http://cvsweb.netbsd.org/bsdweb.cgi/~checkout~/src/sys/kern/syscalls.master' | ./mksysnum_netbsd.pl"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+netbsd_amd64)
+	mkerrors="$mkerrors -m64"
+	mksyscall="./mksyscall.pl -netbsd"
+	mksysnum="curl -s 'http://cvsweb.netbsd.org/bsdweb.cgi/~checkout~/src/sys/kern/syscalls.master' | ./mksysnum_netbsd.pl"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+openbsd_386)
+	mkerrors="$mkerrors -m32"
+	mksyscall="./mksyscall.pl -l32 -openbsd"
+	mksysctl="./mksysctl_openbsd.pl"
+	zsysctl="zsysctl_openbsd.go"
+	mksysnum="curl -s 'http://cvsweb.openbsd.org/cgi-bin/cvsweb/~checkout~/src/sys/kern/syscalls.master' | ./mksysnum_openbsd.pl"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+openbsd_amd64)
+	mkerrors="$mkerrors -m64"
+	mksyscall="./mksyscall.pl -openbsd"
+	mksysctl="./mksysctl_openbsd.pl"
+	zsysctl="zsysctl_openbsd.go"
+	mksysnum="curl -s 'http://cvsweb.openbsd.org/cgi-bin/cvsweb/~checkout~/src/sys/kern/syscalls.master' | ./mksysnum_openbsd.pl"
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+solaris_amd64)
+	mksyscall="./mksyscall_solaris.pl"
+	mkerrors="$mkerrors -m64"
+	mksysnum=
+	mktypes="GOARCH=$GOARCH go tool cgo -godefs"
+	;;
+*)
+	echo 'unrecognized $GOOS_$GOARCH: ' "$GOOSARCH" 1>&2
+	exit 1
+	;;
+esac
+
+(
+	if [ -n "$mkerrors" ]; then echo "$mkerrors |gofmt >$zerrors"; fi
+	case "$GOOS" in
+	*)
+		syscall_goos="syscall_$GOOS.go"
+		case "$GOOS" in
+		darwin | dragonfly | freebsd | netbsd | openbsd)
+			syscall_goos="syscall_bsd.go $syscall_goos"
+			;;
+		esac
+		if [ -n "$mksyscall" ]; then echo "$mksyscall $syscall_goos $GOOSARCH_in |gofmt >zsyscall_$GOOSARCH.go"; fi
+		;;
+	esac
+	if [ -n "$mksysctl" ]; then echo "$mksysctl |gofmt >$zsysctl"; fi
+	if [ -n "$mksysnum" ]; then echo "$mksysnum |gofmt >zsysnum_$GOOSARCH.go"; fi
+	if [ -n "$mktypes" ]; then
+		echo "echo // +build $GOARCH,$GOOS > ztypes_$GOOSARCH.go";
+		echo "$mktypes types_$GOOS.go | go run mkpost.go >>ztypes_$GOOSARCH.go";
+	fi
+) | $run

vendor/golang.org/x/sys/unix/mkerrors.sh

@@ -0,0 +1,483 @@
+#!/usr/bin/env bash
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# Generate Go code listing errors and other #defined constant
+# values (ENAMETOOLONG etc.), by asking the preprocessor
+# about the definitions.
+
+unset LANG
+export LC_ALL=C
+export LC_CTYPE=C
+
+if test -z "$GOARCH" -o -z "$GOOS"; then
+	echo 1>&2 "GOARCH or GOOS not defined in environment"
+	exit 1
+fi
+
+CC=${CC:-cc}
+
+if [[ "$GOOS" -eq "solaris" ]]; then
+	# Assumes GNU versions of utilities in PATH.
+	export PATH=/usr/gnu/bin:$PATH
+fi
+
+uname=$(uname)
+
+includes_Darwin='
+#define _DARWIN_C_SOURCE
+#define KERNEL
+#define _DARWIN_USE_64_BIT_INODE
+#include <sys/types.h>
+#include <sys/event.h>
+#include <sys/ptrace.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_types.h>
+#include <net/route.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <termios.h>
+'
+
+includes_DragonFly='
+#include <sys/types.h>
+#include <sys/event.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sys/ioctl.h>
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_types.h>
+#include <net/route.h>
+#include <netinet/in.h>
+#include <termios.h>
+#include <netinet/ip.h>
+#include <net/ip_mroute/ip_mroute.h>
+'
+
+includes_FreeBSD='
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/event.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sys/ioctl.h>
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_types.h>
+#include <net/route.h>
+#include <netinet/in.h>
+#include <termios.h>
+#include <netinet/ip.h>
+#include <netinet/ip_mroute.h>
+#include <sys/extattr.h>
+
+#if __FreeBSD__ >= 10
+#define IFT_CARP	0xf8	// IFT_CARP is deprecated in FreeBSD 10
+#undef SIOCAIFADDR
+#define SIOCAIFADDR	_IOW(105, 26, struct oifaliasreq)	// ifaliasreq contains if_data
+#undef SIOCSIFPHYADDR
+#define SIOCSIFPHYADDR	_IOW(105, 70, struct oifaliasreq)	// ifaliasreq contains if_data
+#endif
+'
+
+includes_Linux='
+#define _LARGEFILE_SOURCE
+#define _LARGEFILE64_SOURCE
+#ifndef __LP64__
+#define _FILE_OFFSET_BITS 64
+#endif
+#define _GNU_SOURCE
+
+#include <bits/sockaddr.h>
+#include <sys/epoll.h>
+#include <sys/inotify.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/mount.h>
+#include <sys/prctl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/socket.h>
+#include <linux/if.h>
+#include <linux/if_arp.h>
+#include <linux/if_ether.h>
+#include <linux/if_tun.h>
+#include <linux/if_packet.h>
+#include <linux/if_addr.h>
+#include <linux/filter.h>
+#include <linux/netlink.h>
+#include <linux/reboot.h>
+#include <linux/rtnetlink.h>
+#include <linux/ptrace.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/icmpv6.h>
+#include <linux/serial.h>
+#include <net/route.h>
+#include <asm/termbits.h>
+
+#ifndef MSG_FASTOPEN
+#define MSG_FASTOPEN    0x20000000
+#endif
+
+#ifndef PTRACE_GETREGS
+#define PTRACE_GETREGS	0xc
+#endif
+
+#ifndef PTRACE_SETREGS
+#define PTRACE_SETREGS	0xd
+#endif
+
+#ifdef SOL_BLUETOOTH
+// SPARC includes this in /usr/include/sparc64-linux-gnu/bits/socket.h
+// but it is already in bluetooth_linux.go
+#undef SOL_BLUETOOTH
+#endif
+'
+
+includes_NetBSD='
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/event.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/termios.h>
+#include <sys/ttycom.h>
+#include <sys/wait.h>
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_types.h>
+#include <net/route.h>
+#include <netinet/in.h>
+#include <netinet/in_systm.h>
+#include <netinet/ip.h>
+#include <netinet/ip_mroute.h>
+#include <netinet/if_ether.h>
+
+// Needed since <sys/param.h> refers to it...
+#define schedppq 1
+'
+
+includes_OpenBSD='
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/event.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/termios.h>
+#include <sys/ttycom.h>
+#include <sys/wait.h>
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_types.h>
+#include <net/if_var.h>
+#include <net/route.h>
+#include <netinet/in.h>
+#include <netinet/in_systm.h>
+#include <netinet/ip.h>
+#include <netinet/ip_mroute.h>
+#include <netinet/if_ether.h>
+#include <net/if_bridge.h>
+
+// We keep some constants not supported in OpenBSD 5.5 and beyond for
+// the promise of compatibility.
+#define EMUL_ENABLED		0x1
+#define EMUL_NATIVE		0x2
+#define IPV6_FAITH		0x1d
+#define IPV6_OPTIONS		0x1
+#define IPV6_RTHDR_STRICT	0x1
+#define IPV6_SOCKOPT_RESERVED1	0x3
+#define SIOCGIFGENERIC		0xc020693a
+#define SIOCSIFGENERIC		0x80206939
+#define WALTSIG			0x4
+'
+
+includes_SunOS='
+#include <limits.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/mman.h>
+#include <sys/wait.h>
+#include <sys/ioctl.h>
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_arp.h>
+#include <net/if_types.h>
+#include <net/route.h>
+#include <netinet/in.h>
+#include <termios.h>
+#include <netinet/ip.h>
+#include <netinet/ip_mroute.h>
+'
+
+
+includes='
+#include <sys/types.h>
+#include <sys/file.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+#include <errno.h>
+#include <sys/signal.h>
+#include <signal.h>
+#include <sys/resource.h>
+#include <time.h>
+'
+ccflags="$@"
+
+# Write go tool cgo -godefs input.
+(
+	echo package unix
+	echo
+	echo '/*'
+	indirect="includes_$(uname)"
+	echo "${!indirect} $includes"
+	echo '*/'
+	echo 'import "C"'
+	echo 'import "syscall"'
+	echo
+	echo 'const ('
+
+	# The gcc command line prints all the #defines
+	# it encounters while processing the input
+	echo "${!indirect} $includes" | $CC -x c - -E -dM $ccflags |
+	awk '
+		$1 != "#define" || $2 ~ /\(/ || $3 == "" {next}
+
+		$2 ~ /^E([ABCD]X|[BIS]P|[SD]I|S|FL)$/ {next}  # 386 registers
+		$2 ~ /^(SIGEV_|SIGSTKSZ|SIGRT(MIN|MAX))/ {next}
+		$2 ~ /^(SCM_SRCRT)$/ {next}
+		$2 ~ /^(MAP_FAILED)$/ {next}
+		$2 ~ /^ELF_.*$/ {next}# <asm/elf.h> contains ELF_ARCH, etc.
+
+		$2 ~ /^EXTATTR_NAMESPACE_NAMES/ ||
+		$2 ~ /^EXTATTR_NAMESPACE_[A-Z]+_STRING/ {next}
+
+		$2 !~ /^ETH_/ &&
+		$2 !~ /^EPROC_/ &&
+		$2 !~ /^EQUIV_/ &&
+		$2 !~ /^EXPR_/ &&
+		$2 ~ /^E[A-Z0-9_]+$/ ||
+		$2 ~ /^B[0-9_]+$/ ||
+		$2 == "BOTHER" ||
+		$2 ~ /^CI?BAUD(EX)?$/ ||
+		$2 == "IBSHIFT" ||
+		$2 ~ /^V[A-Z0-9]+$/ ||
+		$2 ~ /^CS[A-Z0-9]/ ||
+		$2 ~ /^I(SIG|CANON|CRNL|UCLC|EXTEN|MAXBEL|STRIP|UTF8)$/ ||
+		$2 ~ /^IGN/ ||
+		$2 ~ /^IX(ON|ANY|OFF)$/ ||
+		$2 ~ /^IN(LCR|PCK)$/ ||
+		$2 ~ /(^FLU?SH)|(FLU?SH$)/ ||
+		$2 ~ /^C(LOCAL|READ|MSPAR|RTSCTS)$/ ||
+		$2 == "BRKINT" ||
+		$2 == "HUPCL" ||
+		$2 == "PENDIN" ||
+		$2 == "TOSTOP" ||
+		$2 == "XCASE" ||
+		$2 == "ALTWERASE" ||
+		$2 == "NOKERNINFO" ||
+		$2 ~ /^PAR/ ||
+		$2 ~ /^SIG[^_]/ ||
+		$2 ~ /^O[CNPFPL][A-Z]+[^_][A-Z]+$/ ||
+		$2 ~ /^(NL|CR|TAB|BS|VT|FF)DLY$/ ||
+		$2 ~ /^(NL|CR|TAB|BS|VT|FF)[0-9]$/ ||
+		$2 ~ /^O?XTABS$/ ||
+		$2 ~ /^TC[IO](ON|OFF)$/ ||
+		$2 ~ /^IN_/ ||
+		$2 ~ /^LOCK_(SH|EX|NB|UN)$/ ||
+		$2 ~ /^(AF|SOCK|SO|SOL|IPPROTO|IP|IPV6|ICMP6|TCP|EVFILT|NOTE|EV|SHUT|PROT|MAP|PACKET|MSG|SCM|MCL|DT|MADV|PR)_/ ||
+		$2 == "ICMPV6_FILTER" ||
+		$2 == "SOMAXCONN" ||
+		$2 == "NAME_MAX" ||
+		$2 == "IFNAMSIZ" ||
+		$2 ~ /^CTL_(MAXNAME|NET|QUERY)$/ ||
+		$2 ~ /^SYSCTL_VERS/ ||
+		$2 ~ /^(MS|MNT)_/ ||
+		$2 ~ /^TUN(SET|GET|ATTACH|DETACH)/ ||
+		$2 ~ /^(O|F|FD|NAME|S|PTRACE|PT)_/ ||
+		$2 ~ /^LINUX_REBOOT_CMD_/ ||
+		$2 ~ /^LINUX_REBOOT_MAGIC[12]$/ ||
+		$2 !~ "NLA_TYPE_MASK" &&
+		$2 ~ /^(NETLINK|NLM|NLMSG|NLA|IFA|IFAN|RT|RTCF|RTN|RTPROT|RTNH|ARPHRD|ETH_P)_/ ||
+		$2 ~ /^SIOC/ ||
+		$2 ~ /^TIOC/ ||
+		$2 ~ /^TCGET/ ||
+		$2 ~ /^TCSET/ ||
+		$2 ~ /^TC(FLSH|SBRKP?|XONC)$/ ||
+		$2 !~ "RTF_BITS" &&
+		$2 ~ /^(IFF|IFT|NET_RT|RTM|RTF|RTV|RTA|RTAX)_/ ||
+		$2 ~ /^BIOC/ ||
+		$2 ~ /^RUSAGE_(SELF|CHILDREN|THREAD)/ ||
+		$2 ~ /^RLIMIT_(AS|CORE|CPU|DATA|FSIZE|NOFILE|STACK)|RLIM_INFINITY/ ||
+		$2 ~ /^PRIO_(PROCESS|PGRP|USER)/ ||
+		$2 ~ /^CLONE_[A-Z_]+/ ||
+		$2 !~ /^(BPF_TIMEVAL)$/ &&
+		$2 ~ /^(BPF|DLT)_/ ||
+		$2 ~ /^CLOCK_/ ||
+		$2 !~ "WMESGLEN" &&
+		$2 ~ /^W[A-Z0-9]+$/ {printf("\t%s = C.%s\n", $2, $2)}
+		$2 ~ /^__WCOREFLAG$/ {next}
+		$2 ~ /^__W[A-Z0-9]+$/ {printf("\t%s = C.%s\n", substr($2,3), $2)}
+
+		{next}
+	' | sort
+
+	echo ')'
+) >_const.go
+
+# Pull out the error names for later.
+errors=$(
+	echo '#include <errno.h>' | $CC -x c - -E -dM $ccflags |
+	awk '$1=="#define" && $2 ~ /^E[A-Z0-9_]+$/ { print $2 }' |
+	sort
+)
+
+# Pull out the signal names for later.
+signals=$(
+	echo '#include <signal.h>' | $CC -x c - -E -dM $ccflags |
+	awk '$1=="#define" && $2 ~ /^SIG[A-Z0-9]+$/ { print $2 }' |
+	egrep -v '(SIGSTKSIZE|SIGSTKSZ|SIGRT)' |
+	sort
+)
+
+# Again, writing regexps to a file.
+echo '#include <errno.h>' | $CC -x c - -E -dM $ccflags |
+	awk '$1=="#define" && $2 ~ /^E[A-Z0-9_]+$/ { print "^\t" $2 "[ \t]*=" }' |
+	sort >_error.grep
+echo '#include <signal.h>' | $CC -x c - -E -dM $ccflags |
+	awk '$1=="#define" && $2 ~ /^SIG[A-Z0-9]+$/ { print "^\t" $2 "[ \t]*=" }' |
+	egrep -v '(SIGSTKSIZE|SIGSTKSZ|SIGRT)' |
+	sort >_signal.grep
+
+echo '// mkerrors.sh' "$@"
+echo '// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT'
+echo
+echo "// +build ${GOARCH},${GOOS}"
+echo
+go tool cgo -godefs -- "$@" _const.go >_error.out
+cat _error.out | grep -vf _error.grep | grep -vf _signal.grep
+echo
+echo '// Errors'
+echo 'const ('
+cat _error.out | grep -f _error.grep | sed 's/=\(.*\)/= syscall.Errno(\1)/'
+echo ')'
+
+echo
+echo '// Signals'
+echo 'const ('
+cat _error.out | grep -f _signal.grep | sed 's/=\(.*\)/= syscall.Signal(\1)/'
+echo ')'
+
+# Run C program to print error and syscall strings.
+(
+	echo -E "
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <ctype.h>
+#include <string.h>
+#include <signal.h>
+
+#define nelem(x) (sizeof(x)/sizeof((x)[0]))
+
+enum { A = 'A', Z = 'Z', a = 'a', z = 'z' }; // avoid need for single quotes below
+
+int errors[] = {
+"
+	for i in $errors
+	do
+		echo -E '	'$i,
+	done
+
+	echo -E "
+};
+
+int signals[] = {
+"
+	for i in $signals
+	do
+		echo -E '	'$i,
+	done
+
+	# Use -E because on some systems bash builtin interprets \n itself.
+	echo -E '
+};
+
+static int
+intcmp(const void *a, const void *b)
+{
+	return *(int*)a - *(int*)b;
+}
+
+int
+main(void)
+{
+	int i, j, e;
+	char buf[1024], *p;
+
+	printf("\n\n// Error table\n");
+	printf("var errors = [...]string {\n");
+	qsort(errors, nelem(errors), sizeof errors[0], intcmp);
+	for(i=0; i<nelem(errors); i++) {
+		e = errors[i];
+		if(i > 0 && errors[i-1] == e)
+			continue;
+		strcpy(buf, strerror(e));
+		// lowercase first letter: Bad -> bad, but STREAM -> STREAM.
+		if(A <= buf[0] && buf[0] <= Z && a <= buf[1] && buf[1] <= z)
+			buf[0] += a - A;
+		printf("\t%d: \"%s\",\n", e, buf);
+	}
+	printf("}\n\n");
+	
+	printf("\n\n// Signal table\n");
+	printf("var signals = [...]string {\n");
+	qsort(signals, nelem(signals), sizeof signals[0], intcmp);
+	for(i=0; i<nelem(signals); i++) {
+		e = signals[i];
+		if(i > 0 && signals[i-1] == e)
+			continue;
+		strcpy(buf, strsignal(e));
+		// lowercase first letter: Bad -> bad, but STREAM -> STREAM.
+		if(A <= buf[0] && buf[0] <= Z && a <= buf[1] && buf[1] <= z)
+			buf[0] += a - A;
+		// cut trailing : number.
+		p = strrchr(buf, ":"[0]);
+		if(p)
+			*p = '\0';
+		printf("\t%d: \"%s\",\n", e, buf);
+	}
+	printf("}\n\n");
+
+	return 0;
+}
+
+'
+) >_errors.c
+
+$CC $ccflags -o _errors _errors.c && $GORUN ./_errors && rm -f _errors.c _errors _const.go _error.grep _signal.grep _error.out

vendor/golang.org/x/sys/unix/mkpost.go

@@ -0,0 +1,62 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// mkpost processes the output of cgo -godefs to
+// modify the generated types. It is used to clean up
+// the sys API in an architecture specific manner.
+//
+// mkpost is run after cgo -godefs by mkall.sh.
+package main
+
+import (
+	"fmt"
+	"go/format"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+)
+
+func main() {
+	b, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		log.Fatal(err)
+	}
+	s := string(b)
+
+	goarch := os.Getenv("GOARCH")
+	goos := os.Getenv("GOOS")
+	if goarch == "s390x" && goos == "linux" {
+		// Export the types of PtraceRegs fields.
+		re := regexp.MustCompile("ptrace(Psw|Fpregs|Per)")
+		s = re.ReplaceAllString(s, "Ptrace$1")
+
+		// Replace padding fields inserted by cgo with blank identifiers.
+		re = regexp.MustCompile("Pad_cgo[A-Za-z0-9_]*")
+		s = re.ReplaceAllString(s, "_")
+
+		// Replace other unwanted fields with blank identifiers.
+		re = regexp.MustCompile("X_[A-Za-z0-9_]*")
+		s = re.ReplaceAllString(s, "_")
+
+		// Replace the control_regs union with a blank identifier for now.
+		re = regexp.MustCompile("(Control_regs)\\s+\\[0\\]uint64")
+		s = re.ReplaceAllString(s, "_ [0]uint64")
+	}
+
+	// gofmt
+	b, err = format.Source([]byte(s))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Append this command to the header to show where the new file
+	// came from.
+	re := regexp.MustCompile("(cgo -godefs [a-zA-Z0-9_]+\\.go.*)")
+	b = re.ReplaceAll(b, []byte("$1 | go run mkpost.go"))
+
+	fmt.Printf("%s", b)
+}

vendor/golang.org/x/sys/unix/mksyscall.pl

@@ -0,0 +1,323 @@
+#!/usr/bin/env perl
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This program reads a file containing function prototypes
+# (like syscall_darwin.go) and generates system call bodies.
+# The prototypes are marked by lines beginning with "//sys"
+# and read like func declarations if //sys is replaced by func, but:
+#	* The parameter lists must give a name for each argument.
+#	  This includes return parameters.
+#	* The parameter lists must give a type for each argument:
+#	  the (x, y, z int) shorthand is not allowed.
+#	* If the return parameter is an error number, it must be named errno.
+
+# A line beginning with //sysnb is like //sys, except that the
+# goroutine will not be suspended during the execution of the system
+# call.  This must only be used for system calls which can never
+# block, as otherwise the system call could cause all goroutines to
+# hang.
+
+use strict;
+
+my $cmdline = "mksyscall.pl " . join(' ', @ARGV);
+my $errors = 0;
+my $_32bit = "";
+my $plan9 = 0;
+my $openbsd = 0;
+my $netbsd = 0;
+my $dragonfly = 0;
+my $arm = 0; # 64-bit value should use (even, odd)-pair
+
+if($ARGV[0] eq "-b32") {
+	$_32bit = "big-endian";
+	shift;
+} elsif($ARGV[0] eq "-l32") {
+	$_32bit = "little-endian";
+	shift;
+}
+if($ARGV[0] eq "-plan9") {
+	$plan9 = 1;
+	shift;
+}
+if($ARGV[0] eq "-openbsd") {
+	$openbsd = 1;
+	shift;
+}
+if($ARGV[0] eq "-netbsd") {
+	$netbsd = 1;
+	shift;
+}
+if($ARGV[0] eq "-dragonfly") {
+	$dragonfly = 1;
+	shift;
+}
+if($ARGV[0] eq "-arm") {
+	$arm = 1;
+	shift;
+}
+
+if($ARGV[0] =~ /^-/) {
+	print STDERR "usage: mksyscall.pl [-b32 | -l32] [file ...]\n";
+	exit 1;
+}
+
+if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
+	print STDERR "GOARCH or GOOS not defined in environment\n";
+	exit 1;
+}
+
+sub parseparamlist($) {
+	my ($list) = @_;
+	$list =~ s/^\s*//;
+	$list =~ s/\s*$//;
+	if($list eq "") {
+		return ();
+	}
+	return split(/\s*,\s*/, $list);
+}
+
+sub parseparam($) {
+	my ($p) = @_;
+	if($p !~ /^(\S*) (\S*)$/) {
+		print STDERR "$ARGV:$.: malformed parameter: $p\n";
+		$errors = 1;
+		return ("xx", "int");
+	}
+	return ($1, $2);
+}
+
+my $text = "";
+while(<>) {
+	chomp;
+	s/\s+/ /g;
+	s/^\s+//;
+	s/\s+$//;
+	my $nonblock = /^\/\/sysnb /;
+	next if !/^\/\/sys / && !$nonblock;
+
+	# Line must be of the form
+	#	func Open(path string, mode int, perm int) (fd int, errno error)
+	# Split into name, in params, out params.
+	if(!/^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*((?i)SYS_[A-Z0-9_]+))?$/) {
+		print STDERR "$ARGV:$.: malformed //sys declaration\n";
+		$errors = 1;
+		next;
+	}
+	my ($func, $in, $out, $sysname) = ($2, $3, $4, $5);
+
+	# Split argument lists on comma.
+	my @in = parseparamlist($in);
+	my @out = parseparamlist($out);
+
+	# Try in vain to keep people from editing this file.
+	# The theory is that they jump into the middle of the file
+	# without reading the header.
+	$text .= "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n";
+
+	# Go function header.
+	my $out_decl = @out ? sprintf(" (%s)", join(', ', @out)) : "";
+	$text .= sprintf "func %s(%s)%s {\n", $func, join(', ', @in), $out_decl;
+
+	# Check if err return available
+	my $errvar = "";
+	foreach my $p (@out) {
+		my ($name, $type) = parseparam($p);
+		if($type eq "error") {
+			$errvar = $name;
+			last;
+		}
+	}
+
+	# Prepare arguments to Syscall.
+	my @args = ();
+	my @uses = ();
+	my $n = 0;
+	foreach my $p (@in) {
+		my ($name, $type) = parseparam($p);
+		if($type =~ /^\*/) {
+			push @args, "uintptr(unsafe.Pointer($name))";
+		} elsif($type eq "string" && $errvar ne "") {
+			$text .= "\tvar _p$n *byte\n";
+			$text .= "\t_p$n, $errvar = BytePtrFromString($name)\n";
+			$text .= "\tif $errvar != nil {\n\t\treturn\n\t}\n";
+			push @args, "uintptr(unsafe.Pointer(_p$n))";
+			push @uses, "use(unsafe.Pointer(_p$n))";
+			$n++;
+		} elsif($type eq "string") {
+			print STDERR "$ARGV:$.: $func uses string arguments, but has no error return\n";
+			$text .= "\tvar _p$n *byte\n";
+			$text .= "\t_p$n, _ = BytePtrFromString($name)\n";
+			push @args, "uintptr(unsafe.Pointer(_p$n))";
+			push @uses, "use(unsafe.Pointer(_p$n))";
+			$n++;
+		} elsif($type =~ /^\[\](.*)/) {
+			# Convert slice into pointer, length.
+			# Have to be careful not to take address of &a[0] if len == 0:
+			# pass dummy pointer in that case.
+			# Used to pass nil, but some OSes or simulators reject write(fd, nil, 0).
+			$text .= "\tvar _p$n unsafe.Pointer\n";
+			$text .= "\tif len($name) > 0 {\n\t\t_p$n = unsafe.Pointer(\&${name}[0])\n\t}";
+			$text .= " else {\n\t\t_p$n = unsafe.Pointer(&_zero)\n\t}";
+			$text .= "\n";
+			push @args, "uintptr(_p$n)", "uintptr(len($name))";
+			$n++;
+		} elsif($type eq "int64" && ($openbsd || $netbsd)) {
+			push @args, "0";
+			if($_32bit eq "big-endian") {
+				push @args, "uintptr($name>>32)", "uintptr($name)";
+			} elsif($_32bit eq "little-endian") {
+				push @args, "uintptr($name)", "uintptr($name>>32)";
+			} else {
+				push @args, "uintptr($name)";
+			}
+		} elsif($type eq "int64" && $dragonfly) {
+			if ($func !~ /^extp(read|write)/i) {
+				push @args, "0";
+			}
+			if($_32bit eq "big-endian") {
+				push @args, "uintptr($name>>32)", "uintptr($name)";
+			} elsif($_32bit eq "little-endian") {
+				push @args, "uintptr($name)", "uintptr($name>>32)";
+			} else {
+				push @args, "uintptr($name)";
+			}
+		} elsif($type eq "int64" && $_32bit ne "") {
+			if(@args % 2 && $arm) {
+				# arm abi specifies 64-bit argument uses 
+				# (even, odd) pair
+				push @args, "0"
+			}
+			if($_32bit eq "big-endian") {
+				push @args, "uintptr($name>>32)", "uintptr($name)";
+			} else {
+				push @args, "uintptr($name)", "uintptr($name>>32)";
+			}
+		} else {
+			push @args, "uintptr($name)";
+		}
+	}
+
+	# Determine which form to use; pad args with zeros.
+	my $asm = "Syscall";
+	if ($nonblock) {
+		$asm = "RawSyscall";
+	}
+	if(@args <= 3) {
+		while(@args < 3) {
+			push @args, "0";
+		}
+	} elsif(@args <= 6) {
+		$asm .= "6";
+		while(@args < 6) {
+			push @args, "0";
+		}
+	} elsif(@args <= 9) {
+		$asm .= "9";
+		while(@args < 9) {
+			push @args, "0";
+		}
+	} else {
+		print STDERR "$ARGV:$.: too many arguments to system call\n";
+	}
+
+	# System call number.
+	if($sysname eq "") {
+		$sysname = "SYS_$func";
+		$sysname =~ s/([a-z])([A-Z])/${1}_$2/g;	# turn FooBar into Foo_Bar
+		$sysname =~ y/a-z/A-Z/;
+	}
+
+	# Actual call.
+	my $args = join(', ', @args);
+	my $call = "$asm($sysname, $args)";
+
+	# Assign return values.
+	my $body = "";
+	my @ret = ("_", "_", "_");
+	my $do_errno = 0;
+	for(my $i=0; $i<@out; $i++) {
+		my $p = $out[$i];
+		my ($name, $type) = parseparam($p);
+		my $reg = "";
+		if($name eq "err" && !$plan9) {
+			$reg = "e1";
+			$ret[2] = $reg;
+			$do_errno = 1;
+		} elsif($name eq "err" && $plan9) {
+			$ret[0] = "r0";
+			$ret[2] = "e1";
+			next;
+		} else {
+			$reg = sprintf("r%d", $i);
+			$ret[$i] = $reg;
+		}
+		if($type eq "bool") {
+			$reg = "$reg != 0";
+		}
+		if($type eq "int64" && $_32bit ne "") {
+			# 64-bit number in r1:r0 or r0:r1.
+			if($i+2 > @out) {
+				print STDERR "$ARGV:$.: not enough registers for int64 return\n";
+			}
+			if($_32bit eq "big-endian") {
+				$reg = sprintf("int64(r%d)<<32 | int64(r%d)", $i, $i+1);
+			} else {
+				$reg = sprintf("int64(r%d)<<32 | int64(r%d)", $i+1, $i);
+			}
+			$ret[$i] = sprintf("r%d", $i);
+			$ret[$i+1] = sprintf("r%d", $i+1);
+		}
+		if($reg ne "e1" || $plan9) {
+			$body .= "\t$name = $type($reg)\n";
+		}
+	}
+	if ($ret[0] eq "_" && $ret[1] eq "_" && $ret[2] eq "_") {
+		$text .= "\t$call\n";
+	} else {
+		$text .= "\t$ret[0], $ret[1], $ret[2] := $call\n";
+	}
+	foreach my $use (@uses) {
+		$text .= "\t$use\n";
+	}
+	$text .= $body;
+	
+	if ($plan9 && $ret[2] eq "e1") {
+		$text .= "\tif int32(r0) == -1 {\n";
+		$text .= "\t\terr = e1\n";
+		$text .= "\t}\n";
+	} elsif ($do_errno) {
+		$text .= "\tif e1 != 0 {\n";
+		$text .= "\t\terr = errnoErr(e1)\n";
+		$text .= "\t}\n";
+	}
+	$text .= "\treturn\n";
+	$text .= "}\n\n";
+}
+
+chomp $text;
+chomp $text;
+
+if($errors) {
+	exit 1;
+}
+
+print <<EOF;
+// $cmdline
+// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
+
+// +build $ENV{'GOARCH'},$ENV{'GOOS'}
+
+package unix
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+var _ syscall.Errno
+
+$text
+EOF
+exit 0;

vendor/golang.org/x/sys/unix/mksyscall_solaris.pl

@@ -0,0 +1,294 @@
+#!/usr/bin/env perl
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This program reads a file containing function prototypes
+# (like syscall_solaris.go) and generates system call bodies.
+# The prototypes are marked by lines beginning with "//sys"
+# and read like func declarations if //sys is replaced by func, but:
+#	* The parameter lists must give a name for each argument.
+#	  This includes return parameters.
+#	* The parameter lists must give a type for each argument:
+#	  the (x, y, z int) shorthand is not allowed.
+#	* If the return parameter is an error number, it must be named err.
+#	* If go func name needs to be different than its libc name, 
+#	* or the function is not in libc, name could be specified
+#	* at the end, after "=" sign, like
+#	  //sys getsockopt(s int, level int, name int, val uintptr, vallen *_Socklen) (err error) = libsocket.getsockopt
+
+use strict;
+
+my $cmdline = "mksyscall_solaris.pl " . join(' ', @ARGV);
+my $errors = 0;
+my $_32bit = "";
+
+binmode STDOUT;
+
+if($ARGV[0] eq "-b32") {
+	$_32bit = "big-endian";
+	shift;
+} elsif($ARGV[0] eq "-l32") {
+	$_32bit = "little-endian";
+	shift;
+}
+
+if($ARGV[0] =~ /^-/) {
+	print STDERR "usage: mksyscall_solaris.pl [-b32 | -l32] [file ...]\n";
+	exit 1;
+}
+
+if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
+	print STDERR "GOARCH or GOOS not defined in environment\n";
+	exit 1;
+}
+
+sub parseparamlist($) {
+	my ($list) = @_;
+	$list =~ s/^\s*//;
+	$list =~ s/\s*$//;
+	if($list eq "") {
+		return ();
+	}
+	return split(/\s*,\s*/, $list);
+}
+
+sub parseparam($) {
+	my ($p) = @_;
+	if($p !~ /^(\S*) (\S*)$/) {
+		print STDERR "$ARGV:$.: malformed parameter: $p\n";
+		$errors = 1;
+		return ("xx", "int");
+	}
+	return ($1, $2);
+}
+
+my $package = "";
+my $text = "";
+my $dynimports = "";
+my $linknames = "";
+my @vars = ();
+while(<>) {
+	chomp;
+	s/\s+/ /g;
+	s/^\s+//;
+	s/\s+$//;
+	$package = $1 if !$package && /^package (\S+)$/;
+	my $nonblock = /^\/\/sysnb /;
+	next if !/^\/\/sys / && !$nonblock;
+
+	# Line must be of the form
+	#	func Open(path string, mode int, perm int) (fd int, err error)
+	# Split into name, in params, out params.
+	if(!/^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*(?:(\w*)\.)?(\w*))?$/) {
+		print STDERR "$ARGV:$.: malformed //sys declaration\n";
+		$errors = 1;
+		next;
+	}
+	my ($nb, $func, $in, $out, $modname, $sysname) = ($1, $2, $3, $4, $5, $6);
+
+	# Split argument lists on comma.
+	my @in = parseparamlist($in);
+	my @out = parseparamlist($out);
+
+	# So file name.
+	if($modname eq "") {
+		$modname = "libc";
+	}
+
+	# System call name.
+	if($sysname eq "") {
+		$sysname = "$func";
+	}
+
+	# System call pointer variable name.
+	my $sysvarname = "proc$sysname";
+
+	my $strconvfunc = "BytePtrFromString";
+	my $strconvtype = "*byte";
+
+	$sysname =~ y/A-Z/a-z/; # All libc functions are lowercase.
+
+	# Runtime import of function to allow cross-platform builds.
+	$dynimports .= "//go:cgo_import_dynamic libc_${sysname} ${sysname} \"$modname.so\"\n";
+	# Link symbol to proc address variable.
+	$linknames .= "//go:linkname ${sysvarname} libc_${sysname}\n";
+	# Library proc address variable.
+	push @vars, $sysvarname;
+
+	# Go function header.
+	$out = join(', ', @out);
+	if($out ne "") {
+		$out = " ($out)";
+	}
+	if($text ne "") {
+		$text .= "\n"
+	}
+	$text .= sprintf "func %s(%s)%s {\n", $func, join(', ', @in), $out;
+
+	# Check if err return available
+	my $errvar = "";
+	foreach my $p (@out) {
+		my ($name, $type) = parseparam($p);
+		if($type eq "error") {
+			$errvar = $name;
+			last;
+		}
+	}
+
+	# Prepare arguments to Syscall.
+	my @args = ();
+	my @uses = ();
+	my $n = 0;
+	foreach my $p (@in) {
+		my ($name, $type) = parseparam($p);
+		if($type =~ /^\*/) {
+			push @args, "uintptr(unsafe.Pointer($name))";
+		} elsif($type eq "string" && $errvar ne "") {
+			$text .= "\tvar _p$n $strconvtype\n";
+			$text .= "\t_p$n, $errvar = $strconvfunc($name)\n";
+			$text .= "\tif $errvar != nil {\n\t\treturn\n\t}\n";
+			push @args, "uintptr(unsafe.Pointer(_p$n))";
+			push @uses, "use(unsafe.Pointer(_p$n))";
+			$n++;
+		} elsif($type eq "string") {
+			print STDERR "$ARGV:$.: $func uses string arguments, but has no error return\n";
+			$text .= "\tvar _p$n $strconvtype\n";
+			$text .= "\t_p$n, _ = $strconvfunc($name)\n";
+			push @args, "uintptr(unsafe.Pointer(_p$n))";
+			push @uses, "use(unsafe.Pointer(_p$n))";
+			$n++;
+		} elsif($type =~ /^\[\](.*)/) {
+			# Convert slice into pointer, length.
+			# Have to be careful not to take address of &a[0] if len == 0:
+			# pass nil in that case.
+			$text .= "\tvar _p$n *$1\n";
+			$text .= "\tif len($name) > 0 {\n\t\t_p$n = \&$name\[0]\n\t}\n";
+			push @args, "uintptr(unsafe.Pointer(_p$n))", "uintptr(len($name))";
+			$n++;
+		} elsif($type eq "int64" && $_32bit ne "") {
+			if($_32bit eq "big-endian") {
+				push @args, "uintptr($name >> 32)", "uintptr($name)";
+			} else {
+				push @args, "uintptr($name)", "uintptr($name >> 32)";
+			}
+		} elsif($type eq "bool") {
+ 			$text .= "\tvar _p$n uint32\n";
+			$text .= "\tif $name {\n\t\t_p$n = 1\n\t} else {\n\t\t_p$n = 0\n\t}\n";
+			push @args, "uintptr(_p$n)";
+			$n++;
+		} else {
+			push @args, "uintptr($name)";
+		}
+	}
+	my $nargs = @args;
+
+	# Determine which form to use; pad args with zeros.
+	my $asm = "sysvicall6";
+	if ($nonblock) {
+		$asm = "rawSysvicall6";
+	}
+	if(@args <= 6) {
+		while(@args < 6) {
+			push @args, "0";
+		}
+	} else {
+		print STDERR "$ARGV:$.: too many arguments to system call\n";
+	}
+
+	# Actual call.
+	my $args = join(', ', @args);
+	my $call = "$asm(uintptr(unsafe.Pointer(&$sysvarname)), $nargs, $args)";
+
+	# Assign return values.
+	my $body = "";
+	my $failexpr = "";
+	my @ret = ("_", "_", "_");
+	my @pout= ();
+	my $do_errno = 0;
+	for(my $i=0; $i<@out; $i++) {
+		my $p = $out[$i];
+		my ($name, $type) = parseparam($p);
+		my $reg = "";
+		if($name eq "err") {
+			$reg = "e1";
+			$ret[2] = $reg;
+			$do_errno = 1;
+		} else {
+			$reg = sprintf("r%d", $i);
+			$ret[$i] = $reg;
+		}
+		if($type eq "bool") {
+			$reg = "$reg != 0";
+		}
+		if($type eq "int64" && $_32bit ne "") {
+			# 64-bit number in r1:r0 or r0:r1.
+			if($i+2 > @out) {
+				print STDERR "$ARGV:$.: not enough registers for int64 return\n";
+			}
+			if($_32bit eq "big-endian") {
+				$reg = sprintf("int64(r%d)<<32 | int64(r%d)", $i, $i+1);
+			} else {
+				$reg = sprintf("int64(r%d)<<32 | int64(r%d)", $i+1, $i);
+			}
+			$ret[$i] = sprintf("r%d", $i);
+			$ret[$i+1] = sprintf("r%d", $i+1);
+		}
+		if($reg ne "e1") {
+			$body .= "\t$name = $type($reg)\n";
+		}
+	}
+	if ($ret[0] eq "_" && $ret[1] eq "_" && $ret[2] eq "_") {
+		$text .= "\t$call\n";
+	} else {
+		$text .= "\t$ret[0], $ret[1], $ret[2] := $call\n";
+	}
+	foreach my $use (@uses) {
+		$text .= "\t$use\n";
+	}
+	$text .= $body;
+
+	if ($do_errno) {
+		$text .= "\tif e1 != 0 {\n";
+		$text .= "\t\terr = e1\n";
+		$text .= "\t}\n";
+	}
+	$text .= "\treturn\n";
+	$text .= "}\n";
+}
+
+if($errors) {
+	exit 1;
+}
+
+print <<EOF;
+// $cmdline
+// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
+
+// +build $ENV{'GOARCH'},$ENV{'GOOS'}
+
+package $package
+
+import (
+	"syscall"
+	"unsafe"
+)
+EOF
+
+print "import \"golang.org/x/sys/unix\"\n" if $package ne "unix";
+
+my $vardecls = "\t" . join(",\n\t", @vars);
+$vardecls .= " syscallFunc";
+
+chomp($_=<<EOF);
+
+$dynimports
+$linknames
+var (
+$vardecls
+)
+
+$text
+EOF
+print $_;
+exit 0;

vendor/golang.org/x/sys/unix/mksysctl_openbsd.pl

@@ -0,0 +1,264 @@
+#!/usr/bin/env perl
+
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+#
+# Parse the header files for OpenBSD and generate a Go usable sysctl MIB.
+#
+# Build a MIB with each entry being an array containing the level, type and
+# a hash that will contain additional entries if the current entry is a node.
+# We then walk this MIB and create a flattened sysctl name to OID hash.
+#
+
+use strict;
+
+if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
+	print STDERR "GOARCH or GOOS not defined in environment\n";
+	exit 1;
+}
+
+my $debug = 0;
+my %ctls = ();
+
+my @headers = qw (
+	sys/sysctl.h
+	sys/socket.h
+	sys/tty.h
+	sys/malloc.h
+	sys/mount.h
+	sys/namei.h
+	sys/sem.h
+	sys/shm.h
+	sys/vmmeter.h
+	uvm/uvm_param.h
+	uvm/uvm_swap_encrypt.h
+	ddb/db_var.h
+	net/if.h
+	net/if_pfsync.h
+	net/pipex.h
+	netinet/in.h
+	netinet/icmp_var.h
+	netinet/igmp_var.h
+	netinet/ip_ah.h
+	netinet/ip_carp.h
+	netinet/ip_divert.h
+	netinet/ip_esp.h
+	netinet/ip_ether.h
+	netinet/ip_gre.h
+	netinet/ip_ipcomp.h
+	netinet/ip_ipip.h
+	netinet/pim_var.h
+	netinet/tcp_var.h
+	netinet/udp_var.h
+	netinet6/in6.h
+	netinet6/ip6_divert.h
+	netinet6/pim6_var.h
+	netinet/icmp6.h
+	netmpls/mpls.h
+);
+
+my @ctls = qw (
+	kern
+	vm
+	fs
+	net
+	#debug				# Special handling required
+	hw
+	#machdep			# Arch specific
+	user
+	ddb
+	#vfs				# Special handling required
+	fs.posix
+	kern.forkstat
+	kern.intrcnt
+	kern.malloc
+	kern.nchstats
+	kern.seminfo
+	kern.shminfo
+	kern.timecounter
+	kern.tty
+	kern.watchdog
+	net.bpf
+	net.ifq
+	net.inet
+	net.inet.ah
+	net.inet.carp
+	net.inet.divert
+	net.inet.esp
+	net.inet.etherip
+	net.inet.gre
+	net.inet.icmp
+	net.inet.igmp
+	net.inet.ip
+	net.inet.ip.ifq
+	net.inet.ipcomp
+	net.inet.ipip
+	net.inet.mobileip
+	net.inet.pfsync
+	net.inet.pim
+	net.inet.tcp
+	net.inet.udp
+	net.inet6
+	net.inet6.divert
+	net.inet6.ip6
+	net.inet6.icmp6
+	net.inet6.pim6
+	net.inet6.tcp6
+	net.inet6.udp6
+	net.mpls
+	net.mpls.ifq
+	net.key
+	net.pflow
+	net.pfsync
+	net.pipex
+	net.rt
+	vm.swapencrypt
+	#vfsgenctl			# Special handling required
+);
+
+# Node name "fixups"
+my %ctl_map = (
+	"ipproto" => "net.inet",
+	"net.inet.ipproto" => "net.inet",
+	"net.inet6.ipv6proto" => "net.inet6",
+	"net.inet6.ipv6" => "net.inet6.ip6",
+	"net.inet.icmpv6" => "net.inet6.icmp6",
+	"net.inet6.divert6" => "net.inet6.divert",
+	"net.inet6.tcp6" => "net.inet.tcp",
+	"net.inet6.udp6" => "net.inet.udp",
+	"mpls" => "net.mpls",
+	"swpenc" => "vm.swapencrypt"
+);
+
+# Node mappings
+my %node_map = (
+	"net.inet.ip.ifq" => "net.ifq",
+	"net.inet.pfsync" => "net.pfsync",
+	"net.mpls.ifq" => "net.ifq"
+);
+
+my $ctlname;
+my %mib = ();
+my %sysctl = ();
+my $node;
+
+sub debug() {
+	print STDERR "$_[0]\n" if $debug;
+}
+
+# Walk the MIB and build a sysctl name to OID mapping.
+sub build_sysctl() {
+	my ($node, $name, $oid) = @_;
+	my %node = %{$node};
+	my @oid = @{$oid};
+
+	foreach my $key (sort keys %node) {
+		my @node = @{$node{$key}};
+		my $nodename = $name.($name ne '' ? '.' : '').$key;
+		my @nodeoid = (@oid, $node[0]);
+		if ($node[1] eq 'CTLTYPE_NODE') {
+			if (exists $node_map{$nodename}) {
+				$node = \%mib;
+				$ctlname = $node_map{$nodename};
+				foreach my $part (split /\./, $ctlname) {
+					$node = \%{@{$$node{$part}}[2]};
+				}
+			} else {
+				$node = $node[2];
+			}
+			&build_sysctl($node, $nodename, \@nodeoid);
+		} elsif ($node[1] ne '') {
+			$sysctl{$nodename} = \@nodeoid;
+		}
+	}
+}
+
+foreach my $ctl (@ctls) {
+	$ctls{$ctl} = $ctl;
+}
+
+# Build MIB
+foreach my $header (@headers) {
+	&debug("Processing $header...");
+	open HEADER, "/usr/include/$header" ||
+	    print STDERR "Failed to open $header\n";
+	while (<HEADER>) {
+		if ($_ =~ /^#define\s+(CTL_NAMES)\s+{/ ||
+		    $_ =~ /^#define\s+(CTL_(.*)_NAMES)\s+{/ ||
+		    $_ =~ /^#define\s+((.*)CTL_NAMES)\s+{/) {
+			if ($1 eq 'CTL_NAMES') {
+				# Top level.
+				$node = \%mib;
+			} else {
+				# Node.
+				my $nodename = lc($2);
+				if ($header =~ /^netinet\//) {
+					$ctlname = "net.inet.$nodename";
+				} elsif ($header =~ /^netinet6\//) {
+					$ctlname = "net.inet6.$nodename";
+				} elsif ($header =~ /^net\//) {
+					$ctlname = "net.$nodename";
+				} else {
+					$ctlname = "$nodename";
+					$ctlname =~ s/^(fs|net|kern)_/$1\./;
+				}
+				if (exists $ctl_map{$ctlname}) {
+					$ctlname = $ctl_map{$ctlname};
+				}
+				if (not exists $ctls{$ctlname}) {
+					&debug("Ignoring $ctlname...");
+					next;
+				}
+
+				# Walk down from the top of the MIB.
+				$node = \%mib;
+				foreach my $part (split /\./, $ctlname) {
+					if (not exists $$node{$part}) {
+						&debug("Missing node $part");
+						$$node{$part} = [ 0, '', {} ];
+					}
+					$node = \%{@{$$node{$part}}[2]};
+				}
+			}
+
+			# Populate current node with entries.
+			my $i = -1;
+			while (defined($_) && $_ !~ /^}/) {
+				$_ = <HEADER>;
+				$i++ if $_ =~ /{.*}/;
+				next if $_ !~ /{\s+"(\w+)",\s+(CTLTYPE_[A-Z]+)\s+}/;
+				$$node{$1} = [ $i, $2, {} ];
+			}
+		}
+	}
+	close HEADER;
+}
+
+&build_sysctl(\%mib, "", []);
+
+print <<EOF;
+// mksysctl_openbsd.pl
+// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT
+
+// +build $ENV{'GOARCH'},$ENV{'GOOS'}
+
+package unix;
+
+type mibentry struct {
+	ctlname string
+	ctloid []_C_int
+}
+
+var sysctlMib = []mibentry {
+EOF
+
+foreach my $name (sort keys %sysctl) {
+	my @oid = @{$sysctl{$name}};
+	print "\t{ \"$name\", []_C_int{ ", join(', ', @oid), " } }, \n";
+}
+
+print <<EOF;
+}
+EOF

vendor/golang.org/x/sys/unix/mksysnum_darwin.pl

@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# Generate system call table for Darwin from sys/syscall.h
+
+use strict;
+
+if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
+	print STDERR "GOARCH or GOOS not defined in environment\n";
+	exit 1;
+}
+
+my $command = "mksysnum_darwin.pl " . join(' ', @ARGV);
+
+print <<EOF;
+// $command
+// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT
+
+// +build $ENV{'GOARCH'},$ENV{'GOOS'}
+
+package unix
+
+const (
+EOF
+
+while(<>){
+	if(/^#define\s+SYS_(\w+)\s+([0-9]+)/){
+		my $name = $1;
+		my $num = $2;
+		$name =~ y/a-z/A-Z/;
+		print "	SYS_$name = $num;"
+	}
+}
+
+print <<EOF;
+)
+EOF

vendor/golang.org/x/sys/unix/mksysnum_dragonfly.pl

@@ -0,0 +1,50 @@
+#!/usr/bin/env perl
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# Generate system call table for DragonFly from master list
+# (for example, /usr/src/sys/kern/syscalls.master).
+
+use strict;
+
+if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
+	print STDERR "GOARCH or GOOS not defined in environment\n";
+	exit 1;
+}
+
+my $command = "mksysnum_dragonfly.pl " . join(' ', @ARGV);
+
+print <<EOF;
+// $command
+// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT
+
+// +build $ENV{'GOARCH'},$ENV{'GOOS'}
+
+package unix
+
+const (
+EOF
+
+while(<>){
+	if(/^([0-9]+)\s+STD\s+({ \S+\s+(\w+).*)$/){
+		my $num = $1;
+		my $proto = $2;
+		my $name = "SYS_$3";
+		$name =~ y/a-z/A-Z/;
+
+		# There are multiple entries for enosys and nosys, so comment them out.
+		if($name =~ /^SYS_E?NOSYS$/){
+			$name = "// $name";
+		}
+		if($name eq 'SYS_SYS_EXIT'){
+			$name = 'SYS_EXIT';
+		}
+
+		print "	$name = $num;  // $proto\n";
+	}
+}
+
+print <<EOF;
+)
+EOF

vendor/golang.org/x/sys/unix/mksysnum_freebsd.pl

@@ -0,0 +1,63 @@
+#!/usr/bin/env perl
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# Generate system call table for FreeBSD from master list
+# (for example, /usr/src/sys/kern/syscalls.master).
+
+use strict;
+
+if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
+	print STDERR "GOARCH or GOOS not defined in environment\n";
+	exit 1;
+}
+
+my $command = "mksysnum_freebsd.pl " . join(' ', @ARGV);
+
+print <<EOF;
+// $command
+// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT
+
+// +build $ENV{'GOARCH'},$ENV{'GOOS'}
+
+package unix
+
+const (
+EOF
+
+while(<>){
+	if(/^([0-9]+)\s+\S+\s+STD\s+({ \S+\s+(\w+).*)$/){
+		my $num = $1;
+		my $proto = $2;
+		my $name = "SYS_$3";
+		$name =~ y/a-z/A-Z/;
+
+		# There are multiple entries for enosys and nosys, so comment them out.
+		if($name =~ /^SYS_E?NOSYS$/){
+			$name = "// $name";
+		}
+		if($name eq 'SYS_SYS_EXIT'){
+			$name = 'SYS_EXIT';
+		}
+		if($name =~ /^SYS_CAP_+/ || $name =~ /^SYS___CAP_+/){
+			next
+		}
+
+		print "	$name = $num;  // $proto\n";
+
+		# We keep Capsicum syscall numbers for FreeBSD
+		# 9-STABLE here because we are not sure whether they
+		# are mature and stable.
+		if($num == 513){
+			print " SYS_CAP_NEW = 514 // { int cap_new(int fd, uint64_t rights); }\n";
+			print " SYS_CAP_GETRIGHTS = 515 // { int cap_getrights(int fd, \\\n";
+			print " SYS_CAP_ENTER = 516 // { int cap_enter(void); }\n";
+			print " SYS_CAP_GETMODE = 517 // { int cap_getmode(u_int *modep); }\n";
+		}
+	}
+}
+
+print <<EOF;
+)
+EOF

vendor/golang.org/x/sys/unix/mksysnum_linux.pl

@@ -0,0 +1,58 @@
+#!/usr/bin/env perl
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+use strict;
+
+if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
+	print STDERR "GOARCH or GOOS not defined in environment\n";
+	exit 1;
+}
+
+my $command = "mksysnum_linux.pl ". join(' ', @ARGV);
+
+print <<EOF;
+// $command
+// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT
+
+// +build $ENV{'GOARCH'},$ENV{'GOOS'}
+
+package unix
+
+const(
+EOF
+
+sub fmt {
+	my ($name, $num) = @_;
+	if($num > 999){
+		# ignore deprecated syscalls that are no longer implemented
+		# https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/include/uapi/asm-generic/unistd.h?id=refs/heads/master#n716
+		return;
+	}
+	$name =~ y/a-z/A-Z/;
+	print "	SYS_$name = $num;\n";
+}
+
+my $prev;
+open(GCC, "gcc -E -dD $ARGV[0] |") || die "can't run gcc";
+while(<GCC>){
+	if(/^#define __NR_syscalls\s+/) {
+		# ignore redefinitions of __NR_syscalls
+	}
+	elsif(/^#define __NR_(\w+)\s+([0-9]+)/){
+		$prev = $2;
+		fmt($1, $2);
+	}
+	elsif(/^#define __NR3264_(\w+)\s+([0-9]+)/){
+		$prev = $2;
+		fmt($1, $2);
+	}
+	elsif(/^#define __NR_(\w+)\s+\(\w+\+\s*([0-9]+)\)/){
+		fmt($1, $prev+$2)
+	}
+}
+
+print <<EOF;
+)
+EOF

vendor/golang.org/x/sys/unix/mksysnum_netbsd.pl

@@ -0,0 +1,58 @@
+#!/usr/bin/env perl
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# Generate system call table for OpenBSD from master list
+# (for example, /usr/src/sys/kern/syscalls.master).
+
+use strict;
+
+if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
+	print STDERR "GOARCH or GOOS not defined in environment\n";
+	exit 1;
+}
+
+my $command = "mksysnum_netbsd.pl " . join(' ', @ARGV);
+
+print <<EOF;
+// $command
+// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT
+
+// +build $ENV{'GOARCH'},$ENV{'GOOS'}
+
+package unix
+
+const (
+EOF
+
+my $line = '';
+while(<>){
+	if($line =~ /^(.*)\\$/) {
+		# Handle continuation
+		$line = $1;
+		$_ =~ s/^\s+//;
+		$line .= $_;
+	} else {
+		# New line
+		$line = $_;
+	}
+	next if $line =~ /\\$/;
+	if($line =~ /^([0-9]+)\s+((STD)|(NOERR))\s+(RUMP\s+)?({\s+\S+\s*\*?\s*\|(\S+)\|(\S*)\|(\w+).*\s+})(\s+(\S+))?$/) {
+		my $num = $1;
+		my $proto = $6;
+		my $compat = $8;
+		my $name = "$7_$9";
+
+		$name = "$7_$11" if $11 ne '';
+		$name =~ y/a-z/A-Z/;
+
+		if($compat eq '' || $compat eq '30' || $compat eq '50') {
+			print "	$name = $num;  // $proto\n";
+		}
+	}
+}
+
+print <<EOF;
+)
+EOF

vendor/golang.org/x/sys/unix/mksysnum_openbsd.pl

@@ -0,0 +1,50 @@
+#!/usr/bin/env perl
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+#
+# Generate system call table for OpenBSD from master list
+# (for example, /usr/src/sys/kern/syscalls.master).
+
+use strict;
+
+if($ENV{'GOARCH'} eq "" || $ENV{'GOOS'} eq "") {
+	print STDERR "GOARCH or GOOS not defined in environment\n";
+	exit 1;
+}
+
+my $command = "mksysnum_openbsd.pl " . join(' ', @ARGV);
+
+print <<EOF;
+// $command
+// MACHINE GENERATED BY THE ABOVE COMMAND; DO NOT EDIT
+
+// +build $ENV{'GOARCH'},$ENV{'GOOS'}
+
+package unix
+
+const (
+EOF
+
+while(<>){
+	if(/^([0-9]+)\s+STD\s+(NOLOCK\s+)?({ \S+\s+\*?(\w+).*)$/){
+		my $num = $1;
+		my $proto = $3;
+		my $name = $4;
+		$name =~ y/a-z/A-Z/;
+
+		# There are multiple entries for enosys and nosys, so comment them out.
+		if($name =~ /^SYS_E?NOSYS$/){
+			$name = "// $name";
+		}
+		if($name eq 'SYS_SYS_EXIT'){
+			$name = 'SYS_EXIT';
+		}
+
+		print "	$name = $num;  // $proto\n";
+	}
+}
+
+print <<EOF;
+)
+EOF

vendor/golang.org/x/sys/unix/mmap_unix_test.go

@@ -0,0 +1,23 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package unix_test
+
+import (
+	"testing"
+
+	"golang.org/x/sys/unix"
+)
+
+func TestMmap(t *testing.T) {
+	b, err := unix.Mmap(-1, 0, unix.Getpagesize(), unix.PROT_NONE, unix.MAP_ANON|unix.MAP_PRIVATE)
+	if err != nil {
+		t.Fatalf("Mmap: %v", err)
+	}
+	if err := unix.Munmap(b); err != nil {
+		t.Fatalf("Munmap: %v", err)
+	}
+}

vendor/golang.org/x/sys/unix/race.go

@@ -0,0 +1,30 @@
+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,race linux,race freebsd,race
+
+package unix
+
+import (
+	"runtime"
+	"unsafe"
+)
+
+const raceenabled = true
+
+func raceAcquire(addr unsafe.Pointer) {
+	runtime.RaceAcquire(addr)
+}
+
+func raceReleaseMerge(addr unsafe.Pointer) {
+	runtime.RaceReleaseMerge(addr)
+}
+
+func raceReadRange(addr unsafe.Pointer, len int) {
+	runtime.RaceReadRange(addr, len)
+}
+
+func raceWriteRange(addr unsafe.Pointer, len int) {
+	runtime.RaceWriteRange(addr, len)
+}

vendor/golang.org/x/sys/unix/race0.go

@@ -0,0 +1,25 @@
+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin,!race linux,!race freebsd,!race netbsd openbsd solaris dragonfly
+
+package unix
+
+import (
+	"unsafe"
+)
+
+const raceenabled = false
+
+func raceAcquire(addr unsafe.Pointer) {
+}
+
+func raceReleaseMerge(addr unsafe.Pointer) {
+}
+
+func raceReadRange(addr unsafe.Pointer, len int) {
+}
+
+func raceWriteRange(addr unsafe.Pointer, len int) {
+}

vendor/golang.org/x/sys/unix/sockcmsg_linux.go

@@ -0,0 +1,36 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Socket control messages
+
+package unix
+
+import "unsafe"
+
+// UnixCredentials encodes credentials into a socket control message
+// for sending to another process. This can be used for
+// authentication.
+func UnixCredentials(ucred *Ucred) []byte {
+	b := make([]byte, CmsgSpace(SizeofUcred))
+	h := (*Cmsghdr)(unsafe.Pointer(&b[0]))
+	h.Level = SOL_SOCKET
+	h.Type = SCM_CREDENTIALS
+	h.SetLen(CmsgLen(SizeofUcred))
+	*((*Ucred)(cmsgData(h))) = *ucred
+	return b
+}
+
+// ParseUnixCredentials decodes a socket control message that contains
+// credentials in a Ucred structure. To receive such a message, the
+// SO_PASSCRED option must be enabled on the socket.
+func ParseUnixCredentials(m *SocketControlMessage) (*Ucred, error) {
+	if m.Header.Level != SOL_SOCKET {
+		return nil, EINVAL
+	}
+	if m.Header.Type != SCM_CREDENTIALS {
+		return nil, EINVAL
+	}
+	ucred := *(*Ucred)(unsafe.Pointer(&m.Data[0]))
+	return &ucred, nil
+}

vendor/golang.org/x/sys/unix/sockcmsg_unix.go

@@ -0,0 +1,103 @@
+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+// Socket control messages
+
+package unix
+
+import "unsafe"
+
+// Round the length of a raw sockaddr up to align it properly.
+func cmsgAlignOf(salen int) int {
+	salign := sizeofPtr
+	// NOTE: It seems like 64-bit Darwin and DragonFly BSD kernels
+	// still require 32-bit aligned access to network subsystem.
+	if darwin64Bit || dragonfly64Bit {
+		salign = 4
+	}
+	return (salen + salign - 1) & ^(salign - 1)
+}
+
+// CmsgLen returns the value to store in the Len field of the Cmsghdr
+// structure, taking into account any necessary alignment.
+func CmsgLen(datalen int) int {
+	return cmsgAlignOf(SizeofCmsghdr) + datalen
+}
+
+// CmsgSpace returns the number of bytes an ancillary element with
+// payload of the passed data length occupies.
+func CmsgSpace(datalen int) int {
+	return cmsgAlignOf(SizeofCmsghdr) + cmsgAlignOf(datalen)
+}
+
+func cmsgData(h *Cmsghdr) unsafe.Pointer {
+	return unsafe.Pointer(uintptr(unsafe.Pointer(h)) + uintptr(cmsgAlignOf(SizeofCmsghdr)))
+}
+
+// SocketControlMessage represents a socket control message.
+type SocketControlMessage struct {
+	Header Cmsghdr
+	Data   []byte
+}
+
+// ParseSocketControlMessage parses b as an array of socket control
+// messages.
+func ParseSocketControlMessage(b []byte) ([]SocketControlMessage, error) {
+	var msgs []SocketControlMessage
+	i := 0
+	for i+CmsgLen(0) <= len(b) {
+		h, dbuf, err := socketControlMessageHeaderAndData(b[i:])
+		if err != nil {
+			return nil, err
+		}
+		m := SocketControlMessage{Header: *h, Data: dbuf}
+		msgs = append(msgs, m)
+		i += cmsgAlignOf(int(h.Len))
+	}
+	return msgs, nil
+}
+
+func socketControlMessageHeaderAndData(b []byte) (*Cmsghdr, []byte, error) {
+	h := (*Cmsghdr)(unsafe.Pointer(&b[0]))
+	if h.Len < SizeofCmsghdr || uint64(h.Len) > uint64(len(b)) {
+		return nil, nil, EINVAL
+	}
+	return h, b[cmsgAlignOf(SizeofCmsghdr):h.Len], nil
+}
+
+// UnixRights encodes a set of open file descriptors into a socket
+// control message for sending to another process.
+func UnixRights(fds ...int) []byte {
+	datalen := len(fds) * 4
+	b := make([]byte, CmsgSpace(datalen))
+	h := (*Cmsghdr)(unsafe.Pointer(&b[0]))
+	h.Level = SOL_SOCKET
+	h.Type = SCM_RIGHTS
+	h.SetLen(CmsgLen(datalen))
+	data := cmsgData(h)
+	for _, fd := range fds {
+		*(*int32)(data) = int32(fd)
+		data = unsafe.Pointer(uintptr(data) + 4)
+	}
+	return b
+}
+
+// ParseUnixRights decodes a socket control message that contains an
+// integer array of open file descriptors from another process.
+func ParseUnixRights(m *SocketControlMessage) ([]int, error) {
+	if m.Header.Level != SOL_SOCKET {
+		return nil, EINVAL
+	}
+	if m.Header.Type != SCM_RIGHTS {
+		return nil, EINVAL
+	}
+	fds := make([]int, len(m.Data)>>2)
+	for i, j := 0, 0; i < len(m.Data); i += 4 {
+		fds[j] = int(*(*int32)(unsafe.Pointer(&m.Data[i])))
+		j++
+	}
+	return fds, nil
+}

vendor/golang.org/x/sys/unix/str.go

@@ -0,0 +1,26 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+package unix
+
+func itoa(val int) string { // do it here rather than with fmt to avoid dependency
+	if val < 0 {
+		return "-" + uitoa(uint(-val))
+	}
+	return uitoa(uint(val))
+}
+
+func uitoa(val uint) string {
+	var buf [32]byte // big enough for int64
+	i := len(buf) - 1
+	for val >= 10 {
+		buf[i] = byte(val%10 + '0')
+		i--
+		val /= 10
+	}
+	buf[i] = byte(val + '0')
+	return string(buf[i:])
+}

vendor/golang.org/x/sys/unix/syscall.go

@@ -0,0 +1,76 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd solaris
+
+// Package unix contains an interface to the low-level operating system
+// primitives.  OS details vary depending on the underlying system, and
+// by default, godoc will display OS-specific documentation for the current
+// system.  If you want godoc to display OS documentation for another
+// system, set $GOOS and $GOARCH to the desired system.  For example, if
+// you want to view documentation for freebsd/arm on linux/amd64, set $GOOS
+// to freebsd and $GOARCH to arm.
+// The primary use of this package is inside other packages that provide a more
+// portable interface to the system, such as "os", "time" and "net".  Use
+// those packages rather than this one if you can.
+// For details of the functions and data types in this package consult
+// the manuals for the appropriate operating system.
+// These calls return err == nil to indicate success; otherwise
+// err represents an operating system error describing the failure and
+// holds a value of type syscall.Errno.
+package unix // import "golang.org/x/sys/unix"
+
+import "unsafe"
+
+// ByteSliceFromString returns a NUL-terminated slice of bytes
+// containing the text of s. If s contains a NUL byte at any
+// location, it returns (nil, EINVAL).
+func ByteSliceFromString(s string) ([]byte, error) {
+	for i := 0; i < len(s); i++ {
+		if s[i] == 0 {
+			return nil, EINVAL
+		}
+	}
+	a := make([]byte, len(s)+1)
+	copy(a, s)
+	return a, nil
+}
+
+// BytePtrFromString returns a pointer to a NUL-terminated array of
+// bytes containing the text of s. If s contains a NUL byte at any
+// location, it returns (nil, EINVAL).
+func BytePtrFromString(s string) (*byte, error) {
+	a, err := ByteSliceFromString(s)
+	if err != nil {
+		return nil, err
+	}
+	return &a[0], nil
+}
+
+// Single-word zero for use when we need a valid pointer to 0 bytes.
+// See mkunix.pl.
+var _zero uintptr
+
+func (ts *Timespec) Unix() (sec int64, nsec int64) {
+	return int64(ts.Sec), int64(ts.Nsec)
+}
+
+func (tv *Timeval) Unix() (sec int64, nsec int64) {
+	return int64(tv.Sec), int64(tv.Usec) * 1000
+}
+
+func (ts *Timespec) Nano() int64 {
+	return int64(ts.Sec)*1e9 + int64(ts.Nsec)
+}
+
+func (tv *Timeval) Nano() int64 {
+	return int64(tv.Sec)*1e9 + int64(tv.Usec)*1000
+}
+
+func TimevalToNsec(tv Timeval) int64 { return int64(tv.Sec)*1e9 + int64(tv.Usec)*1e3 }
+
+// use is a no-op, but the compiler cannot see that it is.
+// Calling use(p) ensures that p is kept live until that point.
+//go:noescape
+func use(p unsafe.Pointer)

vendor/golang.org/x/sys/unix/syscall_bsd.go

@@ -0,0 +1,628 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd netbsd openbsd
+
+// BSD system call wrappers shared by *BSD based systems
+// including OS X (Darwin) and FreeBSD.  Like the other
+// syscall_*.go files it is compiled as Go code but also
+// used as input to mksyscall which parses the //sys
+// lines and generates system call stubs.
+
+package unix
+
+import (
+	"runtime"
+	"syscall"
+	"unsafe"
+)
+
+/*
+ * Wrapped
+ */
+
+//sysnb	getgroups(ngid int, gid *_Gid_t) (n int, err error)
+//sysnb	setgroups(ngid int, gid *_Gid_t) (err error)
+
+func Getgroups() (gids []int, err error) {
+	n, err := getgroups(0, nil)
+	if err != nil {
+		return nil, err
+	}
+	if n == 0 {
+		return nil, nil
+	}
+
+	// Sanity check group count.  Max is 16 on BSD.
+	if n < 0 || n > 1000 {
+		return nil, EINVAL
+	}
+
+	a := make([]_Gid_t, n)
+	n, err = getgroups(n, &a[0])
+	if err != nil {
+		return nil, err
+	}
+	gids = make([]int, n)
+	for i, v := range a[0:n] {
+		gids[i] = int(v)
+	}
+	return
+}
+
+func Setgroups(gids []int) (err error) {
+	if len(gids) == 0 {
+		return setgroups(0, nil)
+	}
+
+	a := make([]_Gid_t, len(gids))
+	for i, v := range gids {
+		a[i] = _Gid_t(v)
+	}
+	return setgroups(len(a), &a[0])
+}
+
+func ReadDirent(fd int, buf []byte) (n int, err error) {
+	// Final argument is (basep *uintptr) and the syscall doesn't take nil.
+	// 64 bits should be enough. (32 bits isn't even on 386). Since the
+	// actual system call is getdirentries64, 64 is a good guess.
+	// TODO(rsc): Can we use a single global basep for all calls?
+	var base = (*uintptr)(unsafe.Pointer(new(uint64)))
+	return Getdirentries(fd, buf, base)
+}
+
+// Wait status is 7 bits at bottom, either 0 (exited),
+// 0x7F (stopped), or a signal number that caused an exit.
+// The 0x80 bit is whether there was a core dump.
+// An extra number (exit code, signal causing a stop)
+// is in the high bits.
+
+type WaitStatus uint32
+
+const (
+	mask  = 0x7F
+	core  = 0x80
+	shift = 8
+
+	exited  = 0
+	stopped = 0x7F
+)
+
+func (w WaitStatus) Exited() bool { return w&mask == exited }
+
+func (w WaitStatus) ExitStatus() int {
+	if w&mask != exited {
+		return -1
+	}
+	return int(w >> shift)
+}
+
+func (w WaitStatus) Signaled() bool { return w&mask != stopped && w&mask != 0 }
+
+func (w WaitStatus) Signal() syscall.Signal {
+	sig := syscall.Signal(w & mask)
+	if sig == stopped || sig == 0 {
+		return -1
+	}
+	return sig
+}
+
+func (w WaitStatus) CoreDump() bool { return w.Signaled() && w&core != 0 }
+
+func (w WaitStatus) Stopped() bool { return w&mask == stopped && syscall.Signal(w>>shift) != SIGSTOP }
+
+func (w WaitStatus) Continued() bool { return w&mask == stopped && syscall.Signal(w>>shift) == SIGSTOP }
+
+func (w WaitStatus) StopSignal() syscall.Signal {
+	if !w.Stopped() {
+		return -1
+	}
+	return syscall.Signal(w>>shift) & 0xFF
+}
+
+func (w WaitStatus) TrapCause() int { return -1 }
+
+//sys	wait4(pid int, wstatus *_C_int, options int, rusage *Rusage) (wpid int, err error)
+
+func Wait4(pid int, wstatus *WaitStatus, options int, rusage *Rusage) (wpid int, err error) {
+	var status _C_int
+	wpid, err = wait4(pid, &status, options, rusage)
+	if wstatus != nil {
+		*wstatus = WaitStatus(status)
+	}
+	return
+}
+
+//sys	accept(s int, rsa *RawSockaddrAny, addrlen *_Socklen) (fd int, err error)
+//sys	bind(s int, addr unsafe.Pointer, addrlen _Socklen) (err error)
+//sys	connect(s int, addr unsafe.Pointer, addrlen _Socklen) (err error)
+//sysnb	socket(domain int, typ int, proto int) (fd int, err error)
+//sys	getsockopt(s int, level int, name int, val unsafe.Pointer, vallen *_Socklen) (err error)
+//sys	setsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error)
+//sysnb	getpeername(fd int, rsa *RawSockaddrAny, addrlen *_Socklen) (err error)
+//sysnb	getsockname(fd int, rsa *RawSockaddrAny, addrlen *_Socklen) (err error)
+//sys	Shutdown(s int, how int) (err error)
+
+func (sa *SockaddrInet4) sockaddr() (unsafe.Pointer, _Socklen, error) {
+	if sa.Port < 0 || sa.Port > 0xFFFF {
+		return nil, 0, EINVAL
+	}
+	sa.raw.Len = SizeofSockaddrInet4
+	sa.raw.Family = AF_INET
+	p := (*[2]byte)(unsafe.Pointer(&sa.raw.Port))
+	p[0] = byte(sa.Port >> 8)
+	p[1] = byte(sa.Port)
+	for i := 0; i < len(sa.Addr); i++ {
+		sa.raw.Addr[i] = sa.Addr[i]
+	}
+	return unsafe.Pointer(&sa.raw), _Socklen(sa.raw.Len), nil
+}
+
+func (sa *SockaddrInet6) sockaddr() (unsafe.Pointer, _Socklen, error) {
+	if sa.Port < 0 || sa.Port > 0xFFFF {
+		return nil, 0, EINVAL
+	}
+	sa.raw.Len = SizeofSockaddrInet6
+	sa.raw.Family = AF_INET6
+	p := (*[2]byte)(unsafe.Pointer(&sa.raw.Port))
+	p[0] = byte(sa.Port >> 8)
+	p[1] = byte(sa.Port)
+	sa.raw.Scope_id = sa.ZoneId
+	for i := 0; i < len(sa.Addr); i++ {
+		sa.raw.Addr[i] = sa.Addr[i]
+	}
+	return unsafe.Pointer(&sa.raw), _Socklen(sa.raw.Len), nil
+}
+
+func (sa *SockaddrUnix) sockaddr() (unsafe.Pointer, _Socklen, error) {
+	name := sa.Name
+	n := len(name)
+	if n >= len(sa.raw.Path) || n == 0 {
+		return nil, 0, EINVAL
+	}
+	sa.raw.Len = byte(3 + n) // 2 for Family, Len; 1 for NUL
+	sa.raw.Family = AF_UNIX
+	for i := 0; i < n; i++ {
+		sa.raw.Path[i] = int8(name[i])
+	}
+	return unsafe.Pointer(&sa.raw), _Socklen(sa.raw.Len), nil
+}
+
+func (sa *SockaddrDatalink) sockaddr() (unsafe.Pointer, _Socklen, error) {
+	if sa.Index == 0 {
+		return nil, 0, EINVAL
+	}
+	sa.raw.Len = sa.Len
+	sa.raw.Family = AF_LINK
+	sa.raw.Index = sa.Index
+	sa.raw.Type = sa.Type
+	sa.raw.Nlen = sa.Nlen
+	sa.raw.Alen = sa.Alen
+	sa.raw.Slen = sa.Slen
+	for i := 0; i < len(sa.raw.Data); i++ {
+		sa.raw.Data[i] = sa.Data[i]
+	}
+	return unsafe.Pointer(&sa.raw), SizeofSockaddrDatalink, nil
+}
+
+func anyToSockaddr(rsa *RawSockaddrAny) (Sockaddr, error) {
+	switch rsa.Addr.Family {
+	case AF_LINK:
+		pp := (*RawSockaddrDatalink)(unsafe.Pointer(rsa))
+		sa := new(SockaddrDatalink)
+		sa.Len = pp.Len
+		sa.Family = pp.Family
+		sa.Index = pp.Index
+		sa.Type = pp.Type
+		sa.Nlen = pp.Nlen
+		sa.Alen = pp.Alen
+		sa.Slen = pp.Slen
+		for i := 0; i < len(sa.Data); i++ {
+			sa.Data[i] = pp.Data[i]
+		}
+		return sa, nil
+
+	case AF_UNIX:
+		pp := (*RawSockaddrUnix)(unsafe.Pointer(rsa))
+		if pp.Len < 2 || pp.Len > SizeofSockaddrUnix {
+			return nil, EINVAL
+		}
+		sa := new(SockaddrUnix)
+
+		// Some BSDs include the trailing NUL in the length, whereas
+		// others do not. Work around this by subtracting the leading
+		// family and len. The path is then scanned to see if a NUL
+		// terminator still exists within the length.
+		n := int(pp.Len) - 2 // subtract leading Family, Len
+		for i := 0; i < n; i++ {
+			if pp.Path[i] == 0 {
+				// found early NUL; assume Len included the NUL
+				// or was overestimating.
+				n = i
+				break
+			}
+		}
+		bytes := (*[10000]byte)(unsafe.Pointer(&pp.Path[0]))[0:n]
+		sa.Name = string(bytes)
+		return sa, nil
+
+	case AF_INET:
+		pp := (*RawSockaddrInet4)(unsafe.Pointer(rsa))
+		sa := new(SockaddrInet4)
+		p := (*[2]byte)(unsafe.Pointer(&pp.Port))
+		sa.Port = int(p[0])<<8 + int(p[1])
+		for i := 0; i < len(sa.Addr); i++ {
+			sa.Addr[i] = pp.Addr[i]
+		}
+		return sa, nil
+
+	case AF_INET6:
+		pp := (*RawSockaddrInet6)(unsafe.Pointer(rsa))
+		sa := new(SockaddrInet6)
+		p := (*[2]byte)(unsafe.Pointer(&pp.Port))
+		sa.Port = int(p[0])<<8 + int(p[1])
+		sa.ZoneId = pp.Scope_id
+		for i := 0; i < len(sa.Addr); i++ {
+			sa.Addr[i] = pp.Addr[i]
+		}
+		return sa, nil
+	}
+	return nil, EAFNOSUPPORT
+}
+
+func Accept(fd int) (nfd int, sa Sockaddr, err error) {
+	var rsa RawSockaddrAny
+	var len _Socklen = SizeofSockaddrAny
+	nfd, err = accept(fd, &rsa, &len)
+	if err != nil {
+		return
+	}
+	if runtime.GOOS == "darwin" && len == 0 {
+		// Accepted socket has no address.
+		// This is likely due to a bug in xnu kernels,
+		// where instead of ECONNABORTED error socket
+		// is accepted, but has no address.
+		Close(nfd)
+		return 0, nil, ECONNABORTED
+	}
+	sa, err = anyToSockaddr(&rsa)
+	if err != nil {
+		Close(nfd)
+		nfd = 0
+	}
+	return
+}
+
+func Getsockname(fd int) (sa Sockaddr, err error) {
+	var rsa RawSockaddrAny
+	var len _Socklen = SizeofSockaddrAny
+	if err = getsockname(fd, &rsa, &len); err != nil {
+		return
+	}
+	// TODO(jsing): DragonFly has a "bug" (see issue 3349), which should be
+	// reported upstream.
+	if runtime.GOOS == "dragonfly" && rsa.Addr.Family == AF_UNSPEC && rsa.Addr.Len == 0 {
+		rsa.Addr.Family = AF_UNIX
+		rsa.Addr.Len = SizeofSockaddrUnix
+	}
+	return anyToSockaddr(&rsa)
+}
+
+//sysnb socketpair(domain int, typ int, proto int, fd *[2]int32) (err error)
+
+func GetsockoptByte(fd, level, opt int) (value byte, err error) {
+	var n byte
+	vallen := _Socklen(1)
+	err = getsockopt(fd, level, opt, unsafe.Pointer(&n), &vallen)
+	return n, err
+}
+
+func GetsockoptInet4Addr(fd, level, opt int) (value [4]byte, err error) {
+	vallen := _Socklen(4)
+	err = getsockopt(fd, level, opt, unsafe.Pointer(&value[0]), &vallen)
+	return value, err
+}
+
+func GetsockoptIPMreq(fd, level, opt int) (*IPMreq, error) {
+	var value IPMreq
+	vallen := _Socklen(SizeofIPMreq)
+	err := getsockopt(fd, level, opt, unsafe.Pointer(&value), &vallen)
+	return &value, err
+}
+
+func GetsockoptIPv6Mreq(fd, level, opt int) (*IPv6Mreq, error) {
+	var value IPv6Mreq
+	vallen := _Socklen(SizeofIPv6Mreq)
+	err := getsockopt(fd, level, opt, unsafe.Pointer(&value), &vallen)
+	return &value, err
+}
+
+func GetsockoptIPv6MTUInfo(fd, level, opt int) (*IPv6MTUInfo, error) {
+	var value IPv6MTUInfo
+	vallen := _Socklen(SizeofIPv6MTUInfo)
+	err := getsockopt(fd, level, opt, unsafe.Pointer(&value), &vallen)
+	return &value, err
+}
+
+func GetsockoptICMPv6Filter(fd, level, opt int) (*ICMPv6Filter, error) {
+	var value ICMPv6Filter
+	vallen := _Socklen(SizeofICMPv6Filter)
+	err := getsockopt(fd, level, opt, unsafe.Pointer(&value), &vallen)
+	return &value, err
+}
+
+//sys   recvfrom(fd int, p []byte, flags int, from *RawSockaddrAny, fromlen *_Socklen) (n int, err error)
+//sys   sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen _Socklen) (err error)
+//sys	recvmsg(s int, msg *Msghdr, flags int) (n int, err error)
+
+func Recvmsg(fd int, p, oob []byte, flags int) (n, oobn int, recvflags int, from Sockaddr, err error) {
+	var msg Msghdr
+	var rsa RawSockaddrAny
+	msg.Name = (*byte)(unsafe.Pointer(&rsa))
+	msg.Namelen = uint32(SizeofSockaddrAny)
+	var iov Iovec
+	if len(p) > 0 {
+		iov.Base = (*byte)(unsafe.Pointer(&p[0]))
+		iov.SetLen(len(p))
+	}
+	var dummy byte
+	if len(oob) > 0 {
+		// receive at least one normal byte
+		if len(p) == 0 {
+			iov.Base = &dummy
+			iov.SetLen(1)
+		}
+		msg.Control = (*byte)(unsafe.Pointer(&oob[0]))
+		msg.SetControllen(len(oob))
+	}
+	msg.Iov = &iov
+	msg.Iovlen = 1
+	if n, err = recvmsg(fd, &msg, flags); err != nil {
+		return
+	}
+	oobn = int(msg.Controllen)
+	recvflags = int(msg.Flags)
+	// source address is only specified if the socket is unconnected
+	if rsa.Addr.Family != AF_UNSPEC {
+		from, err = anyToSockaddr(&rsa)
+	}
+	return
+}
+
+//sys	sendmsg(s int, msg *Msghdr, flags int) (n int, err error)
+
+func Sendmsg(fd int, p, oob []byte, to Sockaddr, flags int) (err error) {
+	_, err = SendmsgN(fd, p, oob, to, flags)
+	return
+}
+
+func SendmsgN(fd int, p, oob []byte, to Sockaddr, flags int) (n int, err error) {
+	var ptr unsafe.Pointer
+	var salen _Socklen
+	if to != nil {
+		ptr, salen, err = to.sockaddr()
+		if err != nil {
+			return 0, err
+		}
+	}
+	var msg Msghdr
+	msg.Name = (*byte)(unsafe.Pointer(ptr))
+	msg.Namelen = uint32(salen)
+	var iov Iovec
+	if len(p) > 0 {
+		iov.Base = (*byte)(unsafe.Pointer(&p[0]))
+		iov.SetLen(len(p))
+	}
+	var dummy byte
+	if len(oob) > 0 {
+		// send at least one normal byte
+		if len(p) == 0 {
+			iov.Base = &dummy
+			iov.SetLen(1)
+		}
+		msg.Control = (*byte)(unsafe.Pointer(&oob[0]))
+		msg.SetControllen(len(oob))
+	}
+	msg.Iov = &iov
+	msg.Iovlen = 1
+	if n, err = sendmsg(fd, &msg, flags); err != nil {
+		return 0, err
+	}
+	if len(oob) > 0 && len(p) == 0 {
+		n = 0
+	}
+	return n, nil
+}
+
+//sys	kevent(kq int, change unsafe.Pointer, nchange int, event unsafe.Pointer, nevent int, timeout *Timespec) (n int, err error)
+
+func Kevent(kq int, changes, events []Kevent_t, timeout *Timespec) (n int, err error) {
+	var change, event unsafe.Pointer
+	if len(changes) > 0 {
+		change = unsafe.Pointer(&changes[0])
+	}
+	if len(events) > 0 {
+		event = unsafe.Pointer(&events[0])
+	}
+	return kevent(kq, change, len(changes), event, len(events), timeout)
+}
+
+//sys	sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) = SYS___SYSCTL
+
+// sysctlmib translates name to mib number and appends any additional args.
+func sysctlmib(name string, args ...int) ([]_C_int, error) {
+	// Translate name to mib number.
+	mib, err := nametomib(name)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, a := range args {
+		mib = append(mib, _C_int(a))
+	}
+
+	return mib, nil
+}
+
+func Sysctl(name string) (string, error) {
+	return SysctlArgs(name)
+}
+
+func SysctlArgs(name string, args ...int) (string, error) {
+	mib, err := sysctlmib(name, args...)
+	if err != nil {
+		return "", err
+	}
+
+	// Find size.
+	n := uintptr(0)
+	if err := sysctl(mib, nil, &n, nil, 0); err != nil {
+		return "", err
+	}
+	if n == 0 {
+		return "", nil
+	}
+
+	// Read into buffer of that size.
+	buf := make([]byte, n)
+	if err := sysctl(mib, &buf[0], &n, nil, 0); err != nil {
+		return "", err
+	}
+
+	// Throw away terminating NUL.
+	if n > 0 && buf[n-1] == '\x00' {
+		n--
+	}
+	return string(buf[0:n]), nil
+}
+
+func SysctlUint32(name string) (uint32, error) {
+	return SysctlUint32Args(name)
+}
+
+func SysctlUint32Args(name string, args ...int) (uint32, error) {
+	mib, err := sysctlmib(name, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	n := uintptr(4)
+	buf := make([]byte, 4)
+	if err := sysctl(mib, &buf[0], &n, nil, 0); err != nil {
+		return 0, err
+	}
+	if n != 4 {
+		return 0, EIO
+	}
+	return *(*uint32)(unsafe.Pointer(&buf[0])), nil
+}
+
+func SysctlUint64(name string, args ...int) (uint64, error) {
+	mib, err := sysctlmib(name, args...)
+	if err != nil {
+		return 0, err
+	}
+
+	n := uintptr(8)
+	buf := make([]byte, 8)
+	if err := sysctl(mib, &buf[0], &n, nil, 0); err != nil {
+		return 0, err
+	}
+	if n != 8 {
+		return 0, EIO
+	}
+	return *(*uint64)(unsafe.Pointer(&buf[0])), nil
+}
+
+func SysctlRaw(name string, args ...int) ([]byte, error) {
+	mib, err := sysctlmib(name, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	// Find size.
+	n := uintptr(0)
+	if err := sysctl(mib, nil, &n, nil, 0); err != nil {
+		return nil, err
+	}
+	if n == 0 {
+		return nil, nil
+	}
+
+	// Read into buffer of that size.
+	buf := make([]byte, n)
+	if err := sysctl(mib, &buf[0], &n, nil, 0); err != nil {
+		return nil, err
+	}
+
+	// The actual call may return less than the original reported required
+	// size so ensure we deal with that.
+	return buf[:n], nil
+}
+
+//sys	utimes(path string, timeval *[2]Timeval) (err error)
+
+func Utimes(path string, tv []Timeval) error {
+	if tv == nil {
+		return utimes(path, nil)
+	}
+	if len(tv) != 2 {
+		return EINVAL
+	}
+	return utimes(path, (*[2]Timeval)(unsafe.Pointer(&tv[0])))
+}
+
+func UtimesNano(path string, ts []Timespec) error {
+	if ts == nil {
+		return utimes(path, nil)
+	}
+	// TODO: The BSDs can do utimensat with SYS_UTIMENSAT but it
+	// isn't supported by darwin so this uses utimes instead
+	if len(ts) != 2 {
+		return EINVAL
+	}
+	// Not as efficient as it could be because Timespec and
+	// Timeval have different types in the different OSes
+	tv := [2]Timeval{
+		NsecToTimeval(TimespecToNsec(ts[0])),
+		NsecToTimeval(TimespecToNsec(ts[1])),
+	}
+	return utimes(path, (*[2]Timeval)(unsafe.Pointer(&tv[0])))
+}
+
+//sys	futimes(fd int, timeval *[2]Timeval) (err error)
+
+func Futimes(fd int, tv []Timeval) error {
+	if tv == nil {
+		return futimes(fd, nil)
+	}
+	if len(tv) != 2 {
+		return EINVAL
+	}
+	return futimes(fd, (*[2]Timeval)(unsafe.Pointer(&tv[0])))
+}
+
+//sys	fcntl(fd int, cmd int, arg int) (val int, err error)
+
+// TODO: wrap
+//	Acct(name nil-string) (err error)
+//	Gethostuuid(uuid *byte, timeout *Timespec) (err error)
+//	Madvise(addr *byte, len int, behav int) (err error)
+//	Mprotect(addr *byte, len int, prot int) (err error)
+//	Msync(addr *byte, len int, flags int) (err error)
+//	Ptrace(req int, pid int, addr uintptr, data int) (ret uintptr, err error)
+
+var mapper = &mmapper{
+	active: make(map[*byte][]byte),
+	mmap:   mmap,
+	munmap: munmap,
+}
+
+func Mmap(fd int, offset int64, length int, prot int, flags int) (data []byte, err error) {
+	return mapper.Mmap(fd, offset, length, prot, flags)
+}
+
+func Munmap(b []byte) (err error) {
+	return mapper.Munmap(b)
+}

vendor/golang.org/x/sys/unix/syscall_bsd_test.go

@@ -0,0 +1,62 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd openbsd
+
+package unix_test
+
+import (
+	"os/exec"
+	"runtime"
+	"testing"
+
+	"golang.org/x/sys/unix"
+)
+
+const MNT_WAIT = 1
+const MNT_NOWAIT = 2
+
+func TestGetfsstat(t *testing.T) {
+	const flags = MNT_NOWAIT // see golang.org/issue/16937
+	n, err := unix.Getfsstat(nil, flags)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	data := make([]unix.Statfs_t, n)
+	n2, err := unix.Getfsstat(data, flags)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != n2 {
+		t.Errorf("Getfsstat(nil) = %d, but subsequent Getfsstat(slice) = %d", n, n2)
+	}
+	for i, stat := range data {
+		if stat == (unix.Statfs_t{}) {
+			t.Errorf("index %v is an empty Statfs_t struct", i)
+		}
+	}
+	if t.Failed() {
+		for i, stat := range data[:n2] {
+			t.Logf("data[%v] = %+v", i, stat)
+		}
+		mount, err := exec.Command("mount").CombinedOutput()
+		if err != nil {
+			t.Logf("mount: %v\n%s", err, mount)
+		} else {
+			t.Logf("mount: %s", mount)
+		}
+	}
+}
+
+func TestSysctlRaw(t *testing.T) {
+	if runtime.GOOS == "openbsd" {
+		t.Skip("kern.proc.pid does not exist on OpenBSD")
+	}
+
+	_, err := unix.SysctlRaw("kern.proc.pid", unix.Getpid())
+	if err != nil {
+		t.Fatal(err)
+	}
+}

vendor/golang.org/x/sys/unix/syscall_darwin.go

@@ -0,0 +1,511 @@
+// Copyright 2009,2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Darwin system calls.
+// This file is compiled as ordinary Go code,
+// but it is also input to mksyscall,
+// which parses the //sys lines and generates system call stubs.
+// Note that sometimes we use a lowercase //sys name and wrap
+// it in our own nicer implementation, either here or in
+// syscall_bsd.go or syscall_unix.go.
+
+package unix
+
+import (
+	errorspkg "errors"
+	"syscall"
+	"unsafe"
+)
+
+const ImplementsGetwd = true
+
+func Getwd() (string, error) {
+	buf := make([]byte, 2048)
+	attrs, err := getAttrList(".", attrList{CommonAttr: attrCmnFullpath}, buf, 0)
+	if err == nil && len(attrs) == 1 && len(attrs[0]) >= 2 {
+		wd := string(attrs[0])
+		// Sanity check that it's an absolute path and ends
+		// in a null byte, which we then strip.
+		if wd[0] == '/' && wd[len(wd)-1] == 0 {
+			return wd[:len(wd)-1], nil
+		}
+	}
+	// If pkg/os/getwd.go gets ENOTSUP, it will fall back to the
+	// slow algorithm.
+	return "", ENOTSUP
+}
+
+type SockaddrDatalink struct {
+	Len    uint8
+	Family uint8
+	Index  uint16
+	Type   uint8
+	Nlen   uint8
+	Alen   uint8
+	Slen   uint8
+	Data   [12]int8
+	raw    RawSockaddrDatalink
+}
+
+// Translate "kern.hostname" to []_C_int{0,1,2,3}.
+func nametomib(name string) (mib []_C_int, err error) {
+	const siz = unsafe.Sizeof(mib[0])
+
+	// NOTE(rsc): It seems strange to set the buffer to have
+	// size CTL_MAXNAME+2 but use only CTL_MAXNAME
+	// as the size.  I don't know why the +2 is here, but the
+	// kernel uses +2 for its own implementation of this function.
+	// I am scared that if we don't include the +2 here, the kernel
+	// will silently write 2 words farther than we specify
+	// and we'll get memory corruption.
+	var buf [CTL_MAXNAME + 2]_C_int
+	n := uintptr(CTL_MAXNAME) * siz
+
+	p := (*byte)(unsafe.Pointer(&buf[0]))
+	bytes, err := ByteSliceFromString(name)
+	if err != nil {
+		return nil, err
+	}
+
+	// Magic sysctl: "setting" 0.3 to a string name
+	// lets you read back the array of integers form.
+	if err = sysctl([]_C_int{0, 3}, p, &n, &bytes[0], uintptr(len(name))); err != nil {
+		return nil, err
+	}
+	return buf[0 : n/siz], nil
+}
+
+// ParseDirent parses up to max directory entries in buf,
+// appending the names to names.  It returns the number
+// bytes consumed from buf, the number of entries added
+// to names, and the new names slice.
+func ParseDirent(buf []byte, max int, names []string) (consumed int, count int, newnames []string) {
+	origlen := len(buf)
+	for max != 0 && len(buf) > 0 {
+		dirent := (*Dirent)(unsafe.Pointer(&buf[0]))
+		if dirent.Reclen == 0 {
+			buf = nil
+			break
+		}
+		buf = buf[dirent.Reclen:]
+		if dirent.Ino == 0 { // File absent in directory.
+			continue
+		}
+		bytes := (*[10000]byte)(unsafe.Pointer(&dirent.Name[0]))
+		var name = string(bytes[0:dirent.Namlen])
+		if name == "." || name == ".." { // Useless names
+			continue
+		}
+		max--
+		count++
+		names = append(names, name)
+	}
+	return origlen - len(buf), count, names
+}
+
+//sys   ptrace(request int, pid int, addr uintptr, data uintptr) (err error)
+func PtraceAttach(pid int) (err error) { return ptrace(PT_ATTACH, pid, 0, 0) }
+func PtraceDetach(pid int) (err error) { return ptrace(PT_DETACH, pid, 0, 0) }
+
+const (
+	attrBitMapCount = 5
+	attrCmnFullpath = 0x08000000
+)
+
+type attrList struct {
+	bitmapCount uint16
+	_           uint16
+	CommonAttr  uint32
+	VolAttr     uint32
+	DirAttr     uint32
+	FileAttr    uint32
+	Forkattr    uint32
+}
+
+func getAttrList(path string, attrList attrList, attrBuf []byte, options uint) (attrs [][]byte, err error) {
+	if len(attrBuf) < 4 {
+		return nil, errorspkg.New("attrBuf too small")
+	}
+	attrList.bitmapCount = attrBitMapCount
+
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return nil, err
+	}
+
+	_, _, e1 := Syscall6(
+		SYS_GETATTRLIST,
+		uintptr(unsafe.Pointer(_p0)),
+		uintptr(unsafe.Pointer(&attrList)),
+		uintptr(unsafe.Pointer(&attrBuf[0])),
+		uintptr(len(attrBuf)),
+		uintptr(options),
+		0,
+	)
+	use(unsafe.Pointer(_p0))
+	if e1 != 0 {
+		return nil, e1
+	}
+	size := *(*uint32)(unsafe.Pointer(&attrBuf[0]))
+
+	// dat is the section of attrBuf that contains valid data,
+	// without the 4 byte length header. All attribute offsets
+	// are relative to dat.
+	dat := attrBuf
+	if int(size) < len(attrBuf) {
+		dat = dat[:size]
+	}
+	dat = dat[4:] // remove length prefix
+
+	for i := uint32(0); int(i) < len(dat); {
+		header := dat[i:]
+		if len(header) < 8 {
+			return attrs, errorspkg.New("truncated attribute header")
+		}
+		datOff := *(*int32)(unsafe.Pointer(&header[0]))
+		attrLen := *(*uint32)(unsafe.Pointer(&header[4]))
+		if datOff < 0 || uint32(datOff)+attrLen > uint32(len(dat)) {
+			return attrs, errorspkg.New("truncated results; attrBuf too small")
+		}
+		end := uint32(datOff) + attrLen
+		attrs = append(attrs, dat[datOff:end])
+		i = end
+		if r := i % 4; r != 0 {
+			i += (4 - r)
+		}
+	}
+	return
+}
+
+//sysnb pipe() (r int, w int, err error)
+
+func Pipe(p []int) (err error) {
+	if len(p) != 2 {
+		return EINVAL
+	}
+	p[0], p[1], err = pipe()
+	return
+}
+
+func Getfsstat(buf []Statfs_t, flags int) (n int, err error) {
+	var _p0 unsafe.Pointer
+	var bufsize uintptr
+	if len(buf) > 0 {
+		_p0 = unsafe.Pointer(&buf[0])
+		bufsize = unsafe.Sizeof(Statfs_t{}) * uintptr(len(buf))
+	}
+	r0, _, e1 := Syscall(SYS_GETFSSTAT64, uintptr(_p0), bufsize, uintptr(flags))
+	use(unsafe.Pointer(_p0))
+	n = int(r0)
+	if e1 != 0 {
+		err = e1
+	}
+	return
+}
+
+/*
+ * Wrapped
+ */
+
+//sys	kill(pid int, signum int, posix int) (err error)
+
+func Kill(pid int, signum syscall.Signal) (err error) { return kill(pid, int(signum), 1) }
+
+/*
+ * Exposed directly
+ */
+//sys	Access(path string, mode uint32) (err error)
+//sys	Adjtime(delta *Timeval, olddelta *Timeval) (err error)
+//sys	Chdir(path string) (err error)
+//sys	Chflags(path string, flags int) (err error)
+//sys	Chmod(path string, mode uint32) (err error)
+//sys	Chown(path string, uid int, gid int) (err error)
+//sys	Chroot(path string) (err error)
+//sys	Close(fd int) (err error)
+//sys	Dup(fd int) (nfd int, err error)
+//sys	Dup2(from int, to int) (err error)
+//sys	Exchangedata(path1 string, path2 string, options int) (err error)
+//sys	Exit(code int)
+//sys	Fchdir(fd int) (err error)
+//sys	Fchflags(fd int, flags int) (err error)
+//sys	Fchmod(fd int, mode uint32) (err error)
+//sys	Fchown(fd int, uid int, gid int) (err error)
+//sys	Flock(fd int, how int) (err error)
+//sys	Fpathconf(fd int, name int) (val int, err error)
+//sys	Fstat(fd int, stat *Stat_t) (err error) = SYS_FSTAT64
+//sys	Fstatfs(fd int, stat *Statfs_t) (err error) = SYS_FSTATFS64
+//sys	Fsync(fd int) (err error)
+//sys	Ftruncate(fd int, length int64) (err error)
+//sys	Getdirentries(fd int, buf []byte, basep *uintptr) (n int, err error) = SYS_GETDIRENTRIES64
+//sys	Getdtablesize() (size int)
+//sysnb	Getegid() (egid int)
+//sysnb	Geteuid() (uid int)
+//sysnb	Getgid() (gid int)
+//sysnb	Getpgid(pid int) (pgid int, err error)
+//sysnb	Getpgrp() (pgrp int)
+//sysnb	Getpid() (pid int)
+//sysnb	Getppid() (ppid int)
+//sys	Getpriority(which int, who int) (prio int, err error)
+//sysnb	Getrlimit(which int, lim *Rlimit) (err error)
+//sysnb	Getrusage(who int, rusage *Rusage) (err error)
+//sysnb	Getsid(pid int) (sid int, err error)
+//sysnb	Getuid() (uid int)
+//sysnb	Issetugid() (tainted bool)
+//sys	Kqueue() (fd int, err error)
+//sys	Lchown(path string, uid int, gid int) (err error)
+//sys	Link(path string, link string) (err error)
+//sys	Listen(s int, backlog int) (err error)
+//sys	Lstat(path string, stat *Stat_t) (err error) = SYS_LSTAT64
+//sys	Mkdir(path string, mode uint32) (err error)
+//sys	Mkfifo(path string, mode uint32) (err error)
+//sys	Mknod(path string, mode uint32, dev int) (err error)
+//sys	Mlock(b []byte) (err error)
+//sys	Mlockall(flags int) (err error)
+//sys	Mprotect(b []byte, prot int) (err error)
+//sys	Munlock(b []byte) (err error)
+//sys	Munlockall() (err error)
+//sys	Open(path string, mode int, perm uint32) (fd int, err error)
+//sys	Pathconf(path string, name int) (val int, err error)
+//sys	Pread(fd int, p []byte, offset int64) (n int, err error)
+//sys	Pwrite(fd int, p []byte, offset int64) (n int, err error)
+//sys	read(fd int, p []byte) (n int, err error)
+//sys	Readlink(path string, buf []byte) (n int, err error)
+//sys	Rename(from string, to string) (err error)
+//sys	Revoke(path string) (err error)
+//sys	Rmdir(path string) (err error)
+//sys	Seek(fd int, offset int64, whence int) (newoffset int64, err error) = SYS_LSEEK
+//sys	Select(n int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (err error)
+//sys	Setegid(egid int) (err error)
+//sysnb	Seteuid(euid int) (err error)
+//sysnb	Setgid(gid int) (err error)
+//sys	Setlogin(name string) (err error)
+//sysnb	Setpgid(pid int, pgid int) (err error)
+//sys	Setpriority(which int, who int, prio int) (err error)
+//sys	Setprivexec(flag int) (err error)
+//sysnb	Setregid(rgid int, egid int) (err error)
+//sysnb	Setreuid(ruid int, euid int) (err error)
+//sysnb	Setrlimit(which int, lim *Rlimit) (err error)
+//sysnb	Setsid() (pid int, err error)
+//sysnb	Settimeofday(tp *Timeval) (err error)
+//sysnb	Setuid(uid int) (err error)
+//sys	Stat(path string, stat *Stat_t) (err error) = SYS_STAT64
+//sys	Statfs(path string, stat *Statfs_t) (err error) = SYS_STATFS64
+//sys	Symlink(path string, link string) (err error)
+//sys	Sync() (err error)
+//sys	Truncate(path string, length int64) (err error)
+//sys	Umask(newmask int) (oldmask int)
+//sys	Undelete(path string) (err error)
+//sys	Unlink(path string) (err error)
+//sys	Unmount(path string, flags int) (err error)
+//sys	write(fd int, p []byte) (n int, err error)
+//sys   mmap(addr uintptr, length uintptr, prot int, flag int, fd int, pos int64) (ret uintptr, err error)
+//sys   munmap(addr uintptr, length uintptr) (err error)
+//sys	readlen(fd int, buf *byte, nbuf int) (n int, err error) = SYS_READ
+//sys	writelen(fd int, buf *byte, nbuf int) (n int, err error) = SYS_WRITE
+
+/*
+ * Unimplemented
+ */
+// Profil
+// Sigaction
+// Sigprocmask
+// Getlogin
+// Sigpending
+// Sigaltstack
+// Ioctl
+// Reboot
+// Execve
+// Vfork
+// Sbrk
+// Sstk
+// Ovadvise
+// Mincore
+// Setitimer
+// Swapon
+// Select
+// Sigsuspend
+// Readv
+// Writev
+// Nfssvc
+// Getfh
+// Quotactl
+// Mount
+// Csops
+// Waitid
+// Add_profil
+// Kdebug_trace
+// Sigreturn
+// Mmap
+// Mlock
+// Munlock
+// Atsocket
+// Kqueue_from_portset_np
+// Kqueue_portset
+// Getattrlist
+// Setattrlist
+// Getdirentriesattr
+// Searchfs
+// Delete
+// Copyfile
+// Poll
+// Watchevent
+// Waitevent
+// Modwatch
+// Getxattr
+// Fgetxattr
+// Setxattr
+// Fsetxattr
+// Removexattr
+// Fremovexattr
+// Listxattr
+// Flistxattr
+// Fsctl
+// Initgroups
+// Posix_spawn
+// Nfsclnt
+// Fhopen
+// Minherit
+// Semsys
+// Msgsys
+// Shmsys
+// Semctl
+// Semget
+// Semop
+// Msgctl
+// Msgget
+// Msgsnd
+// Msgrcv
+// Shmat
+// Shmctl
+// Shmdt
+// Shmget
+// Shm_open
+// Shm_unlink
+// Sem_open
+// Sem_close
+// Sem_unlink
+// Sem_wait
+// Sem_trywait
+// Sem_post
+// Sem_getvalue
+// Sem_init
+// Sem_destroy
+// Open_extended
+// Umask_extended
+// Stat_extended
+// Lstat_extended
+// Fstat_extended
+// Chmod_extended
+// Fchmod_extended
+// Access_extended
+// Settid
+// Gettid
+// Setsgroups
+// Getsgroups
+// Setwgroups
+// Getwgroups
+// Mkfifo_extended
+// Mkdir_extended
+// Identitysvc
+// Shared_region_check_np
+// Shared_region_map_np
+// __pthread_mutex_destroy
+// __pthread_mutex_init
+// __pthread_mutex_lock
+// __pthread_mutex_trylock
+// __pthread_mutex_unlock
+// __pthread_cond_init
+// __pthread_cond_destroy
+// __pthread_cond_broadcast
+// __pthread_cond_signal
+// Setsid_with_pid
+// __pthread_cond_timedwait
+// Aio_fsync
+// Aio_return
+// Aio_suspend
+// Aio_cancel
+// Aio_error
+// Aio_read
+// Aio_write
+// Lio_listio
+// __pthread_cond_wait
+// Iopolicysys
+// Mlockall
+// Munlockall
+// __pthread_kill
+// __pthread_sigmask
+// __sigwait
+// __disable_threadsignal
+// __pthread_markcancel
+// __pthread_canceled
+// __semwait_signal
+// Proc_info
+// sendfile
+// Stat64_extended
+// Lstat64_extended
+// Fstat64_extended
+// __pthread_chdir
+// __pthread_fchdir
+// Audit
+// Auditon
+// Getauid
+// Setauid
+// Getaudit
+// Setaudit
+// Getaudit_addr
+// Setaudit_addr
+// Auditctl
+// Bsdthread_create
+// Bsdthread_terminate
+// Stack_snapshot
+// Bsdthread_register
+// Workq_open
+// Workq_ops
+// __mac_execve
+// __mac_syscall
+// __mac_get_file
+// __mac_set_file
+// __mac_get_link
+// __mac_set_link
+// __mac_get_proc
+// __mac_set_proc
+// __mac_get_fd
+// __mac_set_fd
+// __mac_get_pid
+// __mac_get_lcid
+// __mac_get_lctx
+// __mac_set_lctx
+// Setlcid
+// Read_nocancel
+// Write_nocancel
+// Open_nocancel
+// Close_nocancel
+// Wait4_nocancel
+// Recvmsg_nocancel
+// Sendmsg_nocancel
+// Recvfrom_nocancel
+// Accept_nocancel
+// Msync_nocancel
+// Fcntl_nocancel
+// Select_nocancel
+// Fsync_nocancel
+// Connect_nocancel
+// Sigsuspend_nocancel
+// Readv_nocancel
+// Writev_nocancel
+// Sendto_nocancel
+// Pread_nocancel
+// Pwrite_nocancel
+// Waitid_nocancel
+// Poll_nocancel
+// Msgsnd_nocancel
+// Msgrcv_nocancel
+// Sem_wait_nocancel
+// Aio_suspend_nocancel
+// __sigwait_nocancel
+// __semwait_signal_nocancel
+// __mac_mount
+// __mac_get_mount
+// __mac_getfsstat

vendor/golang.org/x/sys/unix/syscall_darwin_386.go

@@ -0,0 +1,77 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build 386,darwin
+
+package unix
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+func Getpagesize() int { return 4096 }
+
+func TimespecToNsec(ts Timespec) int64 { return int64(ts.Sec)*1e9 + int64(ts.Nsec) }
+
+func NsecToTimespec(nsec int64) (ts Timespec) {
+	ts.Sec = int32(nsec / 1e9)
+	ts.Nsec = int32(nsec % 1e9)
+	return
+}
+
+func NsecToTimeval(nsec int64) (tv Timeval) {
+	nsec += 999 // round up to microsecond
+	tv.Usec = int32(nsec % 1e9 / 1e3)
+	tv.Sec = int32(nsec / 1e9)
+	return
+}
+
+//sysnb	gettimeofday(tp *Timeval) (sec int32, usec int32, err error)
+func Gettimeofday(tv *Timeval) (err error) {
+	// The tv passed to gettimeofday must be non-nil
+	// but is otherwise unused.  The answers come back
+	// in the two registers.
+	sec, usec, err := gettimeofday(tv)
+	tv.Sec = int32(sec)
+	tv.Usec = int32(usec)
+	return err
+}
+
+func SetKevent(k *Kevent_t, fd, mode, flags int) {
+	k.Ident = uint32(fd)
+	k.Filter = int16(mode)
+	k.Flags = uint16(flags)
+}
+
+func (iov *Iovec) SetLen(length int) {
+	iov.Len = uint32(length)
+}
+
+func (msghdr *Msghdr) SetControllen(length int) {
+	msghdr.Controllen = uint32(length)
+}
+
+func (cmsg *Cmsghdr) SetLen(length int) {
+	cmsg.Len = uint32(length)
+}
+
+func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) {
+	var length = uint64(count)
+
+	_, _, e1 := Syscall9(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr(*offset>>32), uintptr(unsafe.Pointer(&length)), 0, 0, 0, 0)
+
+	written = int(length)
+
+	if e1 != 0 {
+		err = e1
+	}
+	return
+}
+
+func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno)
+
+// SYS___SYSCTL is used by syscall_bsd.go for all BSDs, but in modern versions
+// of darwin/386 the syscall is called sysctl instead of __sysctl.
+const SYS___SYSCTL = SYS_SYSCTL

vendor/golang.org/x/sys/unix/syscall_darwin_amd64.go

@@ -0,0 +1,79 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build amd64,darwin
+
+package unix
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+//sys	Fchmodat(dirfd int, path string, mode uint32, flags int) (err error)
+
+func Getpagesize() int { return 4096 }
+
+func TimespecToNsec(ts Timespec) int64 { return int64(ts.Sec)*1e9 + int64(ts.Nsec) }
+
+func NsecToTimespec(nsec int64) (ts Timespec) {
+	ts.Sec = nsec / 1e9
+	ts.Nsec = nsec % 1e9
+	return
+}
+
+func NsecToTimeval(nsec int64) (tv Timeval) {
+	nsec += 999 // round up to microsecond
+	tv.Usec = int32(nsec % 1e9 / 1e3)
+	tv.Sec = int64(nsec / 1e9)
+	return
+}
+
+//sysnb	gettimeofday(tp *Timeval) (sec int64, usec int32, err error)
+func Gettimeofday(tv *Timeval) (err error) {
+	// The tv passed to gettimeofday must be non-nil
+	// but is otherwise unused.  The answers come back
+	// in the two registers.
+	sec, usec, err := gettimeofday(tv)
+	tv.Sec = sec
+	tv.Usec = usec
+	return err
+}
+
+func SetKevent(k *Kevent_t, fd, mode, flags int) {
+	k.Ident = uint64(fd)
+	k.Filter = int16(mode)
+	k.Flags = uint16(flags)
+}
+
+func (iov *Iovec) SetLen(length int) {
+	iov.Len = uint64(length)
+}
+
+func (msghdr *Msghdr) SetControllen(length int) {
+	msghdr.Controllen = uint32(length)
+}
+
+func (cmsg *Cmsghdr) SetLen(length int) {
+	cmsg.Len = uint32(length)
+}
+
+func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) {
+	var length = uint64(count)
+
+	_, _, e1 := Syscall6(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr(unsafe.Pointer(&length)), 0, 0)
+
+	written = int(length)
+
+	if e1 != 0 {
+		err = e1
+	}
+	return
+}
+
+func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno)
+
+// SYS___SYSCTL is used by syscall_bsd.go for all BSDs, but in modern versions
+// of darwin/amd64 the syscall is called sysctl instead of __sysctl.
+const SYS___SYSCTL = SYS_SYSCTL

vendor/golang.org/x/sys/unix/syscall_darwin_arm.go

@@ -0,0 +1,71 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unix
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+func Getpagesize() int { return 4096 }
+
+func TimespecToNsec(ts Timespec) int64 { return int64(ts.Sec)*1e9 + int64(ts.Nsec) }
+
+func NsecToTimespec(nsec int64) (ts Timespec) {
+	ts.Sec = int32(nsec / 1e9)
+	ts.Nsec = int32(nsec % 1e9)
+	return
+}
+
+func NsecToTimeval(nsec int64) (tv Timeval) {
+	nsec += 999 // round up to microsecond
+	tv.Usec = int32(nsec % 1e9 / 1e3)
+	tv.Sec = int32(nsec / 1e9)
+	return
+}
+
+//sysnb	gettimeofday(tp *Timeval) (sec int32, usec int32, err error)
+func Gettimeofday(tv *Timeval) (err error) {
+	// The tv passed to gettimeofday must be non-nil
+	// but is otherwise unused.  The answers come back
+	// in the two registers.
+	sec, usec, err := gettimeofday(tv)
+	tv.Sec = int32(sec)
+	tv.Usec = int32(usec)
+	return err
+}
+
+func SetKevent(k *Kevent_t, fd, mode, flags int) {
+	k.Ident = uint32(fd)
+	k.Filter = int16(mode)
+	k.Flags = uint16(flags)
+}
+
+func (iov *Iovec) SetLen(length int) {
+	iov.Len = uint32(length)
+}
+
+func (msghdr *Msghdr) SetControllen(length int) {
+	msghdr.Controllen = uint32(length)
+}
+
+func (cmsg *Cmsghdr) SetLen(length int) {
+	cmsg.Len = uint32(length)
+}
+
+func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) {
+	var length = uint64(count)
+
+	_, _, e1 := Syscall9(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr(*offset>>32), uintptr(unsafe.Pointer(&length)), 0, 0, 0, 0)
+
+	written = int(length)
+
+	if e1 != 0 {
+		err = e1
+	}
+	return
+}
+
+func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) // sic

vendor/golang.org/x/sys/unix/syscall_darwin_arm64.go

@@ -0,0 +1,77 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64,darwin
+
+package unix
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+func Getpagesize() int { return 16384 }
+
+func TimespecToNsec(ts Timespec) int64 { return int64(ts.Sec)*1e9 + int64(ts.Nsec) }
+
+func NsecToTimespec(nsec int64) (ts Timespec) {
+	ts.Sec = nsec / 1e9
+	ts.Nsec = nsec % 1e9
+	return
+}
+
+func NsecToTimeval(nsec int64) (tv Timeval) {
+	nsec += 999 // round up to microsecond
+	tv.Usec = int32(nsec % 1e9 / 1e3)
+	tv.Sec = int64(nsec / 1e9)
+	return
+}
+
+//sysnb	gettimeofday(tp *Timeval) (sec int64, usec int32, err error)
+func Gettimeofday(tv *Timeval) (err error) {
+	// The tv passed to gettimeofday must be non-nil
+	// but is otherwise unused.  The answers come back
+	// in the two registers.
+	sec, usec, err := gettimeofday(tv)
+	tv.Sec = sec
+	tv.Usec = usec
+	return err
+}
+
+func SetKevent(k *Kevent_t, fd, mode, flags int) {
+	k.Ident = uint64(fd)
+	k.Filter = int16(mode)
+	k.Flags = uint16(flags)
+}
+
+func (iov *Iovec) SetLen(length int) {
+	iov.Len = uint64(length)
+}
+
+func (msghdr *Msghdr) SetControllen(length int) {
+	msghdr.Controllen = uint32(length)
+}
+
+func (cmsg *Cmsghdr) SetLen(length int) {
+	cmsg.Len = uint32(length)
+}
+
+func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) {
+	var length = uint64(count)
+
+	_, _, e1 := Syscall6(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr(unsafe.Pointer(&length)), 0, 0)
+
+	written = int(length)
+
+	if e1 != 0 {
+		err = e1
+	}
+	return
+}
+
+func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) // sic
+
+// SYS___SYSCTL is used by syscall_bsd.go for all BSDs, but in modern versions
+// of darwin/arm64 the syscall is called sysctl instead of __sysctl.
+const SYS___SYSCTL = SYS_SYSCTL