mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-05-05 11:21:58 -05:00
Code Issues Packages Projects Releases 159 Wiki Activity

fixing misspellings, fixing IneffAssign reported issues (#2436)

Browse Source 
also removing encode-perf-rig since codec-perf-rig is more current and real
This commit is contained in:
Mike Gray
2016-08-25 13:32:34 -04:00
committed by GitHub
parent 6a661361ad
commit 2f66e67763
17 changed files with 9 additions and 485 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/perf/codec-perf-rig/main.go
-7
View File
@@ -168,9 +168,7 @@ func buildListIncrementally(count uint64, createFn createValueFn) types.Collecti
}
func readList(c types.Collection) {
var outValue types.Value
c.(types.List).IterAll(func(v types.Value, idx uint64) {
outValue = v
})
}
@@ -193,9 +191,7 @@ func buildSetIncrementally(count uint64, createFn createValueFn) types.Collectio
}
func readSet(c types.Collection) {
var outValue types.Value
c.(types.Set).IterAll(func(v types.Value) {
v = outValue
})
}
@@ -219,9 +215,6 @@ func buildMapIncrementally(count uint64, createFn createValueFn) types.Collectio
}
func readMap(c types.Collection) {
var outKey, outValue types.Value
c.(types.Map).IterAll(func(k types.Value, v types.Value) {
outKey = k
outValue = v
})
}
go/perf/encode-perf-rig/.gitignore
-1
View File
@@ -1 +0,0 @@
encode-perf-rig
go/perf/encode-perf-rig/README.md
-57
View File
@@ -1,57 +0,0 @@
This is a performance test rig for testing encoding/decoding of math/big.Float numbers.
Initial silly test is string encoding vs. binary serialization:
For small numbers (<1m), the space saved by writing numbers as strings is pretty large.
Binary is faster no matter the size of the numbers.
As of June 3rd, these are the numbers I get on a MacBook Pro 2.7 GHz Intel Core i5.
```
$ ./run.sh
small numbers save a lot of space as strings
encoding: string from: 0 iterations: 100000
IO 489 kB (100,000 nums) in 11.206898552s (44 kB/s)
encoding: binary from: 0 iterations: 100000
IO 1.2 MB (100,000 nums) in 139.831671ms (8.6 MB/s)
once the numbers are larger than 1m, the space savings are gone
encoding: string from: 900000 to: 1000000 by: 1
IO 600 kB (100,000 nums) in 11.185759014s (54 kB/s)
encoding: string from: 1000000 to: 1100000 by: 1
IO 1.2 MB (100,000 nums) in 11.19562483s (106 kB/s)
encoding: binary from: 900000 to: 1000000 by: 1
IO 1.2 MB (100,000 nums) in 128.295162ms (9.4 MB/s)
encoding: binary from: 1000000 to: 1100000 by: 1
IO 1.2 MB (100,000 nums) in 132.097144ms (9.1 MB/s)
binary is always faster
encoding: string from: 0 to: 1000000 by: 2
IO 2.9 MB (500,000 nums) in 56.650322934s (52 kB/s)
encoding: binary from: 0 to: 1000000 by: 2
IO 6.0 MB (500,000 nums) in 718.012386ms (8.4 MB/s)
larger numbers are the same - binary still faster, but now strings are bigger than binary
encoding: string from: 1000000000000 to: 1000000001000 by: 2
IO 8.9 kB (500 nums) in 58.713475ms (151 kB/s)
encoding: binary from: 1000000000000 to: 1000000001000 by: 2
IO 6.0 kB (500 nums) in 877.172µs (6.8 MB/s)
if we special case non-floating numbers we get some space savings
encoding: binary from: 0 to: 10000000 by: 1
IO 120 MB (10,000,000 nums) in 14.025595909s (8.6 MB/s)
encoding: binary-int from: 0 to: 10000000 by: 1
IO 90 MB (10,000,000 nums) in 11.259321946s (8.0 MB/s)
then if we special case small integers we get even more space savings
encoding: binary-varint from: 0 to: 10000000 by: 1
IO 49 MB (10,000,000 nums) in 10.096619263s (4.8 MB/s)
```
TODO:
- Implement flag for setting precision
- Implement randomized numbers rather than in sequence incremental numbers
- Look at go package "encoding/gob", optimized for performance
- Implement read/write to/from file to connect to JS implementation
go/perf/encode-perf-rig/binary-encoder.go
-48
View File
@@ -1,48 +0,0 @@
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package main
import (
"encoding/binary"
"io"
"math/big"
)
type BinaryEncoderDecoder struct {
tmp *big.Float
}
func NewBinaryEncoderDecoder() BinaryEncoderDecoder {
bed := BinaryEncoderDecoder{}
bed.tmp = new(big.Float)
bed.tmp.SetPrec(ENCODER_DECODER_PREC)
return bed
}
func (bed BinaryEncoderDecoder) Encode(w io.Writer, n *big.Float) error {
exponent := n.MantExp(bed.tmp)
f, _ := bed.tmp.Float64()
if err := binary.Write(w, binary.BigEndian, f); err != nil {
return err
}
return binary.Write(w, binary.BigEndian, int32(exponent))
}
func (bed BinaryEncoderDecoder) Decode(r io.Reader, n *big.Float) error {
var f float64
var exponent int32
n.SetUint64(0)
if err := binary.Read(r, binary.BigEndian, &f); err != nil {
return err
}
if err := binary.Read(r, binary.BigEndian, &exponent); err != nil {
return err
}
bed.tmp.SetFloat64(f)
bed.tmp.SetPrec(ENCODER_DECODER_PREC)
n.SetMantExp(bed.tmp, int(exponent))
return nil
}
go/perf/encode-perf-rig/binary-int-encoder.go
-76
View File
@@ -1,76 +0,0 @@
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package main
import (
"encoding/binary"
"io"
"math/big"
)
type BinaryIntEncoderDecoder struct {
tmp *big.Float
}
func NewBinaryIntEncoderDecoder() BinaryIntEncoderDecoder {
bed := BinaryIntEncoderDecoder{}
bed.tmp = new(big.Float)
bed.tmp.SetPrec(ENCODER_DECODER_PREC)
return bed
}
func (bed BinaryIntEncoderDecoder) Encode(w io.Writer, n *big.Float) error {
if n.IsInt() {
x, _ := n.Int64()
// TODO - if accuracy is not Exact, then use the other path
if err := binary.Write(w, binary.BigEndian, int8(0)); err != nil {
return err
}
return binary.Write(w, binary.BigEndian, x)
} else {
if err := binary.Write(w, binary.BigEndian, int8(1)); err != nil {
return err
}
exponent := n.MantExp(bed.tmp)
f, _ := bed.tmp.Float64()
if err := binary.Write(w, binary.BigEndian, f); err != nil {
return err
}
return binary.Write(w, binary.BigEndian, int32(exponent))
}
}
func (bed BinaryIntEncoderDecoder) Decode(r io.Reader, n *big.Float) error {
var isInteger int8
var f float64
var exponent int32
n.SetUint64(0)
if err := binary.Read(r, binary.BigEndian, &isInteger); err != nil {
return err
}
if isInteger <= 0 {
var x int64
if err := binary.Read(r, binary.BigEndian, &x); err != nil {
return err
}
n.SetInt64(x)
n.SetPrec(ENCODER_DECODER_PREC)
return nil
} else {
if err := binary.Read(r, binary.BigEndian, &f); err != nil {
return err
}
if err := binary.Read(r, binary.BigEndian, &exponent); err != nil {
return err
}
bed.tmp.SetFloat64(f)
bed.tmp.SetPrec(ENCODER_DECODER_PREC)
n.SetMantExp(bed.tmp, int(exponent))
return nil
}
}
go/perf/encode-perf-rig/binary-varint-encoder.go
-92
View File
@@ -1,92 +0,0 @@
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package main
import (
"encoding/binary"
"io"
"math/big"
)
type BinaryVarintEncoderDecoder struct {
tmp *big.Float
}
func NewBinaryVarintEncoderDecoder() BinaryVarintEncoderDecoder {
bed := BinaryVarintEncoderDecoder{}
bed.tmp = new(big.Float)
bed.tmp.SetPrec(ENCODER_DECODER_PREC)
return bed
}
func (bed BinaryVarintEncoderDecoder) Encode(w io.Writer, n *big.Float) error {
if n.IsInt() {
x, _ := n.Int64()
// TODO - if accuracy is not Exact, then use the other path
buf := make([]byte, binary.MaxVarintLen64)
nBytes := binary.PutVarint(buf, x)
if _, err := w.Write([]byte{byte(0)}); err != nil {
return err
}
_, err := w.Write(buf[0:nBytes])
return err
} else {
if err := binary.Write(w, binary.BigEndian, int8(1)); err != nil {
return err
}
exponent := n.MantExp(bed.tmp)
f, _ := bed.tmp.Float64()
if err := binary.Write(w, binary.BigEndian, f); err != nil {
return err
}
return binary.Write(w, binary.BigEndian, int32(exponent))
}
}
func (bed BinaryVarintEncoderDecoder) Decode(r io.Reader, n *big.Float) error {
var isInteger int8
var f float64
var exponent int32
n.SetUint64(0)
if err := binary.Read(r, binary.BigEndian, &isInteger); err != nil {
return err
}
if isInteger <= 0 {
var x int64
var err error
if x, err = binary.ReadVarint(miniByteReader{r}); err != nil {
return err
}
n.SetInt64(x)
n.SetPrec(ENCODER_DECODER_PREC)
return nil
} else {
if err := binary.Read(r, binary.BigEndian, &f); err != nil {
return err
}
if err := binary.Read(r, binary.BigEndian, &exponent); err != nil {
return err
}
bed.tmp.SetFloat64(f)
bed.tmp.SetPrec(ENCODER_DECODER_PREC)
n.SetMantExp(bed.tmp, int(exponent))
return nil
}
}
type miniByteReader struct {
r io.Reader
}
func (r miniByteReader) ReadByte() (byte, error) {
p := []byte{byte(0)}
// io.Reader.Read(p []byte) (n int, err error)
// io.ByteReader.ReadByte() (c byte, err error)
_, err := r.r.Read(p)
return p[0], err
}
go/perf/encode-perf-rig/encoder-type.go
-19
View File
@@ -1,19 +0,0 @@
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package main
import (
"io"
"math/big"
)
const (
ENCODER_DECODER_PREC = 1024
)
type EncoderDecoder interface {
Encode(w io.Writer, n *big.Float) error // write n to w
Decode(r io.Reader, n *big.Float) error // read from r to set n
}
go/perf/encode-perf-rig/main.go
-113
View File
@@ -1,113 +0,0 @@
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package main
import (
"bytes"
"fmt"
"math/big"
"os"
"time"
humanize "github.com/dustin/go-humanize"
flag "github.com/juju/gnuflag"
)
// used to ensure all of the big.Floats end up with the same precision
func newBigFloat(n uint64) *big.Float {
tmp := new(big.Float).SetUint64(n)
tmp.SetPrec(ENCODER_DECODER_PREC)
return tmp
}
// helpful in debugging output
func stringOfBigFloat(n *big.Float) string {
return n.Text('g', -1)
}
func runTest(encoderDecoder EncoderDecoder, n *big.Float) (nBytes uint64) {
y := newBigFloat(0)
buf := new(bytes.Buffer)
err := encoderDecoder.Encode(buf, n)
if err != nil {
panic(err)
}
nBytes += uint64(buf.Len())
buf = bytes.NewBuffer(buf.Bytes())
err = encoderDecoder.Decode(buf, y)
nBytes += uint64(buf.Len())
if n.Cmp(y) != 0 {
panic(fmt.Sprintf("write and read are not the same: %v, %v - %d - %d, %d", stringOfBigFloat(n), stringOfBigFloat(y), nBytes, n.Prec(), y.Prec()))
}
return
}
func getEncoder(name string) EncoderDecoder {
if name == "string" {
return StringEncodedDecoder()
} else if name == "binary" {
return NewBinaryEncoderDecoder()
} else if name == "binary-int" {
return NewBinaryIntEncoderDecoder()
} else if name == "binary-varint" {
return NewBinaryVarintEncoderDecoder()
} else {
fmt.Printf("Unknown encoding specified: %s\n", name)
flag.PrintDefaults()
os.Exit(1)
return nil
}
}
func main() {
// use [from/to/by] or [from/iterations]
nFrom := flag.Uint64("from", 1e2, "start iterations from this number")
nTo := flag.Uint64("to", 1e4, "run iterations until arriving at this number")
nBy := flag.Uint64("by", 1, "increment each iteration by this number")
nIncrements := flag.Uint64("iterations", 0, "number of iterations to execute")
encodingType := flag.String("encoding", "string", "encode/decode as 'string', 'binary', 'binary-int', 'binary-varint'")
flag.Parse(true)
flag.Usage = func() {
fmt.Printf("%s\n", os.Args[0])
flag.PrintDefaults()
return
}
t0 := time.Now()
nBytes := uint64(0)
nIterations := uint64(0)
encoderDecoder := getEncoder(*encodingType)
startingLoop := newBigFloat(*nFrom)
var endLoop *big.Float
var incrementer *big.Float
if *nIncrements > 0 {
// using from/iterations flags
fmt.Printf("encoding: %v from: %v iterations: %v\n", *encodingType, *nFrom, *nIncrements)
incrementer = newBigFloat(1)
n := newBigFloat(*nIncrements)
endLoop = n.Add(n, startingLoop)
} else {
// using from/to/by flags
fmt.Printf("encoding: %v from: %v to: %v by: %v\n", *encodingType, *nFrom, *nTo, *nBy)
incrementer = newBigFloat(*nBy)
endLoop = newBigFloat(*nTo)
}
for i := startingLoop; i.Cmp(endLoop) < 0; i = i.Add(i, incrementer) {
nIterations++
nBytes += runTest(encoderDecoder, i)
}
t1 := time.Now()
d := t1.Sub(t0)
fmt.Printf("IO %s (%v nums) in %s (%s/s)\n", humanize.Bytes(nBytes), humanize.Comma(int64(nIterations)), d, humanize.Bytes(uint64(float64(nBytes)/d.Seconds())))
}
go/perf/encode-perf-rig/run.sh
-27
View File
@@ -1,27 +0,0 @@
#!/bin/sh
echo
echo small numbers save a lot of space as strings
./encode-perf-rig -encoding=string -from=0 -iterations=100000
./encode-perf-rig -encoding=binary -from=0 -iterations=100000
echo
echo once the numbers are larger than 1m, the space savings are gone
./encode-perf-rig -encoding=string -from=900000 -to=1000000
./encode-perf-rig -encoding=string -from=1000000 -to=1100000
./encode-perf-rig -encoding=binary -from=900000 -to=1000000
./encode-perf-rig -encoding=binary -from=1000000 -to=1100000
echo
echo binary is always faster
./encode-perf-rig -encoding=string -from=0 -to=1000000 -by=2
./encode-perf-rig -encoding=binary -from=0 -to=1000000 -by=2
echo
echo larger numbers are the same - binary still faster, but now strings are bigger than binary
./encode-perf-rig -encoding=string -from=1000000000000 -to=1000000001000 -by=2
./encode-perf-rig -encoding=binary -from=1000000000000 -to=1000000001000 -by=2
echo
echo if we special case non-floating numbers we get some space savings
./encode-perf-rig -encoding=binary -from=0 -to=10000000 -by=1
./encode-perf-rig -encoding=binary-int -from=0 -to=10000000 -by=1
echo
echo then if we special case small integers we get even more space savings
./encode-perf-rig -encoding=binary-varint -from=0 -to=10000000 -by=1
go/perf/encode-perf-rig/string-encoder.go
-36
View File
@@ -1,36 +0,0 @@
// Copyright 2016 Attic Labs, Inc. All rights reserved.
// Licensed under the Apache License, version 2.0:
// http://www.apache.org/licenses/LICENSE-2.0
package main
import (
"io"
"math/big"
)
type StringEncoderDecoder struct{}
func StringEncodedDecoder() StringEncoderDecoder {
return StringEncoderDecoder{}
}
func (sed StringEncoderDecoder) Encode(w io.Writer, n *big.Float) error {
// TODO - big.Float.MarshalText?
// TODO - big.Float.Append
str := []byte(n.Text('g', -1))
_, err := w.Write(str)
return err
}
func (sed StringEncoderDecoder) Decode(r io.Reader, n *big.Float) error {
n.SetFloat64(0)
n.SetPrec(ENCODER_DECODER_PREC)
buf := make([]byte, 256)
if _, err := r.Read(buf); err != nil {
return err
}
_, _, err := n.Parse(string(buf), 10)
return err
}
go/perf/suite/suite.go
+1 -1
View File
@@ -24,7 +24,7 @@
//
// Test results are written to Noms, along with a soup of the environment they were recorded in.
//
// Test names are derived from that "non-empty capitalized string": `"Test"` is ommitted because it's redundant, and leading digits are omitted to allow for manual test ordering. For example:
// Test names are derived from that "non-empty capitalized string": `"Test"` is omitted because it's redundant, and leading digits are omitted to allow for manual test ordering. For example:
//
// ```
// > cat ./samples/go/csv/csv-import/perf_test.go
go/types/incremental_test.go
+1 -1
View File
@@ -56,7 +56,7 @@ func TestIncrementalLoadList(t *testing.T) {
assert.Equal(expectedCount+chunkReads[i], cs.Reads)
// Do it again to make sure multiple derefs don't do multiple loads.
v = actual.Get(i)
_ = actual.Get(i)
assert.Equal(expectedCount+chunkReads[i], cs.Reads)
}
}
go/types/type_cache.go
+1 -1
View File
@@ -188,7 +188,7 @@ func resolveStructCycles(t *Type, parentStructTypes []*Type) *Type {
return t
}
// We normalize structs during their construction iff they have no unresolved cycles. Normalizing applies a canonical ordering to the composite types of a union and serializes all types under the struct. To ensure a consistent ordering of the composite types of a union, we generate a unique "order id" or OID for each of those types. The OID is the hash of a unique type encoding that is independant of the extant order of types within any subordinate unions. This encoding for most types is a straightforward serialization of its components; for unions the encoding is a bytewise XOR of the hashes of each of its composite type encodings.
// We normalize structs during their construction iff they have no unresolved cycles. Normalizing applies a canonical ordering to the composite types of a union and serializes all types under the struct. To ensure a consistent ordering of the composite types of a union, we generate a unique "order id" or OID for each of those types. The OID is the hash of a unique type encoding that is independent of the extant order of types within any subordinate unions. This encoding for most types is a straightforward serialization of its components; for unions the encoding is a bytewise XOR of the hashes of each of its composite type encodings.
// We require a consistent order of types within a union to ensure that equivalent types have a single persistent encoding and, therefore, a single hash. The method described above fails for "unrolled" cycles whereby two equivalent, but uniquely described structures, would have different OIDs. Consider for example the following two types that, while equivalent, do not yield the same OID:
// Struct A { a: Cycle<0> }
// Struct A { a: Struct A { a: Cycle<1> } }
samples/go/csv/kind_slice_test.go
+1
View File
@@ -24,5 +24,6 @@ func TestKindSliceJSON(t *testing.T) {
var uks KindSlice
err = json.Unmarshal(b, &uks)
assert.NoError(err, "error with json.Unmarshal")
assert.Equal(ks, uks)
}
samples/go/nomdex/parser.go
+1 -1
View File
@@ -178,7 +178,7 @@ func (s *qScanner) parseExpr(level int, im *indexManager) expr {
case rune(scanner.EOF):
return expr1
default:
tok = s.Scan()
_ = s.Scan()
}
default:
d.PanicIfError(fmt.Errorf("unexpected token in expr: %s, %d", s.TokenText(), tok))
samples/go/nomsfs/rw_test.go
+4
View File
@@ -32,6 +32,7 @@ func (s *fuseTestSuite) TestSimpleFile() {
file, code := testfs.Create("coconut", uint32(os.O_CREATE|os.O_RDWR), 0644, nil)
assert.Equal(s.T(), fuse.OK, code)
n, code := file.Write([]byte("Lime!"), 0)
assert.Equal(s.T(), fuse.OK, code)
assert.Equal(s.T(), uint32(5), n)
assertAttr(s, testfs, "coconut", 0644|fuse.S_IFREG, 5)
@@ -70,6 +71,7 @@ func (s *fuseTestSuite) TestBigFile() {
assert.Equal(s.T(), fuse.OK, code)
n, code := file.Write(buf.Bytes(), 0)
assert.Equal(s.T(), fuse.OK, code)
assert.Equal(s.T(), uint32(size), n)
assertAttr(s, testfs, "shining.txt", 0644|fuse.S_IFREG, size)
}
@@ -86,9 +88,11 @@ func (s *fuseTestSuite) TestOverwrite() {
assert.Equal(s.T(), fuse.OK, code)
n, code := file.Write([]byte("Four score and seven years ago..."), 0)
assert.Equal(s.T(), fuse.OK, code)
assert.Equal(s.T(), uint32(33), n)
assertAttr(s, testfs, "proclamation", 0644|fuse.S_IFREG, 33)
n, code = file.Write([]byte("beers"), 21)
assert.Equal(s.T(), fuse.OK, code)
assert.Equal(s.T(), uint32(5), n)
assertAttr(s, testfs, "proclamation", 0644|fuse.S_IFREG, 33)
tools/run_all_build.go
-5
View File
@@ -39,11 +39,6 @@ func main() {
goPath := os.Getenv("GOPATH")
d.PanicIfTrue(goPath == "", "GOPATH must be set!")
workspace := os.Getenv("WORKSPACE")
if workspace == "" {
fmt.Printf("WORKSPACE not set in environment; using GOPATH (%s).\n", goPath)
workspace = goPath
}
pythonPath := filepath.Join(goPath, nomsCheckoutPath, "tools")
env := runner.Env{
"PYTHONPATH": pythonPath,