From 8d64893d4fb0227a7d46b26462c3126dfd9382ae Mon Sep 17 00:00:00 2001 From: jennifersp Date: Tue, 16 Nov 2021 16:37:16 -0800 Subject: [PATCH 01/15] support parquet export --- go/go.mod | 14 +- go/go.sum | 101 ++++++++++++-- go/libraries/doltcore/mvdata/data_loc.go | 7 + go/libraries/doltcore/mvdata/file_data_loc.go | 5 + .../doltcore/table/typed/parquet/writer.go | 129 ++++++++++++++++++ 5 files changed, 243 insertions(+), 13 deletions(-) create mode 100644 go/libraries/doltcore/table/typed/parquet/writer.go diff --git a/go/go.mod b/go/go.mod index 9ee56f70ae..c9c43571b9 100644 --- a/go/go.mod +++ b/go/go.mod @@ -35,7 +35,7 @@ require ( github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.1 - github.com/google/go-cmp v0.5.5 + github.com/google/go-cmp v0.5.6 github.com/google/uuid v1.2.0 github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/jedib0t/go-pretty v4.3.1-0.20191104025401-85fe5d6a7c4d+incompatible @@ -65,7 +65,7 @@ require ( github.com/uber/jaeger-lib v2.4.0+incompatible // indirect go.mongodb.org/mongo-driver v1.7.0 // indirect go.uber.org/zap v1.15.0 - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a golang.org/x/net v0.0.0-20210505214959-0714010a04ed golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 @@ -79,9 +79,15 @@ require ( gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect ) +require ( + github.com/xitongsys/parquet-go v1.6.1 + github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371 +) + require ( cloud.google.com/go v0.66.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-ole/go-ole v1.2.1 // indirect @@ -89,10 +95,12 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jmespath/go-jmespath v0.3.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/klauspost/compress v1.10.10 // indirect github.com/mattn/go-colorable v0.1.7 // indirect github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 // indirect + github.com/pierrec/lz4/v4 v4.1.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/src-d/go-oniguruma v1.1.0 // indirect diff --git a/go/go.sum b/go/go.sum index 8de7d479ae..c4997e4833 100755 --- a/go/go.sum +++ b/go/go.sum @@ -36,6 +36,14 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -72,8 +80,12 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw= +github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -86,9 +98,23 @@ github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible/go.mod h1:Cp18FeDCvsK+cD2QAGkqerGjrgSXLiJWnjHeY2mneBc= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.32.6 h1:HoswAabUWgnrUF7X/9dr4WRgrr8DyscxXvTDm7Qw/5c= github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250= +github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA= +github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2/go.mod h1:qaqQiHSrOUVOfKe6fhgQ6UzhxjwqVW8aHNegd6Ws4w4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.1/go.mod h1:v33JQ57i2nekYTA70Mb+O18KeH4KqhdqxTJZNK1zdRE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1/go.mod h1:zceowr5Z1Nh2WVP8bf/3ikB41IZW59E4yIYbg+pC6mw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.1/go.mod h1:6EQZIwNNvHpq/2/QSJnp4+ECvqIy55w95Ofs0ze+nGQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1/go.mod h1:XLAGFrEjbvMCLvAtWLLP32yTv8GpBquCApZEycDLunI= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.1/go.mod h1:J3A3RGUvuCZjvSuZEcOpHDnzZP/sKbhDWV2T1EOzFIM= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BVRASvcU7gYZB9PUgPiByXg= +github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/bcicen/jstream v1.0.0 h1:gOi+Sn9mHrpePlENynPKA6Dra/PjLaIpqrTevhfvLAA= github.com/bcicen/jstream v1.0.0/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -119,6 +145,7 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/blake2 v0.0.0-20150924215134-8d10d0420cbf h1:5ZeQB3mThuz5C2MSER6T5GdtXTF9CMMk42F9BOyRsEQ= github.com/codahale/blake2 v0.0.0-20150924215134-8d10d0420cbf/go.mod h1:BO2rLUAZMrpgh6GBVKi0Gjdqw2MgCtJrtmUdDeZRKjY= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -144,8 +171,6 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= -github.com/dolthub/go-mysql-server v0.11.1-0.20211112165413-5ca77bc1ed96 h1:RpezxzOFiQ9AwWrx++hHxvf2ErfQ1rThN7DgCQt13B4= -github.com/dolthub/go-mysql-server v0.11.1-0.20211112165413-5ca77bc1ed96/go.mod h1:qMlUEq8TvoaDqyMQkA7zQYLtBulk5WWXP5B19texrJI= github.com/dolthub/go-mysql-server v0.11.1-0.20211113070552-b67f2dc822ad h1:wO7j7ZUK/oKQ3lIVHN+k0HcoJDsshxs6iaQy5dS3AOY= github.com/dolthub/go-mysql-server v0.11.1-0.20211113070552-b67f2dc822ad/go.mod h1:+TPkBSbbldwRlM8FYMmaCFUz6mNZSGbZe8F94eM0mgw= github.com/dolthub/ishell v0.0.0-20210205014355-16a4ce758446 h1:0ol5pj+QlKUKAtqs1LiPM3ZJKs+rHPgLSsMXmhTrCAM= @@ -156,9 +181,6 @@ github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66 h1:WRPDbpJWEnPxP github.com/dolthub/mmap-go v1.0.4-0.20201107010347-f9f2a9588a66/go.mod h1:N5ZIbMGuDUpTpOFQ7HcsN6WSIpTGQjHP+Mz27AfmAgk= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9XGFa6q5Ap4Z/OhNkAMBaK5YeuEzwJt+NZdhiE= github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY= -github.com/dolthub/vitess v0.0.0-20211108230711-9b4006809d3b/go.mod h1:hUE8oSk2H5JZnvtlLBhJPYC8WZCA5AoSntdLTcBvdBM= -github.com/dolthub/vitess v0.0.0-20211109181029-7a9e87e07665 h1:ivwxM+LRDnWUqvR8csxtnZvemiCPBy3VI+rL3/HRoPE= -github.com/dolthub/vitess v0.0.0-20211109181029-7a9e87e07665/go.mod h1:hUE8oSk2H5JZnvtlLBhJPYC8WZCA5AoSntdLTcBvdBM= github.com/dolthub/vitess v0.0.0-20211112215933-1feb51c6c461 h1:LpF6eRWczB7YtF9T69K219Ddq/I6/n1S3b/t9Bj4+40= github.com/dolthub/vitess v0.0.0-20211112215933-1feb51c6c461/go.mod h1:hUE8oSk2H5JZnvtlLBhJPYC8WZCA5AoSntdLTcBvdBM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -180,6 +202,7 @@ github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:rZfgFAXFS/z/lEd6LJmf9HVZ1LkgYiHx5pHhV5DR16M= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -253,6 +276,7 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -269,6 +293,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -313,8 +338,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -352,6 +379,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -364,6 +392,7 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -379,17 +408,22 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/iancoleman/strcase v0.1.3/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jedib0t/go-pretty v4.3.1-0.20191104025401-85fe5d6a7c4d+incompatible h1:SwOdF+2qzbZnEUsoEv1v0VkoQvoQ2pZLVDjNDzL6nto= github.com/jedib0t/go-pretty v4.3.1-0.20191104025401-85fe5d6a7c4d+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 h1:lrdPtrORjGv1HbbEvKWDUAy97mPpFm4B8hp77tcCUJY= github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= @@ -412,15 +446,22 @@ github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0Lh github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 h1:l6Y3mFnF46A+CeZsTrT8kVIuhayq1266oxWpDKE7hnQ= github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6/go.mod h1:UtDV9qK925GVmbdjR+e1unqoo+wGWNHHC6XB1Eu6wpE= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -436,6 +477,7 @@ github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lyft/protoc-gen-star v0.5.2/go.mod h1:9toiA3cC7z5uVbODF7kEQ91Xn7XNFkVUl+SrEe+ZORU= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -448,6 +490,7 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -496,7 +539,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8= +github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/exhaustive v0.0.0-20200811152831-6cf413ae40e0/go.mod h1:wBEpHwM2OdmeNpdCvRPUlkEbBuaFmcK4Wv8Q7FuGW3c= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -524,13 +568,17 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.6 h1:ueMTcBBFrbT8K4uGDNNZPa8Z7LtPV7Cl0TDjaeHxP44= +github.com/pierrec/lz4/v4 v4.1.6/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -538,6 +586,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug= github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.0/go.mod h1:41g+FIPlQUTDCveupEmEA65IoiQFrtgCeDopC4ajGIM= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -570,6 +620,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -614,6 +665,8 @@ github.com/sourcegraph/go-diff v0.6.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag07 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= @@ -636,6 +689,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -676,6 +730,13 @@ github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= +github.com/xitongsys/parquet-go v1.6.1 h1:F1snhlfL5U1hC1yE7Op8qLWFIZEzqmM46pCEspu9OC0= +github.com/xitongsys/parquet-go v1.6.1/go.mod h1:oNMMTE7vxZkSeV4Y6Q+DSpnR50DdpFM6Jx2CRav1OHI= +github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= +github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371 h1:RfGiOP/lWKBeNgpXmCeandYGV4pAnZsl42kX50p1UgE= +github.com/xitongsys/parquet-go-source v0.0.0-20211010230925-397910c5e371/go.mod h1:qLb2Itmdcp7KPa5KZKvhE9U1q5bYSOmgeOckF/H2rQA= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -711,6 +772,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -718,10 +780,14 @@ golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaE golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -777,6 +843,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190926025831-c00fd9afed17/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -795,6 +862,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210505214959-0714010a04ed h1:V9kAVxLvz1lkufatrpHuUVyJ/5tR3Ms7rk951P4mI98= golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -844,6 +912,7 @@ golang.org/x/sys v0.0.0-20190926180325-855e68c8590b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -876,6 +945,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 h1:cdsMqa2nXzqlgs183pHxtvoVwU7CyzaCTAUOg94af4c= golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -952,6 +1022,7 @@ golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200519015757-0d0afa43d58a/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200701041122-1837592efa10/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -964,6 +1035,7 @@ golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1025,6 +1097,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -1058,6 +1131,7 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= @@ -1079,13 +1153,19 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -1096,6 +1176,7 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= diff --git a/go/libraries/doltcore/mvdata/data_loc.go b/go/libraries/doltcore/mvdata/data_loc.go index 370c546471..6f663f6888 100644 --- a/go/libraries/doltcore/mvdata/data_loc.go +++ b/go/libraries/doltcore/mvdata/data_loc.go @@ -55,6 +55,9 @@ const ( // SqlFile is the format of a data location that is a .sql file SqlFile DataFormat = ".sql" + + // ParquetFile is the format of a data location that is a .paquet file + ParquetFile DataFormat = ".parquet" ) // ReadableStr returns a human readable string for a DataFormat @@ -72,6 +75,8 @@ func (df DataFormat) ReadableStr() string { return "json file" case SqlFile: return "sql file" + case ParquetFile: + return "parquet file" default: return "invalid" } @@ -124,6 +129,8 @@ func NewDataLocation(path, fileFmtStr string) DataLocation { dataFmt = JsonFile case string(SqlFile): dataFmt = SqlFile + case string(ParquetFile): + dataFmt = ParquetFile } } } diff --git a/go/libraries/doltcore/mvdata/file_data_loc.go b/go/libraries/doltcore/mvdata/file_data_loc.go index 1ad821077e..129c7198ed 100644 --- a/go/libraries/doltcore/mvdata/file_data_loc.go +++ b/go/libraries/doltcore/mvdata/file_data_loc.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/parquet" "io" "os" "strings" @@ -47,6 +48,8 @@ func DFFromString(dfStr string) DataFormat { return JsonFile case "sql", ".sql": return SqlFile + case "parquet", ".parquet": + return ParquetFile default: return InvalidDataFormat } @@ -157,6 +160,8 @@ func (dl FileDataLocation) NewCreatingWriter(ctx context.Context, mvOpts DataMov return json.NewJSONWriter(wr, outSch) case SqlFile: return sqlexport.OpenSQLExportWriter(ctx, wr, root, mvOpts.SrcName(), outSch, opts) + case ParquetFile: + return parquet.NewParquetWriter(wr, outSch, mvOpts.SrcName(), mvOpts.DestName()) } panic("Invalid Data Format." + string(dl.Format)) diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go new file mode 100644 index 0000000000..a1ada5160b --- /dev/null +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -0,0 +1,129 @@ +// Copyright 2019 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parquet + +import ( + "context" + "fmt" + "github.com/dolthub/dolt/go/libraries/doltcore/row" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" + "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/source" + "github.com/xitongsys/parquet-go/writer" + "io" +) + +var WriteBufSize = 256 * 1024 + +type ParquetWriter struct { + closer source.ParquetFile + pwriter *writer.CSVWriter + sch schema.Schema +} + +func NewParquetWriter(wr io.WriteCloser, outSch schema.Schema, srcName string, destName string) (*ParquetWriter, error) { + allCols := outSch.GetAllCols() + columns := allCols.GetColumns() + + ColTypeMap := getTypeMap() + var csvSchema []string + for _, col := range columns { + colName := col.Name + colType := col.TypeInfo.GetTypeIdentifier() + csvSchema = append(csvSchema, fmt.Sprintf("name=%s, %s", colName, ColTypeMap[colType])) + } + + fw, err := local.NewLocalFileWriter(destName) + if err != nil { + return nil, err + } + + // np configures the degree of concurrency for our Reader and Writers + // TODO: not sure what default value to set 'np' to + pw, err := writer.NewCSVWriter(csvSchema, fw, 4) + if err != nil { + return nil, err + } + + return &ParquetWriter{closer: fw, pwriter: pw, sch: outSch}, nil +} + +func (pwr *ParquetWriter) GetSchema() schema.Schema { + return pwr.sch +} + +// WriteRow will write a row to a table +func (pwr *ParquetWriter) WriteRow(ctx context.Context, r row.Row) error { + allCols := pwr.sch.GetAllCols() + colValStrs := make([]*string, allCols.Size()) + + sqlRow, err := sqlutil.DoltRowToSqlRow(r, pwr.GetSchema()) + if err != nil { + return err + } + + for i, val := range sqlRow { + if val == nil { + colValStrs[i] = nil + } else { + v := sqlutil.SqlColToStr(ctx, val) + colValStrs[i] = &v + } + } + + err = pwr.pwriter.WriteString(colValStrs) + if err != nil { + return err + } + return nil +} + +// Close should flush all writes, release resources being held +func (pwr *ParquetWriter) Close(ctx context.Context) error { + // WriteStop writes footer, stops writing and flushes + err := pwr.pwriter.WriteStop() + if err != nil{ + return err + } + pwr.closer.Close() + return nil +} + +func getTypeMap() map[typeinfo.Identifier]string { + + typeMap := map[typeinfo.Identifier]string{ + typeinfo.DatetimeTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.DecimalTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.EnumTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.InlineBlobTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.SetTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.TimeTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.TupleTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.UuidTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.VarBinaryTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.YearTypeIdentifier : `type=BYTE_ARRAY`, + + typeinfo.BitTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.BoolTypeIdentifier : `type=BOOLEAN`, + typeinfo.VarStringTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.UintTypeIdentifier : `type=INT64, convertedtype=UINT_64`, + typeinfo.IntTypeIdentifier : `type=INT64, convertedtype=INT_64`, + typeinfo.FloatTypeIdentifier : `type=DOUBLE`, + } + + return typeMap +} \ No newline at end of file From f8234517355172cc0ccd8160f6037af182df0ab9 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Wed, 17 Nov 2021 00:43:02 +0000 Subject: [PATCH 02/15] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/Godeps/LICENSES | 797 ++++++++++++++++++ go/libraries/doltcore/mvdata/file_data_loc.go | 3 +- .../doltcore/table/typed/parquet/writer.go | 50 +- 3 files changed, 825 insertions(+), 25 deletions(-) diff --git a/go/Godeps/LICENSES b/go/Godeps/LICENSES index bd523d65e8..dd8662fddc 100644 --- a/go/Godeps/LICENSES +++ b/go/Godeps/LICENSES @@ -981,6 +981,319 @@ SOFTWARE. = LICENSE 78b3d88d4101969f3415353aea94c9f4cce7ca238b012137e57e0273 = ================================================================================ +================================================================================ += github.com/apache/thrift licensed under: = + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------- +SOFTWARE DISTRIBUTED WITH THRIFT: + +The Apache Thrift software includes a number of subcomponents with +separate copyright notices and license terms. Your use of the source +code for the these subcomponents is subject to the terms and +conditions of the following licenses. + +-------------------------------------------------- +Portions of the following files are licensed under the MIT License: + + lib/erl/src/Makefile.am + +Please see doc/otp-base-license.txt for the full terms of this license. + +-------------------------------------------------- +For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: + +# Copyright (c) 2007 Thomas Porschberg +# +# Copying and distribution of this file, with or without +# modification, are permitted in any medium without royalty provided +# the copyright notice and this notice are preserved. + +-------------------------------------------------- +For the lib/nodejs/lib/thrift/json_parse.js: + +/* + json_parse.js + 2015-05-02 + Public Domain. + NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. + +*/ +(By Douglas Crockford ) + +-------------------------------------------------- +For lib/cpp/src/thrift/windows/SocketPair.cpp + +/* socketpair.c + * Copyright 2007 by Nathan C. Myers ; some rights reserved. + * This code is Free Software. It may be copied freely, in original or + * modified form, subject only to the restrictions that (1) the author is + * relieved from all responsibilities for any use for any purpose, and (2) + * this copyright notice must be retained, unchanged, in its entirety. If + * for any reason the author might be held responsible for any consequences + * of copying or use, license is withheld. + */ + + +-------------------------------------------------- +For lib/py/compat/win32/stdint.h + +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2008 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. The name of the author may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + + +-------------------------------------------------- +Codegen template in t_html_generator.h + +* Bootstrap v2.0.3 +* +* Copyright 2012 Twitter, Inc +* Licensed under the Apache License v2.0 +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Designed and built with all the love in the world @twitter by @mdo and @fat. + +--------------------------------------------------- +For t_cl_generator.cc + + * Copyright (c) 2008- Patrick Collison + * Copyright (c) 2006- Facebook + +--------------------------------------------------- + += LICENSE 42c3d433e7147c55c3974e209eca70f89310d4fa5feb6a0f162df904 = +================================================================================ + ================================================================================ = github.com/asaskevich/govalidator licensed under: = @@ -4279,6 +4592,41 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. = LICENSE 03d3e3813a51a67f46a07063e39f5d6451619f4ede2ae1c4c4cccb9e = ================================================================================ +================================================================================ += github.com/klauspost/compress licensed under: = + +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2019 Klaus Post. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + += LICENSE b6f05c0b9dba1bb0e91a34890d46579fbbc42d0b80de9ce3194d06f3 = +================================================================================ + ================================================================================ = github.com/lestrrat-go/strftime licensed under: = @@ -4643,6 +4991,41 @@ THE SOFTWARE. = LICENSE 1044f2bd0e8ae330197ea5c7e20a86bc3c960e9b96f1f6bb0f5f8ae1 = ================================================================================ +================================================================================ += github.com/pierrec/lz4/v4 licensed under: = + +Copyright (c) 2015, Pierre Curto +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of xxHash nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + += LICENSE 49eaff8bb6d3372c260d19f33798c6de96dae7278c9d0d2ddef68c13 = +================================================================================ + ================================================================================ = github.com/pkg/errors licensed under: = @@ -5598,6 +5981,420 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. = LICENSE 41cbff0d41b7d20dd9d70de1e0380fdca6ec1f42d2533c75c5c1bec3 = ================================================================================ +================================================================================ += github.com/xitongsys/parquet-go licensed under: = + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Xitong Zhang + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. += LICENSE 962c42d65cf9c1ea95cab890eaf9f17e3c20d9b39d41ac9957c0dee8 = +================================================================================ + +================================================================================ += github.com/xitongsys/parquet-go-source licensed under: = + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Xitong Zhang + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. += LICENSE 962c42d65cf9c1ea95cab890eaf9f17e3c20d9b39d41ac9957c0dee8 = +================================================================================ + ================================================================================ = go.mongodb.org/mongo-driver licensed under: = diff --git a/go/libraries/doltcore/mvdata/file_data_loc.go b/go/libraries/doltcore/mvdata/file_data_loc.go index 129c7198ed..7cba0dbe8a 100644 --- a/go/libraries/doltcore/mvdata/file_data_loc.go +++ b/go/libraries/doltcore/mvdata/file_data_loc.go @@ -18,11 +18,12 @@ import ( "context" "errors" "fmt" - "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/parquet" "io" "os" "strings" + "github.com/dolthub/dolt/go/libraries/doltcore/table/typed/parquet" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/table" diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go index a1ada5160b..26e382af29 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer.go +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -17,22 +17,24 @@ package parquet import ( "context" "fmt" + "io" + + "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/source" + "github.com/xitongsys/parquet-go/writer" + "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" - "github.com/xitongsys/parquet-go-source/local" - "github.com/xitongsys/parquet-go/source" - "github.com/xitongsys/parquet-go/writer" - "io" ) var WriteBufSize = 256 * 1024 type ParquetWriter struct { - closer source.ParquetFile - pwriter *writer.CSVWriter - sch schema.Schema + closer source.ParquetFile + pwriter *writer.CSVWriter + sch schema.Schema } func NewParquetWriter(wr io.WriteCloser, outSch schema.Schema, srcName string, destName string) (*ParquetWriter, error) { @@ -96,7 +98,7 @@ func (pwr *ParquetWriter) WriteRow(ctx context.Context, r row.Row) error { func (pwr *ParquetWriter) Close(ctx context.Context) error { // WriteStop writes footer, stops writing and flushes err := pwr.pwriter.WriteStop() - if err != nil{ + if err != nil { return err } pwr.closer.Close() @@ -106,24 +108,24 @@ func (pwr *ParquetWriter) Close(ctx context.Context) error { func getTypeMap() map[typeinfo.Identifier]string { typeMap := map[typeinfo.Identifier]string{ - typeinfo.DatetimeTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.DecimalTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.EnumTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.InlineBlobTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.SetTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.TimeTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.TupleTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.UuidTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.VarBinaryTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.YearTypeIdentifier : `type=BYTE_ARRAY`, + typeinfo.DatetimeTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.DecimalTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.EnumTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.InlineBlobTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.SetTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.TimeTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.TupleTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.UuidTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.VarBinaryTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.YearTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.BitTypeIdentifier : `type=BYTE_ARRAY`, - typeinfo.BoolTypeIdentifier : `type=BOOLEAN`, + typeinfo.BitTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.BoolTypeIdentifier: `type=BOOLEAN`, typeinfo.VarStringTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.UintTypeIdentifier : `type=INT64, convertedtype=UINT_64`, - typeinfo.IntTypeIdentifier : `type=INT64, convertedtype=INT_64`, - typeinfo.FloatTypeIdentifier : `type=DOUBLE`, + typeinfo.UintTypeIdentifier: `type=INT64, convertedtype=UINT_64`, + typeinfo.IntTypeIdentifier: `type=INT64, convertedtype=INT_64`, + typeinfo.FloatTypeIdentifier: `type=DOUBLE`, } return typeMap -} \ No newline at end of file +} From 104dc5646cd941040b2913129b450e0162b9faa8 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 18 Nov 2021 14:46:31 -0800 Subject: [PATCH 03/15] added tests for parquetwriter --- go/libraries/doltcore/mvdata/file_data_loc.go | 2 +- .../doltcore/table/typed/parquet/writer.go | 48 +++--- .../table/typed/parquet/writer_test.go | 154 ++++++++++++++++++ integration-tests/bats/export-tables.bats | 45 +++++ 4 files changed, 227 insertions(+), 22 deletions(-) create mode 100644 go/libraries/doltcore/table/typed/parquet/writer_test.go diff --git a/go/libraries/doltcore/mvdata/file_data_loc.go b/go/libraries/doltcore/mvdata/file_data_loc.go index 7cba0dbe8a..b5f072eab6 100644 --- a/go/libraries/doltcore/mvdata/file_data_loc.go +++ b/go/libraries/doltcore/mvdata/file_data_loc.go @@ -162,7 +162,7 @@ func (dl FileDataLocation) NewCreatingWriter(ctx context.Context, mvOpts DataMov case SqlFile: return sqlexport.OpenSQLExportWriter(ctx, wr, root, mvOpts.SrcName(), outSch, opts) case ParquetFile: - return parquet.NewParquetWriter(wr, outSch, mvOpts.SrcName(), mvOpts.DestName()) + return parquet.NewParquetWriter(outSch, mvOpts.DestName()) } panic("Invalid Data Format." + string(dl.Format)) diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go index 26e382af29..b91e9ff5c5 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer.go +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -17,9 +17,8 @@ package parquet import ( "context" "fmt" - "io" - "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/source" "github.com/xitongsys/parquet-go/writer" @@ -32,21 +31,27 @@ import ( var WriteBufSize = 256 * 1024 type ParquetWriter struct { - closer source.ParquetFile - pwriter *writer.CSVWriter - sch schema.Schema + filewriter source.ParquetFile + pwriter *writer.CSVWriter + sch schema.Schema } -func NewParquetWriter(wr io.WriteCloser, outSch schema.Schema, srcName string, destName string) (*ParquetWriter, error) { +func NewParquetWriter(outSch schema.Schema, destName string) (*ParquetWriter, error) { allCols := outSch.GetAllCols() columns := allCols.GetColumns() ColTypeMap := getTypeMap() var csvSchema []string + var repetitionType string for _, col := range columns { colName := col.Name colType := col.TypeInfo.GetTypeIdentifier() - csvSchema = append(csvSchema, fmt.Sprintf("name=%s, %s", colName, ColTypeMap[colType])) + if col.IsNullable() { + repetitionType = `, repetitiontype=OPTIONAL` + } else { + repetitionType = `` + } + csvSchema = append(csvSchema, fmt.Sprintf("name=%s, %s%s", colName, ColTypeMap[colType], repetitionType)) } fw, err := local.NewLocalFileWriter(destName) @@ -56,12 +61,12 @@ func NewParquetWriter(wr io.WriteCloser, outSch schema.Schema, srcName string, d // np configures the degree of concurrency for our Reader and Writers // TODO: not sure what default value to set 'np' to - pw, err := writer.NewCSVWriter(csvSchema, fw, 4) + pw, err := writer.NewCSVWriter(csvSchema, fw, int64(WriteBufSize)) if err != nil { return nil, err } - return &ParquetWriter{closer: fw, pwriter: pw, sch: outSch}, nil + return &ParquetWriter{filewriter: fw, pwriter: pw, sch: outSch}, nil } func (pwr *ParquetWriter) GetSchema() schema.Schema { @@ -101,27 +106,28 @@ func (pwr *ParquetWriter) Close(ctx context.Context) error { if err != nil { return err } - pwr.closer.Close() + pwr.filewriter.Close() return nil } func getTypeMap() map[typeinfo.Identifier]string { typeMap := map[typeinfo.Identifier]string{ - typeinfo.DatetimeTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.DecimalTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.EnumTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.InlineBlobTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.SetTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.TimeTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.TupleTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.UuidTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.VarBinaryTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.YearTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.DatetimeTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.DecimalTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.EnumTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.InlineBlobTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.SetTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.TimeTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.TupleTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.UuidTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.VarBinaryTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.YearTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, + typeinfo.BlobStringTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, typeinfo.BitTypeIdentifier: `type=BYTE_ARRAY`, typeinfo.BoolTypeIdentifier: `type=BOOLEAN`, - typeinfo.VarStringTypeIdentifier: `type=BYTE_ARRAY`, + typeinfo.VarStringTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, typeinfo.UintTypeIdentifier: `type=INT64, convertedtype=UINT_64`, typeinfo.IntTypeIdentifier: `type=INT64, convertedtype=INT_64`, typeinfo.FloatTypeIdentifier: `type=DOUBLE`, diff --git a/go/libraries/doltcore/table/typed/parquet/writer_test.go b/go/libraries/doltcore/table/typed/parquet/writer_test.go new file mode 100644 index 0000000000..644e9b5b27 --- /dev/null +++ b/go/libraries/doltcore/table/typed/parquet/writer_test.go @@ -0,0 +1,154 @@ +// Copyright 2019 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parquet + +import ( + "context" + "fmt" + "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/reader" + "io/ioutil" + "os" + "testing" + + "github.com/dolthub/dolt/go/libraries/doltcore/row" + "github.com/dolthub/dolt/go/libraries/doltcore/schema" + "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" + "github.com/dolthub/dolt/go/store/types" +) + +const ( + nameColName = "name" + ageColName = "age" + titleColName = "title" + nameColTag = 0 + ageColTag = 1 + titleColTag = 2 +) + +var inCols = []schema.Column{ + {Name: nameColName, Tag: nameColTag, Kind: types.StringKind, IsPartOfPK: true, Constraints: nil, TypeInfo: typeinfo.StringDefaultType}, + {Name: ageColName, Tag: ageColTag, Kind: types.UintKind, IsPartOfPK: false, Constraints: nil, TypeInfo: typeinfo.Uint64Type}, + {Name: titleColName, Tag: titleColTag, Kind: types.StringKind, IsPartOfPK: false, Constraints: nil, TypeInfo: typeinfo.StringDefaultType}, +} +var colColl = schema.NewColCollection(inCols...) +var rowSch = schema.MustSchemaFromCols(colColl) + +type Person struct { + Name string `parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` + Age int64 `parquet:"name=age, type=INT64, repetitiontype=OPTIONAL"` + Title string `parquet:"name=title, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"` +} + +func mustRow(r row.Row, err error) row.Row { + if err != nil { + panic(err) + } + + return r +} + +func getSampleRows() (rows []row.Row) { + return []row.Row{ + mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ + nameColTag: types.String("Bill Billerson"), + ageColTag: types.Uint(32), + titleColTag: types.String("Senior Dufus")})), + mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ + nameColTag: types.String("Rob Robertson"), + ageColTag: types.Uint(25), + titleColTag: types.String("Dufus")})), + mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ + nameColTag: types.String("John Johnson"), + ageColTag: types.Uint(21), + titleColTag: types.String("")})), + mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ + nameColTag: types.String("Andy Anderson"), + ageColTag: types.Uint(27), + //titleColTag: types.String(""), + + /* title = NULL */})), + } +} + +func writeToParquet(pWr *ParquetWriter, rows []row.Row, t *testing.T) { + func() { + defer pWr.Close(context.Background()) + + for _, row := range rows { + err := pWr.WriteRow(context.Background(), row) + + if err != nil { + t.Fatal("Failed to write row") + } + } + }() +} + +func TestWriter(t *testing.T) { + const root = "" + file, err := ioutil.TempFile(root, "parquet") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + path := file.Name() + + const expected = `Bill Billerson,32,Senior Dufus +Rob Robertson,25,Dufus +John Johnson,21, +Andy Anderson,27, +` + + rows := getSampleRows() + + pWr, err := NewParquetWriter(rowSch, path) + if err != nil { + t.Fatal("Could not open CSVWriter", err) + } + + writeToParquet(pWr, rows, t) + + pRd, err := local.NewLocalFileReader(path) + if err != nil { + t.Fatal("Cannot open file", err) + } + + pr, err := reader.NewParquetReader(pRd, new(Person), int64(WriteBufSize)) + if err != nil { + t.Fatal("Cannot create parquet reader", err) + } + + num := int(pr.GetNumRows()) + + res, err := pr.ReadByNumber(num) + if err != nil { + t.Fatal("Cannot read", err) + } + + var result string + for _, person := range res { + p, ok := person.(Person) + if !ok { + t.Fatal("cant convert") + } + result += fmt.Sprintf("%s,%d,%s\n", p.Name, p.Age, p.Title) + } + + if result != expected { + t.Errorf(`%s != %s`, result, expected) + } + +} diff --git a/integration-tests/bats/export-tables.bats b/integration-tests/bats/export-tables.bats index b91c2fae17..52518b3b52 100644 --- a/integration-tests/bats/export-tables.bats +++ b/integration-tests/bats/export-tables.bats @@ -308,3 +308,48 @@ SQL run dolt sql -q "SELECT * FROM timetable" -r csv [[ "$output" =~ "1,2021-06-02 15:37:24 +0000 UTC" ]] || false } + +@test "export-tables: parquet file export check with parquet tools" { + dolt sql -q "CREATE TABLE test_table (pk int primary key, col1 text, col2 int);" + dolt sql -q "INSERT INTO test_table VALUES (1, 'row1', 22), (2, 'row2', 33), (3, 'row3', 22);" + + run dolt table export -f test_table result.parquet + [ "$status" -eq 0 ] + [[ "$output" =~ "Successfully exported data." ]] || false + [ -f result.parquet ] + + run parquet-tools cat --json result.parquet > output.json + [ "$status" -eq 0 ] + row1='{"pk":1,"col1":"row1","col2":22}' + row2='{"pk":2,"col1":"row2","col2":33}' + row3='{"pk":3,"col1":"row3","col2":22}' + [[ "$output" =~ "$row1" ]] || false + [[ "$output" =~ "$row2" ]] || false + [[ "$output" =~ "$row3" ]] || false +} + +@test "export-tables: parquet file export compare pandas and pyarrow reads" { + dolt sql -q "CREATE TABLE test_table (pk int primary key, col1 text, col2 int);" + dolt sql -q "INSERT INTO test_table VALUES (1, 'row1', 22), (2, 'row2', 33), (3, 'row3', 22);" + + run dolt table export -f test_table result.parquet + [ "$status" -eq 0 ] + [[ "$output" =~ "Successfully exported data." ]] || false + [ -f result.parquet ] + + echo "import pandas as pd" > panda.py + echo "df = pd.read_parquet('result.parquet')" >> panda.py + echo "print(df)" >> panda.py + python3 panda.py > pandas.txt + [ -f pandas.txt ] + + echo "import pyarrow.parquet as pq" > arrow.py + echo "table = pq.read_table('result.parquet')" >> arrow.py + echo "print(table.to_pandas())" >> arrow.py + python3 arrow.py > pyarrow.txt + [ -f pyarrow.txt ] + + run diff pandas.txt pyarrow.txt + [ "$status" -eq 0 ] + [[ "$output" = "" ]] || false +} From 6890336d25ec2bb53e7871a951afa489f515e2d9 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 18 Nov 2021 22:55:19 +0000 Subject: [PATCH 04/15] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/Godeps/LICENSES | 10 ---------- go/libraries/doltcore/table/typed/parquet/writer.go | 8 ++++---- .../doltcore/table/typed/parquet/writer_test.go | 13 +++++++------ 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/go/Godeps/LICENSES b/go/Godeps/LICENSES index a2397c13d5..1c9c545469 100644 --- a/go/Godeps/LICENSES +++ b/go/Godeps/LICENSES @@ -4582,16 +4582,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. = LICENSE cd114db727544ba06cf58a8d6ab0a48a4c7dfe64d33ff2aaaa6b7a49 = ================================================================================ -================================================================================ -= github.com/kch42/buzhash licensed under: = - - DO WHATEVER THE FUCK YOU WANT, PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHATEVER THE FUCK YOU WANT. -= LICENSE 03d3e3813a51a67f46a07063e39f5d6451619f4ede2ae1c4c4cccb9e = -================================================================================ - ================================================================================ = github.com/klauspost/compress licensed under: = diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go index b91e9ff5c5..ad9cb80ddd 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer.go +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -17,8 +17,8 @@ package parquet import ( "context" "fmt" - "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go-source/local" "github.com/xitongsys/parquet-go/source" "github.com/xitongsys/parquet-go/writer" @@ -31,9 +31,9 @@ import ( var WriteBufSize = 256 * 1024 type ParquetWriter struct { - filewriter source.ParquetFile - pwriter *writer.CSVWriter - sch schema.Schema + filewriter source.ParquetFile + pwriter *writer.CSVWriter + sch schema.Schema } func NewParquetWriter(outSch schema.Schema, destName string) (*ParquetWriter, error) { diff --git a/go/libraries/doltcore/table/typed/parquet/writer_test.go b/go/libraries/doltcore/table/typed/parquet/writer_test.go index 644e9b5b27..70b1f3b0a6 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer_test.go +++ b/go/libraries/doltcore/table/typed/parquet/writer_test.go @@ -17,12 +17,13 @@ package parquet import ( "context" "fmt" - "github.com/xitongsys/parquet-go-source/local" - "github.com/xitongsys/parquet-go/reader" "io/ioutil" "os" "testing" + "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/reader" + "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" "github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo" @@ -47,9 +48,9 @@ var colColl = schema.NewColCollection(inCols...) var rowSch = schema.MustSchemaFromCols(colColl) type Person struct { - Name string `parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` - Age int64 `parquet:"name=age, type=INT64, repetitiontype=OPTIONAL"` - Title string `parquet:"name=title, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"` + Name string `parquet:"name=name, type=BYTE_ARRAY, convertedtype=UTF8"` + Age int64 `parquet:"name=age, type=INT64, repetitiontype=OPTIONAL"` + Title string `parquet:"name=title, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"` } func mustRow(r row.Row, err error) row.Row { @@ -79,7 +80,7 @@ func getSampleRows() (rows []row.Row) { ageColTag: types.Uint(27), //titleColTag: types.String(""), - /* title = NULL */})), + /* title = NULL */})), } } From 7c5d26bd7201996013d529f4759efbc3d7b79689 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 18 Nov 2021 15:36:46 -0800 Subject: [PATCH 05/15] add installs in bats --- integration-tests/bats/export-tables.bats | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration-tests/bats/export-tables.bats b/integration-tests/bats/export-tables.bats index 52518b3b52..16d0456580 100644 --- a/integration-tests/bats/export-tables.bats +++ b/integration-tests/bats/export-tables.bats @@ -310,6 +310,7 @@ SQL } @test "export-tables: parquet file export check with parquet tools" { + brew install parquet-tools dolt sql -q "CREATE TABLE test_table (pk int primary key, col1 text, col2 int);" dolt sql -q "INSERT INTO test_table VALUES (1, 'row1', 22), (2, 'row2', 33), (3, 'row3', 22);" @@ -329,6 +330,8 @@ SQL } @test "export-tables: parquet file export compare pandas and pyarrow reads" { + pip install pandas + pip install pyarrow dolt sql -q "CREATE TABLE test_table (pk int primary key, col1 text, col2 int);" dolt sql -q "INSERT INTO test_table VALUES (1, 'row1', 22), (2, 'row2', 33), (3, 'row3', 22);" From bef965ff0b9344e4f762fa3130e2ee3b1e22fe01 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 18 Nov 2021 15:51:51 -0800 Subject: [PATCH 06/15] add dependencies to yaml --- .github/workflows/ci-bats-unix.yaml | 4 ++++ .github/workflows/ci-bats-windows.yaml | 2 ++ integration-tests/bats/export-tables.bats | 4 +--- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-bats-unix.yaml b/.github/workflows/ci-bats-unix.yaml index 3b1cdc33a4..bbcc269092 100644 --- a/.github/workflows/ci-bats-unix.yaml +++ b/.github/workflows/ci-bats-unix.yaml @@ -61,6 +61,10 @@ jobs: - name: Install Python Deps run: | pip install mysql-connector-python + pip install pandas + pip install pyarrow + - name: Install Parquet-tools + run: brew install parquet-tools - name: Install Dolt working-directory: ./go run: | diff --git a/.github/workflows/ci-bats-windows.yaml b/.github/workflows/ci-bats-windows.yaml index e83c76df70..d74d860027 100644 --- a/.github/workflows/ci-bats-windows.yaml +++ b/.github/workflows/ci-bats-windows.yaml @@ -114,6 +114,8 @@ jobs: - name: Install Python Deps run: | pip install mysql-connector-python + pip install pandas + pip install pyarrow - name: Install Dolt working-directory: ./go run: | diff --git a/integration-tests/bats/export-tables.bats b/integration-tests/bats/export-tables.bats index 16d0456580..59f6353baf 100644 --- a/integration-tests/bats/export-tables.bats +++ b/integration-tests/bats/export-tables.bats @@ -310,7 +310,7 @@ SQL } @test "export-tables: parquet file export check with parquet tools" { - brew install parquet-tools + skiponwindows "Has dependencies that are missing on the Jenkins Windows installation." dolt sql -q "CREATE TABLE test_table (pk int primary key, col1 text, col2 int);" dolt sql -q "INSERT INTO test_table VALUES (1, 'row1', 22), (2, 'row2', 33), (3, 'row3', 22);" @@ -330,8 +330,6 @@ SQL } @test "export-tables: parquet file export compare pandas and pyarrow reads" { - pip install pandas - pip install pyarrow dolt sql -q "CREATE TABLE test_table (pk int primary key, col1 text, col2 int);" dolt sql -q "INSERT INTO test_table VALUES (1, 'row1', 22), (2, 'row2', 33), (3, 'row3', 22);" From 67ef43860bd1124d32e49929c1eeb5a1d0ff59e8 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 18 Nov 2021 15:56:18 -0800 Subject: [PATCH 07/15] typo --- .github/workflows/ci-bats-unix.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-bats-unix.yaml b/.github/workflows/ci-bats-unix.yaml index bbcc269092..d4f1c3aef2 100644 --- a/.github/workflows/ci-bats-unix.yaml +++ b/.github/workflows/ci-bats-unix.yaml @@ -64,7 +64,8 @@ jobs: pip install pandas pip install pyarrow - name: Install Parquet-tools - run: brew install parquet-tools + run: | + brew install parquet-tools - name: Install Dolt working-directory: ./go run: | From 0ae10903f44993f0bf6cc447cc553b3fab59a017 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 18 Nov 2021 16:07:16 -0800 Subject: [PATCH 08/15] fix yaml fail --- .github/workflows/ci-bats-unix.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-bats-unix.yaml b/.github/workflows/ci-bats-unix.yaml index d4f1c3aef2..23c2121c70 100644 --- a/.github/workflows/ci-bats-unix.yaml +++ b/.github/workflows/ci-bats-unix.yaml @@ -63,9 +63,6 @@ jobs: pip install mysql-connector-python pip install pandas pip install pyarrow - - name: Install Parquet-tools - run: | - brew install parquet-tools - name: Install Dolt working-directory: ./go run: | @@ -81,6 +78,8 @@ jobs: - name: Install expect if: matrix.os == 'ubuntu-18.04' run: sudo apt-get install -y expect + - name: Install Parquet-tools + run: brew install parquet-tools - name: Check expect run: expect -v - name: Test all Unix From cf1e5acc600e9e9ba24360632ff30faa8c391691 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Thu, 18 Nov 2021 17:01:15 -0800 Subject: [PATCH 09/15] update writer files --- .../doltcore/table/typed/parquet/writer.go | 2 +- .../doltcore/table/typed/parquet/writer_test.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go index ad9cb80ddd..44c2a772c4 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer.go +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -61,7 +61,7 @@ func NewParquetWriter(outSch schema.Schema, destName string) (*ParquetWriter, er // np configures the degree of concurrency for our Reader and Writers // TODO: not sure what default value to set 'np' to - pw, err := writer.NewCSVWriter(csvSchema, fw, int64(WriteBufSize)) + pw, err := writer.NewCSVWriter(csvSchema, fw, 4) if err != nil { return nil, err } diff --git a/go/libraries/doltcore/table/typed/parquet/writer_test.go b/go/libraries/doltcore/table/typed/parquet/writer_test.go index 70b1f3b0a6..5db4d10e44 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer_test.go +++ b/go/libraries/doltcore/table/typed/parquet/writer_test.go @@ -100,6 +100,12 @@ func writeToParquet(pWr *ParquetWriter, rows []row.Row, t *testing.T) { func TestWriter(t *testing.T) { const root = "" + const expected = `Bill Billerson,32,Senior Dufus +Rob Robertson,25,Dufus +John Johnson,21, +Andy Anderson,27, +` + file, err := ioutil.TempFile(root, "parquet") if err != nil { t.Fatal(err) @@ -107,12 +113,6 @@ func TestWriter(t *testing.T) { defer os.Remove(file.Name()) path := file.Name() - const expected = `Bill Billerson,32,Senior Dufus -Rob Robertson,25,Dufus -John Johnson,21, -Andy Anderson,27, -` - rows := getSampleRows() pWr, err := NewParquetWriter(rowSch, path) @@ -127,7 +127,7 @@ Andy Anderson,27, t.Fatal("Cannot open file", err) } - pr, err := reader.NewParquetReader(pRd, new(Person), int64(WriteBufSize)) + pr, err := reader.NewParquetReader(pRd, new(Person), 4) if err != nil { t.Fatal("Cannot create parquet reader", err) } From 57378cecc677cec85746bfda0240044672261fac Mon Sep 17 00:00:00 2001 From: jennifersp Date: Mon, 22 Nov 2021 12:40:33 -0800 Subject: [PATCH 10/15] resolve issues --- go/cmd/dolt/commands/dump.go | 23 ++++-- .../doltcore/table/typed/parquet/writer.go | 81 +++++++++---------- .../table/typed/parquet/writer_test.go | 12 ++- integration-tests/bats/export-tables.bats | 64 +++++++++++++++ integration-tests/bats/helper/pandas.py | 3 + integration-tests/bats/helper/pyarrow.py | 3 + 6 files changed, 130 insertions(+), 56 deletions(-) create mode 100644 integration-tests/bats/helper/pandas.py create mode 100644 integration-tests/bats/helper/pyarrow.py diff --git a/go/cmd/dolt/commands/dump.go b/go/cmd/dolt/commands/dump.go index ffe364199a..d6414d92c7 100644 --- a/go/cmd/dolt/commands/dump.go +++ b/go/cmd/dolt/commands/dump.go @@ -1,4 +1,4 @@ -// Copyright 2019 Dolthub, Inc. +// Copyright 2021 Dolthub, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -44,11 +44,12 @@ const ( directoryFlag = "directory" filenameFlag = "file-name" - sqlFileExt = "sql" - csvFileExt = "csv" - jsonFileExt = "json" - emptyFileExt = "" - emptyStr = "" + sqlFileExt = "sql" + csvFileExt = "csv" + jsonFileExt = "json" + parquetFileExt = "parquet" + emptyFileExt = "" + emptyStr = "" ) var dumpDocs = cli.CommandDocumentationContent{ @@ -164,6 +165,11 @@ func (cmd DumpCmd) Exec(ctx context.Context, commandStr string, args []string, d if err != nil { return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage) } + case parquetFileExt: + err = dumpTables(ctx, root, dEnv, force, tblNames, parquetFileExt, name) + if err != nil { + return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage) + } default: return HandleVErrAndExitCode(errhand.BuildDError("invalid result format").SetPrintUsage().Build(), usage) } @@ -329,6 +335,11 @@ func validateArgs(apr *argparser.ArgParseResults) (string, errhand.VerboseError) return emptyStr, errhand.BuildDError("%s is not supported for %s exports", filenameFlag, jsonFileExt).SetPrintUsage().Build() } return dn, nil + case parquetFileExt: + if fnOk { + return emptyStr, errhand.BuildDError("%s is not supported for %s exports", filenameFlag, parquetFileExt).SetPrintUsage().Build() + } + return dn, nil default: return emptyStr, errhand.BuildDError("invalid result format").SetPrintUsage().Build() } diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go index 44c2a772c4..8cddc85f36 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer.go +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -1,4 +1,4 @@ -// Copyright 2019 Dolthub, Inc. +// Copyright 2021 Dolthub, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,10 +17,10 @@ package parquet import ( "context" "fmt" - "github.com/xitongsys/parquet-go-source/local" "github.com/xitongsys/parquet-go/source" "github.com/xitongsys/parquet-go/writer" + "time" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" @@ -28,30 +28,48 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil" ) -var WriteBufSize = 256 * 1024 - type ParquetWriter struct { filewriter source.ParquetFile pwriter *writer.CSVWriter sch schema.Schema } -func NewParquetWriter(outSch schema.Schema, destName string) (*ParquetWriter, error) { - allCols := outSch.GetAllCols() - columns := allCols.GetColumns() +var typeMap = map[typeinfo.Identifier]string{ + typeinfo.DatetimeTypeIdentifier: "type=INT64, convertedtype=TIME_MICROS", + typeinfo.DecimalTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.EnumTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.InlineBlobTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.SetTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.TimeTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.TupleTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.UuidTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.VarBinaryTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.YearTypeIdentifier: "type=INT32, convertedtype=DATE", + typeinfo.UnknownTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.JSONTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.BlobStringTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + + typeinfo.BitTypeIdentifier: "type=BYTE_ARRAY", + typeinfo.BoolTypeIdentifier: "type=BOOLEAN", + typeinfo.VarStringTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.UintTypeIdentifier: "type=INT64, convertedtype=UINT_64", + typeinfo.IntTypeIdentifier: "type=INT64, convertedtype=INT_64", + typeinfo.FloatTypeIdentifier: "type=DOUBLE", +} + +func NewParquetWriter(outSch schema.Schema, destName string) (*ParquetWriter, error) { + columns := outSch.GetAllCols().GetColumns() - ColTypeMap := getTypeMap() var csvSchema []string var repetitionType string + // creates csv schema for handling parquet format using NewCSVWriter for _, col := range columns { - colName := col.Name + repetitionType = "" colType := col.TypeInfo.GetTypeIdentifier() if col.IsNullable() { - repetitionType = `, repetitiontype=OPTIONAL` - } else { - repetitionType = `` + repetitionType = ", repetitiontype=OPTIONAL" } - csvSchema = append(csvSchema, fmt.Sprintf("name=%s, %s%s", colName, ColTypeMap[colType], repetitionType)) + csvSchema = append(csvSchema, fmt.Sprintf("name=%s, %s%s", col.Name, typeMap[colType], repetitionType)) } fw, err := local.NewLocalFileWriter(destName) @@ -59,13 +77,13 @@ func NewParquetWriter(outSch schema.Schema, destName string) (*ParquetWriter, er return nil, err } - // np configures the degree of concurrency for our Reader and Writers - // TODO: not sure what default value to set 'np' to + // default np (degree of concurrency) is 4 recommended from the package pw, err := writer.NewCSVWriter(csvSchema, fw, 4) if err != nil { return nil, err } + // pw.CompressionType defaults to parquet.CompressionCodec_SNAPPY return &ParquetWriter{filewriter: fw, pwriter: pw, sch: outSch}, nil } @@ -75,8 +93,7 @@ func (pwr *ParquetWriter) GetSchema() schema.Schema { // WriteRow will write a row to a table func (pwr *ParquetWriter) WriteRow(ctx context.Context, r row.Row) error { - allCols := pwr.sch.GetAllCols() - colValStrs := make([]*string, allCols.Size()) + colValStrs := make([]*string, pwr.sch.GetAllCols().Size()) sqlRow, err := sqlutil.DoltRowToSqlRow(r, pwr.GetSchema()) if err != nil { @@ -87,6 +104,10 @@ func (pwr *ParquetWriter) WriteRow(ctx context.Context, r row.Row) error { if val == nil { colValStrs[i] = nil } else { + switch val.(type) { + case time.Time: + val = val.(time.Time).Unix() + } v := sqlutil.SqlColToStr(ctx, val) colValStrs[i] = &v } @@ -109,29 +130,3 @@ func (pwr *ParquetWriter) Close(ctx context.Context) error { pwr.filewriter.Close() return nil } - -func getTypeMap() map[typeinfo.Identifier]string { - - typeMap := map[typeinfo.Identifier]string{ - typeinfo.DatetimeTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.DecimalTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.EnumTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.InlineBlobTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.SetTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.TimeTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.TupleTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.UuidTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.VarBinaryTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.YearTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.BlobStringTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - - typeinfo.BitTypeIdentifier: `type=BYTE_ARRAY`, - typeinfo.BoolTypeIdentifier: `type=BOOLEAN`, - typeinfo.VarStringTypeIdentifier: `type=BYTE_ARRAY, convertedtype=UTF8`, - typeinfo.UintTypeIdentifier: `type=INT64, convertedtype=UINT_64`, - typeinfo.IntTypeIdentifier: `type=INT64, convertedtype=INT_64`, - typeinfo.FloatTypeIdentifier: `type=DOUBLE`, - } - - return typeMap -} diff --git a/go/libraries/doltcore/table/typed/parquet/writer_test.go b/go/libraries/doltcore/table/typed/parquet/writer_test.go index 5db4d10e44..01fc646c5a 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer_test.go +++ b/go/libraries/doltcore/table/typed/parquet/writer_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 Dolthub, Inc. +// Copyright 2021 Dolthub, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package parquet import ( "context" "fmt" + "github.com/stretchr/testify/assert" "io/ioutil" "os" "testing" @@ -99,14 +100,13 @@ func writeToParquet(pWr *ParquetWriter, rows []row.Row, t *testing.T) { } func TestWriter(t *testing.T) { - const root = "" const expected = `Bill Billerson,32,Senior Dufus Rob Robertson,25,Dufus John Johnson,21, Andy Anderson,27, ` - file, err := ioutil.TempFile(root, "parquet") + file, err := ioutil.TempFile("", "parquet") if err != nil { t.Fatal(err) } @@ -133,6 +133,7 @@ Andy Anderson,27, } num := int(pr.GetNumRows()) + assert.Equal(t, num, 4) res, err := pr.ReadByNumber(num) if err != nil { @@ -148,8 +149,5 @@ Andy Anderson,27, result += fmt.Sprintf("%s,%d,%s\n", p.Name, p.Age, p.Title) } - if result != expected { - t.Errorf(`%s != %s`, result, expected) - } - + assert.Equal(t, expected, result) } diff --git a/integration-tests/bats/export-tables.bats b/integration-tests/bats/export-tables.bats index 59f6353baf..c6caeb4d4c 100644 --- a/integration-tests/bats/export-tables.bats +++ b/integration-tests/bats/export-tables.bats @@ -354,3 +354,67 @@ SQL [ "$status" -eq 0 ] [[ "$output" = "" ]] || false } + +@test "export-tables: table export datetime, bool, enum types to parquet" { + skiponwindows "Has dependencies that are missing on the Jenkins Windows installation." + dolt sql < output.json + [ "$status" -eq 0 ] + row1='{"pk":1,"v1":1586304000,"v2":"11:11:11","v3":2020,"v4":1586344271,"v5":1,"v6":"one"}' + row2='{"pk":2,"v1":1586304000,"v2":"12:12:12","v3":2020,"v4":1586347932,"v5":0,"v6":"three"}' + row3='{"pk":3,"v1":1633737600,"v2":"04:12:34","v3":2019,"v4":1570594354,"v5":1}' + [[ "$output" =~ "$row1" ]] || false + [[ "$output" =~ "$row2" ]] || false + [[ "$output" =~ "$row3" ]] || false +} + + +@test "export-tables: table export more types to parquet" { + skiponwindows "Has dependencies that are missing on the Jenkins Windows installation." + dolt sql < output.json + [ "$status" -eq 0 ] + row1='{"pk":0,"int":0,"string":"asdf","boolean":1,"float":0.0,"uint":0,"uuid":"00000000-0000-0000-0000-000000000000"}' + row2='{"pk":1,"int":-1,"string":"qwerty","boolean":0,"float":-1.0,"uint":1,"uuid":"00000000-0000-0000-0000-000000000001"}' + row3='{"pk":2,"int":1,"string":"","boolean":1,"float":0.0,"uint":0,"uuid":"123e4567-e89b-12d3-a456-426655440000"}' + [[ "$output" =~ "$row1" ]] || false + [[ "$output" =~ "$row2" ]] || false + [[ "$output" =~ "$row3" ]] || false +} diff --git a/integration-tests/bats/helper/pandas.py b/integration-tests/bats/helper/pandas.py new file mode 100644 index 0000000000..6d43cd02be --- /dev/null +++ b/integration-tests/bats/helper/pandas.py @@ -0,0 +1,3 @@ +import pandas as pd +df = pd.read_parquet('result.parquet') +print(df) \ No newline at end of file diff --git a/integration-tests/bats/helper/pyarrow.py b/integration-tests/bats/helper/pyarrow.py new file mode 100644 index 0000000000..e0140eabca --- /dev/null +++ b/integration-tests/bats/helper/pyarrow.py @@ -0,0 +1,3 @@ +import pyarrow.parquet as pq +table = pq.read_table('result.parquet') +print(table.to_pandas()) \ No newline at end of file From 3096a6bb1320cb6ddeaed409afb65f5418b59cbc Mon Sep 17 00:00:00 2001 From: jennifersp Date: Mon, 22 Nov 2021 20:41:40 +0000 Subject: [PATCH 11/15] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/cmd/dolt/commands/dump.go | 12 ++++++------ go/libraries/doltcore/table/typed/parquet/writer.go | 3 ++- .../doltcore/table/typed/parquet/writer_test.go | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go/cmd/dolt/commands/dump.go b/go/cmd/dolt/commands/dump.go index d6414d92c7..290e0f4588 100644 --- a/go/cmd/dolt/commands/dump.go +++ b/go/cmd/dolt/commands/dump.go @@ -44,12 +44,12 @@ const ( directoryFlag = "directory" filenameFlag = "file-name" - sqlFileExt = "sql" - csvFileExt = "csv" - jsonFileExt = "json" - parquetFileExt = "parquet" - emptyFileExt = "" - emptyStr = "" + sqlFileExt = "sql" + csvFileExt = "csv" + jsonFileExt = "json" + parquetFileExt = "parquet" + emptyFileExt = "" + emptyStr = "" ) var dumpDocs = cli.CommandDocumentationContent{ diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go index 8cddc85f36..31cff3f1a5 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer.go +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -17,10 +17,11 @@ package parquet import ( "context" "fmt" + "time" + "github.com/xitongsys/parquet-go-source/local" "github.com/xitongsys/parquet-go/source" "github.com/xitongsys/parquet-go/writer" - "time" "github.com/dolthub/dolt/go/libraries/doltcore/row" "github.com/dolthub/dolt/go/libraries/doltcore/schema" diff --git a/go/libraries/doltcore/table/typed/parquet/writer_test.go b/go/libraries/doltcore/table/typed/parquet/writer_test.go index 01fc646c5a..6b33514aa1 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer_test.go +++ b/go/libraries/doltcore/table/typed/parquet/writer_test.go @@ -17,11 +17,11 @@ package parquet import ( "context" "fmt" - "github.com/stretchr/testify/assert" "io/ioutil" "os" "testing" + "github.com/stretchr/testify/assert" "github.com/xitongsys/parquet-go-source/local" "github.com/xitongsys/parquet-go/reader" From cd5d618e528d414deba20de20ba619e89fcc468f Mon Sep 17 00:00:00 2001 From: jennifersp Date: Mon, 22 Nov 2021 13:13:13 -0800 Subject: [PATCH 12/15] fix bats test --- integration-tests/bats/export-tables.bats | 18 ++++++++++-------- integration-tests/bats/helper/pandas.py | 3 --- integration-tests/bats/helper/pyarrow.py | 3 --- 3 files changed, 10 insertions(+), 14 deletions(-) delete mode 100644 integration-tests/bats/helper/pandas.py delete mode 100644 integration-tests/bats/helper/pyarrow.py diff --git a/integration-tests/bats/export-tables.bats b/integration-tests/bats/export-tables.bats index c6caeb4d4c..15410d7284 100644 --- a/integration-tests/bats/export-tables.bats +++ b/integration-tests/bats/export-tables.bats @@ -338,16 +338,18 @@ SQL [[ "$output" =~ "Successfully exported data." ]] || false [ -f result.parquet ] - echo "import pandas as pd" > panda.py - echo "df = pd.read_parquet('result.parquet')" >> panda.py - echo "print(df)" >> panda.py - python3 panda.py > pandas.txt + echo "import pandas as pd +df = pd.read_parquet('result.parquet') +print(df) +" > pandas.py + run python3 pandas.py > pandas.txt [ -f pandas.txt ] - echo "import pyarrow.parquet as pq" > arrow.py - echo "table = pq.read_table('result.parquet')" >> arrow.py - echo "print(table.to_pandas())" >> arrow.py - python3 arrow.py > pyarrow.txt + echo "import pyarrow.parquet as pq +table = pq.read_table('result.parquet') +print(table.to_pandas()) +" > arrow.py + run python3 arrow.py > pyarrow.txt [ -f pyarrow.txt ] run diff pandas.txt pyarrow.txt diff --git a/integration-tests/bats/helper/pandas.py b/integration-tests/bats/helper/pandas.py deleted file mode 100644 index 6d43cd02be..0000000000 --- a/integration-tests/bats/helper/pandas.py +++ /dev/null @@ -1,3 +0,0 @@ -import pandas as pd -df = pd.read_parquet('result.parquet') -print(df) \ No newline at end of file diff --git a/integration-tests/bats/helper/pyarrow.py b/integration-tests/bats/helper/pyarrow.py deleted file mode 100644 index e0140eabca..0000000000 --- a/integration-tests/bats/helper/pyarrow.py +++ /dev/null @@ -1,3 +0,0 @@ -import pyarrow.parquet as pq -table = pq.read_table('result.parquet') -print(table.to_pandas()) \ No newline at end of file From 246930ff915b3a5e2324d9c9fa15cb61c5c1dfd7 Mon Sep 17 00:00:00 2001 From: jennifersp Date: Mon, 22 Nov 2021 17:40:25 -0800 Subject: [PATCH 13/15] fix dec and bit type --- .../doltcore/table/typed/parquet/writer.go | 5 +++-- integration-tests/bats/export-tables.bats | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go index 31cff3f1a5..e8ea81f045 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer.go +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -37,7 +37,8 @@ type ParquetWriter struct { var typeMap = map[typeinfo.Identifier]string{ typeinfo.DatetimeTypeIdentifier: "type=INT64, convertedtype=TIME_MICROS", - typeinfo.DecimalTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", + typeinfo.DecimalTypeIdentifier: "type=BYTE_ARRAY, convertedtype=DECIMAL, scale=2, precision=20" + + "", typeinfo.EnumTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", typeinfo.InlineBlobTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", typeinfo.SetTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", @@ -50,7 +51,7 @@ var typeMap = map[typeinfo.Identifier]string{ typeinfo.JSONTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", typeinfo.BlobStringTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", - typeinfo.BitTypeIdentifier: "type=BYTE_ARRAY", + typeinfo.BitTypeIdentifier: "type=INT32, convertedtype=INT_16", typeinfo.BoolTypeIdentifier: "type=BOOLEAN", typeinfo.VarStringTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", typeinfo.UintTypeIdentifier: "type=INT64, convertedtype=UINT_64", diff --git a/integration-tests/bats/export-tables.bats b/integration-tests/bats/export-tables.bats index 15410d7284..20adebf3d1 100644 --- a/integration-tests/bats/export-tables.bats +++ b/integration-tests/bats/export-tables.bats @@ -420,3 +420,22 @@ SQL [[ "$output" =~ "$row2" ]] || false [[ "$output" =~ "$row3" ]] || false } + +@test "export-tables: table export decimal and bit types to parquet" { + skiponwindows "Has dependencies that are missing on the Jenkins Windows installation." + dolt sql -q "CREATE TABLE more (pk BIGINT NOT NULL,v DECIMAL(9,5),b BIT(10),PRIMARY KEY (pk));" + dolt sql -q "INSERT INTO more VALUES (1, 1234.56789, 511);" + dolt sql -q "INSERT INTO more VALUES (2, 5235.66789, 514);" + + run dolt table export more more.parquet + [ "$status" -eq 0 ] + [[ "$output" =~ "Successfully exported data." ]] || false + [ -f more.parquet ] + + run parquet-tools cat --json more.parquet > output.json + [ "$status" -eq 0 ] + row1='{"pk":1,"v":1234.57,"b":511}' + row2='{"pk":2,"v":5235.67,"b":514}' + [[ "$output" =~ "$row1" ]] || false + [[ "$output" =~ "$row2" ]] || false +} \ No newline at end of file From 4dcb8712731946975d536f5c27e894af8c01324e Mon Sep 17 00:00:00 2001 From: jennifersp Date: Tue, 23 Nov 2021 01:41:51 +0000 Subject: [PATCH 14/15] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/table/typed/parquet/writer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/libraries/doltcore/table/typed/parquet/writer.go b/go/libraries/doltcore/table/typed/parquet/writer.go index e8ea81f045..28595a4bbd 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer.go +++ b/go/libraries/doltcore/table/typed/parquet/writer.go @@ -36,8 +36,8 @@ type ParquetWriter struct { } var typeMap = map[typeinfo.Identifier]string{ - typeinfo.DatetimeTypeIdentifier: "type=INT64, convertedtype=TIME_MICROS", - typeinfo.DecimalTypeIdentifier: "type=BYTE_ARRAY, convertedtype=DECIMAL, scale=2, precision=20" + + typeinfo.DatetimeTypeIdentifier: "type=INT64, convertedtype=TIME_MICROS", + typeinfo.DecimalTypeIdentifier: "type=BYTE_ARRAY, convertedtype=DECIMAL, scale=2, precision=20" + "", typeinfo.EnumTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", typeinfo.InlineBlobTypeIdentifier: "type=BYTE_ARRAY, convertedtype=UTF8", From 20c9070543e20b618be609464a933103b1497ea2 Mon Sep 17 00:00:00 2001 From: jennifersp <44716627+jennifersp@users.noreply.github.com> Date: Mon, 22 Nov 2021 20:27:02 -0800 Subject: [PATCH 15/15] Update writer_test.go --- go/libraries/doltcore/table/typed/parquet/writer_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/go/libraries/doltcore/table/typed/parquet/writer_test.go b/go/libraries/doltcore/table/typed/parquet/writer_test.go index 6b33514aa1..91f5d1560f 100644 --- a/go/libraries/doltcore/table/typed/parquet/writer_test.go +++ b/go/libraries/doltcore/table/typed/parquet/writer_test.go @@ -79,8 +79,6 @@ func getSampleRows() (rows []row.Row) { mustRow(row.New(types.Format_Default, rowSch, row.TaggedValues{ nameColTag: types.String("Andy Anderson"), ageColTag: types.Uint(27), - //titleColTag: types.String(""), - /* title = NULL */})), } } @@ -91,7 +89,6 @@ func writeToParquet(pWr *ParquetWriter, rows []row.Row, t *testing.T) { for _, row := range rows { err := pWr.WriteRow(context.Background(), row) - if err != nil { t.Fatal("Failed to write row") }