mirror/dolt
1
0
Fork 0
mirror of https://github.com/dolthub/dolt.git synced 2026-03-13 19:29:58 -05:00
Code Issues Packages Projects Releases 146 Wiki Activity

upload correctness checking (#160)

Browse Source 
Now sending the md5, as well as the content length of the table files we will upload to the dolthub remote api.  This is used to make sure what is uploaded is complete and correct.
This commit is contained in:
Brian Hendriks
2019-10-22 10:06:54 -07:00
committed by GitHub
parent 84b84323bc
commit 67be3c36f6
14 changed files with 335 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

253
go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go
View File

@@ -585,19 +585,75 @@ func (m *GetDownloadLocsResponse) GetLocs() []*DownloadLoc {
return nil
}
type GetUploadLocsRequest struct {
RepoId *RepoId `protobuf:"bytes,1,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"`
TableFileHashes [][]byte `protobuf:"bytes,2,rep,name=table_file_hashes,json=tableFileHashes,proto3" json:"table_file_hashes,omitempty"`
type TableFileDetails struct {
Id []byte `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
ContentLength uint64 `protobuf:"varint,2,opt,name=content_length,json=contentLength,proto3" json:"content_length,omitempty"`
ContentHash []byte `protobuf:"bytes,3,opt,name=content_hash,json=contentHash,proto3" json:"content_hash,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *TableFileDetails) Reset() { *m = TableFileDetails{} }
func (m *TableFileDetails) String() string { return proto.CompactTextString(m) }
func (*TableFileDetails) ProtoMessage() {}
func (*TableFileDetails) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{11}
}
func (m *TableFileDetails) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_TableFileDetails.Unmarshal(m, b)
}
func (m *TableFileDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_TableFileDetails.Marshal(b, m, deterministic)
}
func (m *TableFileDetails) XXX_Merge(src proto.Message) {
xxx_messageInfo_TableFileDetails.Merge(m, src)
}
func (m *TableFileDetails) XXX_Size() int {
return xxx_messageInfo_TableFileDetails.Size(m)
}
func (m *TableFileDetails) XXX_DiscardUnknown() {
xxx_messageInfo_TableFileDetails.DiscardUnknown(m)
}
var xxx_messageInfo_TableFileDetails proto.InternalMessageInfo
func (m *TableFileDetails) GetId() []byte {
if m != nil {
return m.Id
}
return nil
}
func (m *TableFileDetails) GetContentLength() uint64 {
if m != nil {
return m.ContentLength
}
return 0
}
func (m *TableFileDetails) GetContentHash() []byte {
if m != nil {
return m.ContentHash
}
return nil
}
type GetUploadLocsRequest struct {
RepoId *RepoId `protobuf:"bytes,1,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"`
TableFileHashes [][]byte `protobuf:"bytes,2,rep,name=table_file_hashes,json=tableFileHashes,proto3" json:"table_file_hashes,omitempty"`
TableFileDetails []*TableFileDetails `protobuf:"bytes,3,rep,name=table_file_details,json=tableFileDetails,proto3" json:"table_file_details,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *GetUploadLocsRequest) Reset() { *m = GetUploadLocsRequest{} }
func (m *GetUploadLocsRequest) String() string { return proto.CompactTextString(m) }
func (*GetUploadLocsRequest) ProtoMessage() {}
func (*GetUploadLocsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{11}
return fileDescriptor_702c187af9ca94ec, []int{12}
}
func (m *GetUploadLocsRequest) XXX_Unmarshal(b []byte) error {
@@ -632,6 +688,13 @@ func (m *GetUploadLocsRequest) GetTableFileHashes() [][]byte {
return nil
}
func (m *GetUploadLocsRequest) GetTableFileDetails() []*TableFileDetails {
if m != nil {
return m.TableFileDetails
}
return nil
}
type GetUploadLocsResponse struct {
Locs []*UploadLoc `protobuf:"bytes,1,rep,name=locs,proto3" json:"locs,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@@ -643,7 +706,7 @@ func (m *GetUploadLocsResponse) Reset() { *m = GetUploadLocsResponse{} }
func (m *GetUploadLocsResponse) String() string { return proto.CompactTextString(m) }
func (*GetUploadLocsResponse) ProtoMessage() {}
func (*GetUploadLocsResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{12}
return fileDescriptor_702c187af9ca94ec, []int{13}
}
func (m *GetUploadLocsResponse) XXX_Unmarshal(b []byte) error {
@@ -682,7 +745,7 @@ func (m *RebaseRequest) Reset() { *m = RebaseRequest{} }
func (m *RebaseRequest) String() string { return proto.CompactTextString(m) }
func (*RebaseRequest) ProtoMessage() {}
func (*RebaseRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{13}
return fileDescriptor_702c187af9ca94ec, []int{14}
}
func (m *RebaseRequest) XXX_Unmarshal(b []byte) error {
@@ -720,7 +783,7 @@ func (m *RebaseResponse) Reset() { *m = RebaseResponse{} }
func (m *RebaseResponse) String() string { return proto.CompactTextString(m) }
func (*RebaseResponse) ProtoMessage() {}
func (*RebaseResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{14}
return fileDescriptor_702c187af9ca94ec, []int{15}
}
func (m *RebaseResponse) XXX_Unmarshal(b []byte) error {
@@ -752,7 +815,7 @@ func (m *RootRequest) Reset() { *m = RootRequest{} }
func (m *RootRequest) String() string { return proto.CompactTextString(m) }
func (*RootRequest) ProtoMessage() {}
func (*RootRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{15}
return fileDescriptor_702c187af9ca94ec, []int{16}
}
func (m *RootRequest) XXX_Unmarshal(b []byte) error {
@@ -791,7 +854,7 @@ func (m *RootResponse) Reset() { *m = RootResponse{} }
func (m *RootResponse) String() string { return proto.CompactTextString(m) }
func (*RootResponse) ProtoMessage() {}
func (*RootResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{16}
return fileDescriptor_702c187af9ca94ec, []int{17}
}
func (m *RootResponse) XXX_Unmarshal(b []byte) error {
@@ -831,7 +894,7 @@ func (m *ChunkTableInfo) Reset() { *m = ChunkTableInfo{} }
func (m *ChunkTableInfo) String() string { return proto.CompactTextString(m) }
func (*ChunkTableInfo) ProtoMessage() {}
func (*ChunkTableInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{17}
return fileDescriptor_702c187af9ca94ec, []int{18}
}
func (m *ChunkTableInfo) XXX_Unmarshal(b []byte) error {
@@ -881,7 +944,7 @@ func (m *CommitRequest) Reset() { *m = CommitRequest{} }
func (m *CommitRequest) String() string { return proto.CompactTextString(m) }
func (*CommitRequest) ProtoMessage() {}
func (*CommitRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{18}
return fileDescriptor_702c187af9ca94ec, []int{19}
}
func (m *CommitRequest) XXX_Unmarshal(b []byte) error {
@@ -948,7 +1011,7 @@ func (m *CommitResponse) Reset() { *m = CommitResponse{} }
func (m *CommitResponse) String() string { return proto.CompactTextString(m) }
func (*CommitResponse) ProtoMessage() {}
func (*CommitResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{19}
return fileDescriptor_702c187af9ca94ec, []int{20}
}
func (m *CommitResponse) XXX_Unmarshal(b []byte) error {
@@ -988,7 +1051,7 @@ func (m *GetRepoMetadataRequest) Reset() { *m = GetRepoMetadataRequest{}
func (m *GetRepoMetadataRequest) String() string { return proto.CompactTextString(m) }
func (*GetRepoMetadataRequest) ProtoMessage() {}
func (*GetRepoMetadataRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{20}
return fileDescriptor_702c187af9ca94ec, []int{21}
}
func (m *GetRepoMetadataRequest) XXX_Unmarshal(b []byte) error {
@@ -1039,7 +1102,7 @@ func (m *GetRepoMetadataResponse) Reset() { *m = GetRepoMetadataResponse
func (m *GetRepoMetadataResponse) String() string { return proto.CompactTextString(m) }
func (*GetRepoMetadataResponse) ProtoMessage() {}
func (*GetRepoMetadataResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{21}
return fileDescriptor_702c187af9ca94ec, []int{22}
}
func (m *GetRepoMetadataResponse) XXX_Unmarshal(b []byte) error {
@@ -1086,7 +1149,7 @@ func (m *ClientRepoFormat) Reset() { *m = ClientRepoFormat{} }
func (m *ClientRepoFormat) String() string { return proto.CompactTextString(m) }
func (*ClientRepoFormat) ProtoMessage() {}
func (*ClientRepoFormat) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{22}
return fileDescriptor_702c187af9ca94ec, []int{23}
}
func (m *ClientRepoFormat) XXX_Unmarshal(b []byte) error {
@@ -1132,7 +1195,7 @@ func (m *ListTableFilesRequest) Reset() { *m = ListTableFilesRequest{} }
func (m *ListTableFilesRequest) String() string { return proto.CompactTextString(m) }
func (*ListTableFilesRequest) ProtoMessage() {}
func (*ListTableFilesRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{23}
return fileDescriptor_702c187af9ca94ec, []int{24}
}
func (m *ListTableFilesRequest) XXX_Unmarshal(b []byte) error {
@@ -1173,7 +1236,7 @@ func (m *TableFileInfo) Reset() { *m = TableFileInfo{} }
func (m *TableFileInfo) String() string { return proto.CompactTextString(m) }
func (*TableFileInfo) ProtoMessage() {}
func (*TableFileInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{24}
return fileDescriptor_702c187af9ca94ec, []int{25}
}
func (m *TableFileInfo) XXX_Unmarshal(b []byte) error {
@@ -1227,7 +1290,7 @@ func (m *ListTableFilesResponse) Reset() { *m = ListTableFilesResponse{}
func (m *ListTableFilesResponse) String() string { return proto.CompactTextString(m) }
func (*ListTableFilesResponse) ProtoMessage() {}
func (*ListTableFilesResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{25}
return fileDescriptor_702c187af9ca94ec, []int{26}
}
func (m *ListTableFilesResponse) XXX_Unmarshal(b []byte) error {
@@ -1275,7 +1338,7 @@ func (m *AddTableFilesRequest) Reset() { *m = AddTableFilesRequest{} }
func (m *AddTableFilesRequest) String() string { return proto.CompactTextString(m) }
func (*AddTableFilesRequest) ProtoMessage() {}
func (*AddTableFilesRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{26}
return fileDescriptor_702c187af9ca94ec, []int{27}
}
func (m *AddTableFilesRequest) XXX_Unmarshal(b []byte) error {
@@ -1328,7 +1391,7 @@ func (m *AddTableFilesResponse) Reset() { *m = AddTableFilesResponse{} }
func (m *AddTableFilesResponse) String() string { return proto.CompactTextString(m) }
func (*AddTableFilesResponse) ProtoMessage() {}
func (*AddTableFilesResponse) Descriptor() ([]byte, []int) {
return fileDescriptor_702c187af9ca94ec, []int{27}
return fileDescriptor_702c187af9ca94ec, []int{28}
}
func (m *AddTableFilesResponse) XXX_Unmarshal(b []byte) error {
@@ -1368,6 +1431,7 @@ func init() {
proto.RegisterType((*UploadLoc)(nil), "dolt.services.remotesapi.v1alpha1.UploadLoc")
proto.RegisterType((*GetDownloadLocsRequest)(nil), "dolt.services.remotesapi.v1alpha1.GetDownloadLocsRequest")
proto.RegisterType((*GetDownloadLocsResponse)(nil), "dolt.services.remotesapi.v1alpha1.GetDownloadLocsResponse")
proto.RegisterType((*TableFileDetails)(nil), "dolt.services.remotesapi.v1alpha1.TableFileDetails")
proto.RegisterType((*GetUploadLocsRequest)(nil), "dolt.services.remotesapi.v1alpha1.GetUploadLocsRequest")
proto.RegisterType((*GetUploadLocsResponse)(nil), "dolt.services.remotesapi.v1alpha1.GetUploadLocsResponse")
proto.RegisterType((*RebaseRequest)(nil), "dolt.services.remotesapi.v1alpha1.RebaseRequest")
@@ -1392,78 +1456,83 @@ func init() {
}
var fileDescriptor_702c187af9ca94ec = []byte{
// 1136 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0xdd, 0x6f, 0x1b, 0x45,
0x10, 0xcf, 0xd9, 0xc1, 0x89, 0xc7, 0x1f, 0x71, 0x4e, 0x4d, 0x6a, 0x19, 0xa1, 0xa6, 0x2b, 0x81,
0x42, 0x4b, 0x7d, 0x8d, 0x5b, 0xa9, 0x2d, 0xbc, 0x40, 0x42, 0x1b, 0x47, 0x0a, 0xa8, 0x6c, 0x0a,
0x25, 0x44, 0xc8, 0x3a, 0x9f, 0xd7, 0xf6, 0x89, 0xf3, 0xae, 0x7b, 0xbb, 0x17, 0x1e, 0x91, 0x10,
0x5f, 0x2f, 0x88, 0x17, 0x9e, 0xf8, 0x67, 0x80, 0xff, 0x0c, 0xed, 0xc7, 0x39, 0x77, 0xae, 0xab,
0x6c, 0x2a, 0xd3, 0x37, 0xef, 0x64, 0x7f, 0x33, 0xbf, 0xdf, 0xec, 0xce, 0xec, 0xe4, 0xa0, 0x33,
0x60, 0x91, 0xf0, 0x38, 0x89, 0xcf, 0xc3, 0x80, 0x70, 0x2f, 0x26, 0x13, 0x26, 0x08, 0xf7, 0xa7,
0xa1, 0x77, 0xbe, 0xe7, 0x47, 0xd3, 0xb1, 0xbf, 0xe7, 0x05, 0xe3, 0x84, 0x7e, 0xc7, 0x05, 0x8b,
0x49, 0x7b, 0x1a, 0x33, 0xc1, 0xdc, 0x9b, 0x12, 0xd3, 0x4e, 0x31, 0xed, 0x0b, 0x4c, 0x3b, 0xc5,
0xa0, 0x07, 0x50, 0xc2, 0x64, 0xca, 0x8e, 0x06, 0x6e, 0x03, 0x8a, 0x2c, 0x1e, 0x35, 0x9d, 0x1d,
0x67, 0xb7, 0x8c, 0xe5, 0x4f, 0xf7, 0x6d, 0x28, 0xc7, 0x64, 0xca, 0x7a, 0xd4, 0x9f, 0x90, 0x66,
0x41, 0xd9, 0xd7, 0xa5, 0xe1, 0x73, 0x7f, 0x42, 0x10, 0x85, 0x46, 0xd7, 0xe7, 0x07, 0x2a, 0x24,
0x26, 0x2f, 0x12, 0xc2, 0x85, 0xbb, 0x0f, 0x6b, 0x0a, 0x10, 0x0e, 0x94, 0x9b, 0x4a, 0xe7, 0xfd,
0xf6, 0xa5, 0x0c, 0xda, 0x3a, 0x3c, 0x2e, 0xc5, 0x9a, 0xc6, 0x36, 0x94, 0xc6, 0x3e, 0x1f, 0x13,
0xde, 0x2c, 0xec, 0x14, 0x77, 0xab, 0xd8, 0xac, 0xd0, 0x6d, 0xd8, 0xcc, 0xc4, 0xe3, 0x53, 0x46,
0x39, 0x91, 0x9b, 0xfd, 0x3e, 0x27, 0x54, 0x34, 0x9d, 0x9d, 0xe2, 0xee, 0x5b, 0xd8, 0xac, 0xd0,
0x43, 0xa8, 0x76, 0x85, 0x98, 0x1e, 0x12, 0xa1, 0x00, 0x52, 0x5b, 0x12, 0x47, 0xa9, 0xb6, 0x24,
0x8e, 0x5e, 0x19, 0xe6, 0x29, 0x00, 0xf6, 0xe9, 0x88, 0x68, 0x9c, 0x0b, 0xab, 0xd2, 0xae, 0x80,
0x55, 0xac, 0x7e, 0x4b, 0x24, 0x1b, 0x0e, 0x39, 0x11, 0x2a, 0x25, 0xab, 0xd8, 0xac, 0xa4, 0x3d,
0x22, 0x74, 0x24, 0xc6, 0xcd, 0xe2, 0x8e, 0xb3, 0x5b, 0xc3, 0x66, 0x85, 0x46, 0x33, 0x2e, 0xca,
0xf1, 0x02, 0x2e, 0x8f, 0xa1, 0x14, 0xcb, 0x3f, 0x69, 0x2e, 0x95, 0xce, 0x1d, 0x9b, 0xac, 0xcd,
0x48, 0x62, 0x03, 0x46, 0xff, 0x38, 0x50, 0xf9, 0x94, 0x7d, 0x4f, 0x23, 0xe6, 0x0f, 0x8e, 0x59,
0xe0, 0x1e, 0xc3, 0xfa, 0x58, 0x88, 0x69, 0x6f, 0x44, 0x84, 0x39, 0x0e, 0xcf, 0xc2, 0x71, 0x36,
0x6f, 0xdd, 0x15, 0xbc, 0x36, 0xd6, 0x6b, 0xf7, 0x39, 0xd4, 0x53, 0x6f, 0x3d, 0x15, 0x50, 0xc9,
0xbf, 0x92, 0x4f, 0xc5, 0xb9, 0xbb, 0x82, 0xab, 0xe3, 0xcc, 0x7a, 0x1f, 0x60, 0x3d, 0x62, 0x81,
0x2f, 0x42, 0x46, 0xd1, 0xbb, 0xb0, 0x29, 0xf7, 0x3e, 0x65, 0x5c, 0x3c, 0xf3, 0xfb, 0x11, 0x79,
0x12, 0x46, 0x0b, 0x12, 0x86, 0xfe, 0x74, 0xa0, 0xfc, 0xe5, 0x34, 0xd5, 0xf9, 0x1e, 0x6c, 0x08,
0xb9, 0xb9, 0x37, 0x0c, 0x23, 0xd2, 0xcb, 0x9c, 0x57, 0x4d, 0xa4, 0x3e, 0xba, 0xf2, 0xe0, 0x4e,
0xa0, 0xac, 0x14, 0x4c, 0x19, 0x17, 0x86, 0xfc, 0x7d, 0x4b, 0xf2, 0x39, 0x42, 0xdd, 0x15, 0xac,
0x12, 0x2b, 0x8d, 0x39, 0xf6, 0x3f, 0xc0, 0xf6, 0x21, 0x11, 0x99, 0x23, 0x58, 0x6a, 0x61, 0xdc,
0x84, 0xaa, 0x2a, 0xf0, 0x5e, 0xee, 0xde, 0x56, 0x94, 0xad, 0xab, 0x2f, 0xef, 0xb7, 0x70, 0xfd,
0x25, 0x02, 0xa6, 0x52, 0xf6, 0x61, 0x35, 0x62, 0x01, 0x57, 0x75, 0x52, 0xe9, 0xb4, 0x2d, 0xc2,
0x67, 0xdc, 0x60, 0x85, 0x45, 0xbf, 0x38, 0x70, 0xed, 0x90, 0x88, 0x59, 0xe6, 0x97, 0x2a, 0xef,
0x16, 0x6c, 0xce, 0x9d, 0xe2, 0x4c, 0xe3, 0x46, 0xee, 0x1c, 0x09, 0x47, 0xa7, 0xb0, 0x35, 0xc7,
0xc3, 0xa8, 0xfc, 0x38, 0xa7, 0xf2, 0x03, 0x0b, 0x16, 0x33, 0x27, 0x46, 0xe3, 0x09, 0xd4, 0x30,
0xe9, 0xfb, 0x9c, 0x2c, 0x51, 0x1b, 0x6a, 0x40, 0x3d, 0x75, 0xaa, 0x89, 0xa2, 0x2f, 0xa0, 0x82,
0x19, 0x13, 0xcb, 0x0c, 0x72, 0x1b, 0xaa, 0xda, 0xa5, 0xc9, 0x85, 0xec, 0xde, 0x8c, 0x89, 0x6c,
0x41, 0xac, 0x4b, 0x83, 0xcc, 0x21, 0x7a, 0x0c, 0x75, 0x55, 0xe1, 0xea, 0x52, 0x1f, 0xd1, 0x21,
0x5b, 0xd8, 0xea, 0x6e, 0x80, 0xbe, 0x5e, 0xbd, 0x80, 0x25, 0x54, 0xd7, 0x4c, 0x0d, 0x83, 0x32,
0x1d, 0x48, 0x0b, 0xfa, 0xb7, 0x00, 0xb5, 0x03, 0x36, 0x99, 0x84, 0xcb, 0x54, 0xe2, 0x36, 0x61,
0x2d, 0x48, 0xe2, 0x98, 0x98, 0x90, 0x55, 0x9c, 0x2e, 0x25, 0xc9, 0xc8, 0xe7, 0x42, 0x75, 0xd8,
0x2a, 0x56, 0xbf, 0xdd, 0x33, 0x68, 0x68, 0x92, 0xfa, 0xfa, 0x84, 0x74, 0xc8, 0x9a, 0xab, 0xea,
0xfc, 0xf7, 0x2c, 0x42, 0xe7, 0xb3, 0x80, 0xeb, 0x41, 0x3e, 0x2b, 0x3e, 0xb8, 0x41, 0x14, 0x12,
0x2a, 0x7a, 0x4a, 0xd5, 0x90, 0xc5, 0x13, 0x5f, 0x34, 0xeb, 0x4a, 0xd9, 0x3d, 0x1b, 0xf7, 0x0a,
0x2c, 0xf5, 0x3d, 0x51, 0x50, 0xdc, 0x08, 0xe6, 0x2c, 0xe8, 0x16, 0xd4, 0xd3, 0x14, 0x9a, 0x93,
0x6b, 0xc2, 0x1a, 0x4f, 0x82, 0x80, 0x70, 0xae, 0x72, 0xb8, 0x8e, 0xd3, 0x25, 0xfa, 0xdb, 0x51,
0x2d, 0x46, 0xa2, 0x3f, 0x23, 0xc2, 0x1f, 0xf8, 0xc2, 0x5f, 0x66, 0xe2, 0xdf, 0x80, 0xda, 0x33,
0xd5, 0xa2, 0xf2, 0x02, 0x8c, 0xec, 0x1b, 0x50, 0xa1, 0xfd, 0x61, 0xef, 0x9c, 0xc4, 0x3c, 0x64,
0xd4, 0xf4, 0x7b, 0xa0, 0xfd, 0xe1, 0x57, 0xda, 0xa2, 0x37, 0xf0, 0xd9, 0x86, 0x42, 0xba, 0x81,
0x9b, 0x0d, 0xe8, 0x19, 0x34, 0xe6, 0x29, 0x2c, 0xc1, 0xeb, 0x19, 0x6c, 0x1d, 0x87, 0x99, 0xfe,
0xbf, 0xcc, 0xb6, 0x87, 0x4e, 0xa1, 0x36, 0x73, 0xac, 0x6e, 0xdc, 0x75, 0x58, 0x53, 0x1d, 0xd0,
0x38, 0x2d, 0xe3, 0x92, 0x5c, 0x1e, 0x0d, 0xdc, 0x77, 0x00, 0x68, 0x32, 0xe9, 0xe9, 0x21, 0xcf,
0xd4, 0x62, 0x99, 0x26, 0x13, 0x3d, 0x12, 0xa5, 0xaf, 0x64, 0xf1, 0xe2, 0x95, 0xfc, 0xc3, 0x81,
0xed, 0x79, 0xe2, 0x16, 0xbd, 0xc1, 0xfd, 0x3a, 0xf7, 0x9e, 0xaa, 0x7a, 0xd2, 0x73, 0xc9, 0x5d,
0x0b, 0x79, 0x39, 0x31, 0x99, 0x17, 0x58, 0x2e, 0xd1, 0x5f, 0x05, 0xb8, 0xf6, 0xc9, 0x60, 0xf0,
0xbf, 0x64, 0xf2, 0x15, 0x97, 0xb7, 0xb0, 0xc4, 0xcb, 0xbb, 0xb0, 0xd5, 0x14, 0x97, 0xd4, 0x6a,
0xd0, 0x1e, 0x6c, 0xcd, 0xe5, 0xe6, 0xb2, 0x76, 0xd0, 0xf9, 0xa9, 0x0c, 0x9b, 0xca, 0xeb, 0x89,
0x1c, 0xfa, 0x4f, 0x74, 0x70, 0xf7, 0x37, 0x07, 0x36, 0xe6, 0x6a, 0xcc, 0x7d, 0x64, 0xc1, 0x6f,
0x71, 0x63, 0x69, 0x7d, 0xf8, 0x3a, 0x50, 0x43, 0xfd, 0x1c, 0xca, 0xb3, 0xa1, 0xdd, 0xb5, 0x39,
0x84, 0xf9, 0x7f, 0x29, 0x5a, 0xf7, 0xaf, 0x06, 0x32, 0x71, 0x7f, 0xd7, 0x93, 0x4a, 0x66, 0x84,
0x51, 0x13, 0x1a, 0xb7, 0xcd, 0xc3, 0x82, 0x19, 0xce, 0x36, 0x0f, 0x0b, 0xa7, 0xaf, 0x5f, 0x1d,
0x70, 0xb3, 0x13, 0x8b, 0x61, 0xf3, 0xc0, 0xce, 0xe5, 0x4b, 0x03, 0x57, 0xeb, 0xe1, 0xd5, 0x81,
0x86, 0xc9, 0x44, 0xfe, 0xbf, 0x27, 0x47, 0x11, 0xf7, 0xae, 0x55, 0x89, 0x65, 0x46, 0xa1, 0xd6,
0xde, 0x15, 0x10, 0x26, 0xdc, 0x08, 0x56, 0xe5, 0x50, 0xe2, 0xda, 0x0c, 0x9c, 0x99, 0x81, 0xa8,
0xe5, 0x59, 0xef, 0xbf, 0xd0, 0xa5, 0x5f, 0x51, 0x2b, 0x5d, 0xb9, 0x99, 0xc5, 0x4a, 0xd7, 0xdc,
0x13, 0xfd, 0xb3, 0x03, 0xf5, 0x7c, 0x6f, 0x75, 0x6d, 0xce, 0x64, 0xe1, 0x3b, 0xd2, 0x7a, 0xf4,
0x1a, 0x48, 0xc3, 0xe3, 0x47, 0x07, 0x6a, 0xb9, 0xae, 0x61, 0x75, 0xa7, 0x16, 0xf5, 0x60, 0xab,
0x3b, 0xb5, 0xb0, 0x41, 0xed, 0x9f, 0x7e, 0xf3, 0x7c, 0x14, 0x8a, 0x71, 0xd2, 0x6f, 0x07, 0x6c,
0xe2, 0x45, 0xe1, 0x8b, 0x24, 0x94, 0x5d, 0xe0, 0x4e, 0x48, 0x03, 0x4f, 0x7d, 0xb5, 0x18, 0x31,
0x6f, 0x44, 0xa8, 0xa7, 0x3e, 0x4a, 0x78, 0x97, 0x7e, 0xc7, 0xf8, 0xe8, 0xc2, 0xd6, 0x2f, 0x29,
0xcc, 0xbd, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0xac, 0x42, 0x71, 0x29, 0xfe, 0x10, 0x00, 0x00,
// 1210 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0xdf, 0x6f, 0x1b, 0xc5,
0x13, 0xef, 0xd9, 0xf9, 0x3a, 0xf1, 0xf8, 0x47, 0x9d, 0x55, 0x93, 0x5a, 0xfe, 0x0a, 0x35, 0x3d,
0xa9, 0x28, 0xb4, 0xd4, 0x6e, 0xd2, 0x4a, 0x6d, 0xe1, 0x05, 0x92, 0xb6, 0x71, 0xa4, 0x80, 0xca,
0xa6, 0x50, 0x42, 0x84, 0xac, 0xf3, 0x79, 0x6d, 0x9f, 0x38, 0xdf, 0xba, 0xb7, 0xeb, 0xf0, 0x88,
0x84, 0x40, 0xf0, 0x82, 0x78, 0xe1, 0x89, 0x7f, 0x06, 0xf8, 0x73, 0xf8, 0x2f, 0xd0, 0xce, 0xee,
0x39, 0x77, 0x57, 0x57, 0xd9, 0x54, 0x86, 0xb7, 0xec, 0x64, 0x67, 0xe6, 0x33, 0x9f, 0xf9, 0xb1,
0x73, 0x86, 0xdd, 0x01, 0x0f, 0x65, 0x47, 0xb0, 0xf8, 0x2c, 0xf0, 0x99, 0xe8, 0xc4, 0x6c, 0xc2,
0x25, 0x13, 0xde, 0x34, 0xe8, 0x9c, 0xed, 0x78, 0xe1, 0x74, 0xec, 0xed, 0x74, 0xfc, 0xf1, 0x2c,
0xfa, 0x46, 0x48, 0x1e, 0xb3, 0xf6, 0x34, 0xe6, 0x92, 0x93, 0x9b, 0x4a, 0xa7, 0x9d, 0xe8, 0xb4,
0xcf, 0x75, 0xda, 0x89, 0x8e, 0xfb, 0x10, 0x4a, 0x94, 0x4d, 0xf9, 0xe1, 0x80, 0x34, 0xa0, 0xc8,
0xe3, 0x51, 0xd3, 0xd9, 0x72, 0xb6, 0xcb, 0x54, 0xfd, 0x49, 0xfe, 0x0f, 0xe5, 0x98, 0x4d, 0x79,
0x2f, 0xf2, 0x26, 0xac, 0x59, 0x40, 0xf9, 0x9a, 0x12, 0x7c, 0xea, 0x4d, 0x98, 0x1b, 0x41, 0xa3,
0xeb, 0x89, 0x7d, 0x74, 0x49, 0xd9, 0xab, 0x19, 0x13, 0x92, 0xec, 0xc1, 0x2a, 0x2a, 0x04, 0x03,
0x34, 0x53, 0xd9, 0x7d, 0xaf, 0x7d, 0x21, 0x82, 0xb6, 0x76, 0x4f, 0x4b, 0xb1, 0x86, 0xb1, 0x09,
0xa5, 0xb1, 0x27, 0xc6, 0x4c, 0x34, 0x0b, 0x5b, 0xc5, 0xed, 0x2a, 0x35, 0x27, 0xf7, 0x0e, 0xac,
0xa7, 0xfc, 0x89, 0x29, 0x8f, 0x04, 0x53, 0x97, 0xbd, 0xbe, 0x60, 0x91, 0x6c, 0x3a, 0x5b, 0xc5,
0xed, 0xff, 0x51, 0x73, 0x72, 0x1f, 0x41, 0xb5, 0x2b, 0xe5, 0xf4, 0x80, 0x49, 0x54, 0x50, 0xb1,
0xcd, 0xe2, 0x30, 0x89, 0x6d, 0x16, 0x87, 0x6f, 0x74, 0xf3, 0x1c, 0x80, 0x7a, 0xd1, 0x88, 0x69,
0x3d, 0x02, 0x2b, 0x4a, 0x8e, 0x8a, 0x55, 0x8a, 0x7f, 0x2b, 0x4d, 0x3e, 0x1c, 0x0a, 0x26, 0x91,
0x92, 0x15, 0x6a, 0x4e, 0x4a, 0x1e, 0xb2, 0x68, 0x24, 0xc7, 0xcd, 0xe2, 0x96, 0xb3, 0x5d, 0xa3,
0xe6, 0xe4, 0x8e, 0xe6, 0x58, 0xd0, 0xf0, 0x02, 0x2c, 0x4f, 0xa1, 0x14, 0xab, 0x7f, 0x69, 0x2c,
0x95, 0xdd, 0xbb, 0x36, 0xac, 0xcd, 0x41, 0x52, 0xa3, 0xec, 0xfe, 0xe9, 0x40, 0xe5, 0x09, 0xff,
0x36, 0x0a, 0xb9, 0x37, 0x38, 0xe2, 0x3e, 0x39, 0x82, 0xb5, 0xb1, 0x94, 0xd3, 0xde, 0x88, 0x49,
0x93, 0x8e, 0x8e, 0x85, 0xe1, 0x34, 0x6f, 0xdd, 0x2b, 0x74, 0x75, 0xac, 0xcf, 0xe4, 0x25, 0xd4,
0x13, 0x6b, 0x3d, 0x74, 0x88, 0xe1, 0x5f, 0xca, 0x26, 0x62, 0xee, 0x5e, 0xa1, 0xd5, 0x71, 0xea,
0xbc, 0x07, 0xb0, 0x16, 0x72, 0xdf, 0x93, 0x01, 0x8f, 0xdc, 0x5b, 0xb0, 0xae, 0xee, 0x3e, 0xe7,
0x42, 0xbe, 0xf0, 0xfa, 0x21, 0x7b, 0x16, 0x84, 0x0b, 0x08, 0x73, 0x7f, 0x73, 0xa0, 0xfc, 0xf9,
0x34, 0x89, 0xf3, 0x5d, 0xb8, 0x2a, 0xd5, 0xe5, 0xde, 0x30, 0x08, 0x59, 0x2f, 0x95, 0xaf, 0x9a,
0x4c, 0x6c, 0x74, 0x55, 0xe2, 0x8e, 0xa1, 0x8c, 0x11, 0x4c, 0xb9, 0x90, 0x06, 0xfc, 0x03, 0x4b,
0xf0, 0x19, 0x40, 0xdd, 0x2b, 0x14, 0x89, 0x55, 0xc2, 0x0c, 0xfa, 0xef, 0x60, 0xf3, 0x80, 0xc9,
0x54, 0x0a, 0x96, 0xda, 0x18, 0x37, 0xa1, 0x8a, 0x0d, 0xde, 0xcb, 0xd4, 0x6d, 0x05, 0x65, 0x5d,
0x5d, 0xbc, 0x5f, 0xc3, 0xf5, 0xd7, 0x00, 0x98, 0x4e, 0xd9, 0x83, 0x95, 0x90, 0xfb, 0x02, 0xfb,
0xa4, 0xb2, 0xdb, 0xb6, 0x70, 0x9f, 0x32, 0x43, 0x51, 0xd7, 0x0d, 0xa1, 0x31, 0x27, 0xe1, 0x09,
0x93, 0x5e, 0x10, 0x0a, 0x52, 0x87, 0x82, 0x09, 0xaa, 0x4a, 0x0b, 0xc1, 0x80, 0xdc, 0x82, 0xba,
0xcf, 0x23, 0xc9, 0x22, 0xd9, 0x33, 0xdd, 0xa0, 0xbb, 0xa4, 0x66, 0xa4, 0x47, 0x28, 0xc4, 0x60,
0xcc, 0x35, 0x4c, 0x58, 0x11, 0x0d, 0x54, 0x8c, 0x4c, 0x85, 0xe3, 0xfe, 0xed, 0xc0, 0xb5, 0x03,
0x26, 0xe7, 0x79, 0x5e, 0x2a, 0x99, 0xb7, 0x61, 0x3d, 0x57, 0x33, 0x73, 0x46, 0xaf, 0x66, 0xaa,
0x86, 0x09, 0xe2, 0x01, 0x49, 0xdd, 0x1d, 0xe8, 0xc0, 0x9b, 0x45, 0x24, 0xf2, 0xbe, 0x85, 0xeb,
0x3c, 0x67, 0xb4, 0x21, 0x73, 0x12, 0xf7, 0x04, 0x36, 0x72, 0xa1, 0x9a, 0xb4, 0x7d, 0x94, 0x49,
0xdb, 0xfb, 0x16, 0xde, 0xe6, 0x46, 0x4c, 0xd2, 0x8e, 0xa1, 0x46, 0x59, 0xdf, 0x13, 0x6c, 0x89,
0xf4, 0xb9, 0x0d, 0xa8, 0x27, 0x46, 0x35, 0x50, 0xf7, 0x33, 0xa8, 0x50, 0xce, 0xe5, 0x32, 0x9d,
0xdc, 0x81, 0xaa, 0x36, 0x69, 0xb8, 0x50, 0xcf, 0x11, 0xe7, 0x32, 0xdd, 0xe1, 0x6b, 0x4a, 0x80,
0xd5, 0xf2, 0x14, 0xea, 0x38, 0xb2, 0x90, 0xec, 0xc3, 0x68, 0xc8, 0x17, 0xce, 0xee, 0x1b, 0xa0,
0xfb, 0xa5, 0xe7, 0xf3, 0x59, 0xa4, 0x87, 0x40, 0x8d, 0x02, 0x8a, 0xf6, 0x95, 0xc4, 0xfd, 0xab,
0x00, 0xb5, 0x7d, 0x3e, 0x99, 0x04, 0xcb, 0x8c, 0x84, 0x34, 0x61, 0xd5, 0x9f, 0xc5, 0x31, 0x33,
0x2e, 0xab, 0x34, 0x39, 0x2a, 0x90, 0xa1, 0x27, 0xa4, 0xa9, 0x7f, 0xfc, 0x9b, 0x9c, 0x42, 0x43,
0x83, 0xd4, 0x55, 0x17, 0x44, 0x43, 0xde, 0x5c, 0xc1, 0xfc, 0xef, 0x58, 0xb8, 0xce, 0xb2, 0x40,
0xeb, 0x7e, 0x96, 0x15, 0x0f, 0x88, 0x1f, 0x06, 0xaa, 0xef, 0x30, 0xaa, 0x21, 0x8f, 0x27, 0x9e,
0x6c, 0xd6, 0x31, 0x32, 0x9b, 0x62, 0xde, 0x47, 0x65, 0x15, 0xdf, 0x33, 0x54, 0xa5, 0x0d, 0x3f,
0x27, 0x71, 0x6f, 0x43, 0x3d, 0xa1, 0xd0, 0x64, 0xae, 0x09, 0xab, 0x62, 0xe6, 0xfb, 0x4c, 0x08,
0xe4, 0x70, 0x8d, 0x26, 0x47, 0xf7, 0x0f, 0x07, 0x67, 0xa6, 0xd2, 0xfe, 0x84, 0x49, 0x6f, 0xe0,
0x49, 0x6f, 0x99, 0xc4, 0xff, 0x07, 0xd1, 0x9e, 0xe2, 0xcc, 0xcd, 0x06, 0x60, 0xc2, 0xbe, 0x01,
0x95, 0xa8, 0x3f, 0xec, 0x9d, 0xb1, 0x58, 0x04, 0x3c, 0x32, 0x0f, 0x18, 0x44, 0xfd, 0xe1, 0x17,
0x5a, 0xa2, 0x2f, 0x88, 0xf9, 0x85, 0x42, 0x72, 0x41, 0x98, 0x0b, 0xee, 0x0b, 0x68, 0xe4, 0x21,
0x2c, 0xc1, 0xea, 0x29, 0x6c, 0x1c, 0x05, 0xa9, 0x07, 0x6d, 0x99, 0x93, 0xd5, 0x3d, 0x81, 0xda,
0xdc, 0x30, 0x56, 0xdc, 0x75, 0x58, 0xc5, 0xc1, 0x69, 0x8c, 0x96, 0x69, 0x49, 0x1d, 0x0f, 0x07,
0xe4, 0x1d, 0x80, 0x68, 0x36, 0xe9, 0xe9, 0xad, 0xd5, 0xf4, 0x62, 0x39, 0x9a, 0x4d, 0xf4, 0x8e,
0x97, 0x3c, 0xfb, 0xc5, 0xf3, 0x67, 0xff, 0x57, 0x07, 0x36, 0xf3, 0xc0, 0x2d, 0x66, 0x03, 0xf9,
0x32, 0xb3, 0x20, 0x60, 0x3f, 0xe9, 0x45, 0xeb, 0xde, 0x65, 0xa6, 0x37, 0xb6, 0xd3, 0xf9, 0x4a,
0xa1, 0x8e, 0xee, 0xef, 0x05, 0xb8, 0xf6, 0xf1, 0x60, 0xf0, 0xaf, 0x30, 0xf9, 0x86, 0xe2, 0x2d,
0x2c, 0xb1, 0x78, 0x17, 0x8e, 0x9a, 0xe2, 0x92, 0x46, 0x8d, 0xbb, 0x03, 0x1b, 0x39, 0x6e, 0x2e,
0x1a, 0x07, 0xbb, 0x3f, 0x94, 0x61, 0x1d, 0xad, 0x1e, 0xab, 0xaf, 0x98, 0x63, 0xed, 0x9c, 0xfc,
0xec, 0xc0, 0xd5, 0x5c, 0x8f, 0x91, 0xc7, 0x16, 0xf8, 0x16, 0x0f, 0x96, 0xd6, 0x07, 0x6f, 0xa3,
0x6a, 0xa0, 0x9f, 0x41, 0x79, 0xfe, 0x15, 0x42, 0x6c, 0x92, 0x90, 0xff, 0x46, 0x6a, 0x3d, 0xb8,
0x9c, 0x92, 0xf1, 0xfb, 0x8b, 0x5e, 0x86, 0x52, 0x3b, 0x19, 0xae, 0x9c, 0xc2, 0x96, 0x87, 0x05,
0x4b, 0xa9, 0x2d, 0x0f, 0x0b, 0xd7, 0xc9, 0x9f, 0x1c, 0x20, 0xe9, 0x8d, 0xc5, 0xa0, 0x79, 0x68,
0x67, 0xf2, 0xb5, 0x9d, 0xae, 0xf5, 0xe8, 0xf2, 0x8a, 0x06, 0xc9, 0x44, 0x7d, 0xc0, 0xaa, 0x55,
0x84, 0xdc, 0xb3, 0x6a, 0xb1, 0xd4, 0x2a, 0xd4, 0xda, 0xb9, 0x84, 0x86, 0x71, 0x37, 0x82, 0x15,
0xb5, 0x94, 0x10, 0x9b, 0x0d, 0x3a, 0xb5, 0x10, 0xb5, 0x3a, 0xd6, 0xf7, 0xcf, 0xe3, 0xd2, 0xaf,
0xa8, 0x55, 0x5c, 0x99, 0x9d, 0xc5, 0x2a, 0xae, 0xdc, 0x13, 0xfd, 0xa3, 0x03, 0xf5, 0xec, 0x6c,
0x25, 0x36, 0x39, 0x59, 0xf8, 0x8e, 0xb4, 0x1e, 0xbf, 0x85, 0xa6, 0xc1, 0xf1, 0xbd, 0x03, 0xb5,
0xcc, 0xd4, 0xb0, 0xaa, 0xa9, 0x45, 0x33, 0xd8, 0xaa, 0xa6, 0x16, 0x0e, 0xa8, 0xbd, 0x93, 0xaf,
0x5e, 0x8e, 0x02, 0x39, 0x9e, 0xf5, 0xdb, 0x3e, 0x9f, 0x74, 0xc2, 0xe0, 0xd5, 0x2c, 0x50, 0x53,
0xe0, 0x6e, 0x10, 0xf9, 0x1d, 0xfc, 0x19, 0x66, 0xc4, 0x3b, 0x23, 0x16, 0x75, 0xf0, 0x57, 0x96,
0xce, 0x85, 0x3f, 0xcc, 0x7c, 0x78, 0x2e, 0xeb, 0x97, 0x50, 0xe7, 0xfe, 0x3f, 0x01, 0x00, 0x00,
0xff, 0xff, 0x9b, 0x26, 0xa3, 0x10, 0xcf, 0x11, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.

2
go/go.sum
View File

@@ -196,6 +196,8 @@ github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/liquidata-inc/go-mysql-server v0.4.1-0.20191003230430-3a1a09c79ada h1:2D7VUQT4JHEB1N3xUG/MqULhDfWWg65LmPDBjXWBZcs=
github.com/liquidata-inc/go-mysql-server v0.4.1-0.20191003230430-3a1a09c79ada/go.mod h1:DdWE0ku/mNfuLsRJIrHeHpDtB7am+6oopxEsQKmVkx8=
github.com/liquidata-inc/go-mysql-server v0.4.1-0.20191017183442-4b9329eafa5b h1:fdht6iaUhHYJEfSUY1m1KAghyi6zfyfv+12gaSsaQ+o=
github.com/liquidata-inc/go-mysql-server v0.4.1-0.20191017183442-4b9329eafa5b/go.mod h1:DdWE0ku/mNfuLsRJIrHeHpDtB7am+6oopxEsQKmVkx8=
github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0 h1:phMgajKClMUiIr+hF2LGt8KRuUa2Vd2GI1sNgHgSXoU=
github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0/go.mod h1:YC1rI9k5gx8D02ljlbxDfZe80s/iq8bGvaaQsvR+qxs=
github.com/liquidata-inc/mmap-go v1.0.3 h1:2LndAeAtup9rpvUmu4wZSYCsjCQ0Zpc+NqE+6+PnT7g=

15
go/libraries/doltcore/remotestorage/chunk_store.go
View File

@@ -893,9 +893,20 @@ func (dcs *DoltChunkStore) getDownloadWorkForLoc(ctx context.Context, getRange *
}
// WriteTableFile reads a table file from the provided reader and writes it to the chunk store.
func (dcs *DoltChunkStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader) error {
func (dcs *DoltChunkStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader, contentLength uint64, contentHash []byte) error {
fileIdBytes := hash.Parse(fileId)
req := &remotesapi.GetUploadLocsRequest{RepoId: dcs.getRepoId(), TableFileHashes: [][]byte{fileIdBytes[:]}}
tfd := &remotesapi.TableFileDetails{
Id: fileIdBytes[:],
ContentLength: contentLength,
ContentHash: contentHash,
}
req := &remotesapi.GetUploadLocsRequest{
RepoId: dcs.getRepoId(),
TableFileDetails: []*remotesapi.TableFileDetails{tfd},
// redundant and deprecated. Still setting for compatibility, but will remove promptly.
TableFileHashes: [][]byte{fileIdBytes[:]},
}
resp, err := dcs.csClient.GetUploadLocations(ctx, req)
if err != nil {

2
go/store/datas/pull.go
View File

@@ -132,7 +132,7 @@ func clone(ctx context.Context, srcTS, sinkTS nbs.TableFileStore, eventCh chan<-
}
}()
err = sinkTS.WriteTableFile(ctx, tblFile.FileID(), tblFile.NumChunks(), rd)
err = sinkTS.WriteTableFile(ctx, tblFile.FileID(), tblFile.NumChunks(), rd, 0, nil)
if err != nil {
if eventCh != nil {

2
go/store/datas/pull_test.go
View File

@@ -421,7 +421,7 @@ func (ttfs *TestTableFileStore) Sources(ctx context.Context) (hash.Hash, []nbs.T
return ttfs.root, tblFiles, nil
}
func (ttfs *TestTableFileStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader) error {
func (ttfs *TestTableFileStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader, contentLength uint64, contentHash []byte) error {
tblFile := &TestTableFileWriter{fileId, numChunks, bytes.NewBuffer(nil), ttfs}
_, err := io.Copy(tblFile, rd)

18
go/store/datas/puller.go
View File

@@ -182,9 +182,11 @@ func NewPuller(ctx context.Context, tempDir string, chunksPerTF int, srcDB, sink
func (p *Puller) processCompletedTables(ctx context.Context, ae *atomicerr.AtomicError, completedTables <-chan FilledWriters) {
type tempTblFile struct {
id string
path string
numChunks int
id string
path string
numChunks int
contentLen uint64
contentHash []byte
}
var tblFiles []tempTblFile
@@ -209,7 +211,13 @@ func (p *Puller) processCompletedTables(ctx context.Context, ae *atomicerr.Atomi
continue
}
tblFiles = append(tblFiles, tempTblFile{id, path, tblFile.wr.Size()})
tblFiles = append(tblFiles, tempTblFile{
id: id,
path: path,
numChunks: tblFile.wr.Size(),
contentLen: tblFile.wr.ContentLength(),
contentHash: tblFile.wr.GetMD5(),
})
}
if ae.IsSet() {
@@ -239,7 +247,7 @@ func (p *Puller) processCompletedTables(ctx context.Context, ae *atomicerr.Atomi
p.eventCh <- NewTFPullerEvent(StartUploadTableFile, details)
fWithSize := FileReaderWithSize{f, fi.Size()}
err = p.sinkDB.chunkStore().(nbs.TableFileStore).WriteTableFile(ctx, tmpTblFile.id, tmpTblFile.numChunks, fWithSize)
err = p.sinkDB.chunkStore().(nbs.TableFileStore).WriteTableFile(ctx, tmpTblFile.id, tmpTblFile.numChunks, fWithSize, tmpTblFile.contentLen, tmpTblFile.contentHash)
go func() {
_ = os.Remove(tmpTblFile.path)

57
go/store/nbs/byte_sink.go
View File

@@ -15,7 +15,9 @@
package nbs
import (
"crypto/md5"
"errors"
"hash"
"io"
"os"
"path/filepath"
@@ -287,3 +289,58 @@ func (sink *BufferedFileByteSink) FlushToFile(path string) (err error) {
return os.Rename(sink.path, path)
}
// HashingByteSink is a ByteSink that keeps an md5 hash of all the data written to it.
type HashingByteSink struct {
backingSink ByteSink
hasher hash.Hash
size uint64
}
func NewHashingByteSink(backingSink ByteSink) *HashingByteSink {
return &HashingByteSink{backingSink: backingSink, hasher: md5.New(), size: 0}
}
// Write writes a byte array to the sink.
func (sink *HashingByteSink) Write(src []byte) (int, error) {
nWritten, err := sink.backingSink.Write(src)
if err != nil {
return 0, err
}
nHashed, err := sink.hasher.Write(src[:nWritten])
if err != nil {
return 0, err
} else if nWritten != nHashed {
return 0, errors.New("failed to hash all the data that was written to the byte sink.")
}
sink.size += uint64(nWritten)
return nWritten, nil
}
// Flush writes all the data that was written to the ByteSink to the supplied writer
func (sink *HashingByteSink) Flush(wr io.Writer) error {
return sink.backingSink.Flush(wr)
}
// FlushToFile writes all the data that was written to the ByteSink to a file at the given path
func (sink *HashingByteSink) FlushToFile(path string) error {
return sink.backingSink.FlushToFile(path)
}
// GetMD5 gets the MD5 hash of all the bytes written to the sink
func (sink *HashingByteSink) GetMD5() []byte {
result := make([]byte, 0, 128)
sink.hasher.Sum(result)
return result
}
// Size gets the number of bytes written to the sink
func (sink *HashingByteSink) Size() uint64 {
return sink.size
}

14
go/store/nbs/cmp_chunk_table_writer.go
View File

@@ -36,7 +36,7 @@ var ErrAlreadyFinished = errors.New("already Finished")
// CmpChunkTableWriter writes CompressedChunks to a table file
type CmpChunkTableWriter struct {
sink ByteSink
sink *HashingByteSink
totalCompressedData uint64
totalUncompressedData uint64
prefixes prefixIndexSlice // TODO: This is in danger of exploding memory
@@ -51,7 +51,7 @@ func NewCmpChunkTableWriter() (*CmpChunkTableWriter, error) {
return nil, err
}
return &CmpChunkTableWriter{s, 0, 0, nil, nil}, nil
return &CmpChunkTableWriter{NewHashingByteSink(s), 0, 0, nil, nil}, nil
}
// Size returns the number of compressed chunks that have been added
@@ -59,6 +59,16 @@ func (tw *CmpChunkTableWriter) Size() int {
return len(tw.prefixes)
}
// Gets the size of the entire table file in bytes
func (tw *CmpChunkTableWriter) ContentLength() uint64 {
return tw.sink.Size()
}
// Gets the MD5 of the entire table file
func (tw *CmpChunkTableWriter) GetMD5() []byte {
return tw.sink.GetMD5()
}
// AddCmpChunk adds a compressed chunk
func (tw *CmpChunkTableWriter) AddCmpChunk(c CompressedChunk) error {
if len(c.CompressedData) == 0 {

2
go/store/nbs/store.go
View File

@@ -933,7 +933,7 @@ func (nbs *NomsBlockStore) Sources(ctx context.Context) (hash.Hash, []TableFile,
}
// WriteTableFile will read a table file from the provided reader and write it to the TableFileStore
func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader) error {
func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader, contentLength uint64, contentHash []byte) error {
fsPersister, ok := nbs.p.(*fsTablePersister)
if !ok {

2
go/store/nbs/store_test.go
View File

@@ -51,7 +51,7 @@ func TestNBSAsTableFileStore(t *testing.T) {
data, addr, err := buildTable(chunkData)
fileID := addr.String()
fileToData[fileID] = data
err = st.WriteTableFile(ctx, fileID, i+1, bytes.NewReader(data))
err = st.WriteTableFile(ctx, fileID, i+1, bytes.NewReader(data), 0, nil)
require.NoError(t, err)
}

2
go/store/nbs/table.go
View File

@@ -286,7 +286,7 @@ type TableFileStore interface {
Sources(ctx context.Context) (hash.Hash, []TableFile, error)
// WriteTableFile will read a table file from the provided reader and write it to the TableFileStore
WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader) error
WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader, contentLength uint64, contentHash []byte) error
// SetRootChunk changes the root chunk hash from the previous value to the new root.
SetRootChunk(ctx context.Context, root, previous hash.Hash) error

35
go/utils/remotesrv/grpc.go
View File

@@ -127,6 +127,25 @@ func (rs *RemoteChunkStore) getDownloadUrl(logger func(string), org, repoName, f
return fmt.Sprintf("http://%s/%s/%s/%s", rs.HttpHost, org, repoName, fileId), nil
}
func parseTableFileDetails(req *remotesapi.GetUploadLocsRequest) []*remotesapi.TableFileDetails {
tfd := req.GetTableFileDetails()
if len(tfd) == 0 {
_, hashToIdx := remotestorage.ParseByteSlices(req.TableFileHashes)
tfd = make([]*remotesapi.TableFileDetails, len(hashToIdx))
for h, i := range hashToIdx {
tfd[i] = &remotesapi.TableFileDetails{
Id: h[:],
ContentLength: 0,
ContentHash: nil,
}
}
}
return tfd
}
func (rs *RemoteChunkStore) GetUploadLocations(ctx context.Context, req *remotesapi.GetUploadLocsRequest) (*remotesapi.GetUploadLocsResponse, error) {
logger := getReqLogger("GRPC", "GetUploadLocations")
defer func() { logger("finished") }()
@@ -141,19 +160,19 @@ func (rs *RemoteChunkStore) GetUploadLocations(ctx context.Context, req *remotes
org := req.RepoId.Org
repoName := req.RepoId.RepoName
hashes, _ := remotestorage.ParseByteSlices(req.TableFileHashes)
tfds := parseTableFileDetails(req)
var locs []*remotesapi.UploadLoc
for h := range hashes {
tmp := h
url, err := rs.getUploadUrl(logger, org, repoName, h.String())
for _, tfd := range tfds {
h := hash.New(tfd.Id)
url, err := rs.getUploadUrl(logger, org, repoName, tfd)
if err != nil {
return nil, status.Error(codes.Internal, "Failed to get upload Url.")
}
loc := &remotesapi.UploadLoc_HttpPost{HttpPost: &remotesapi.HttpPostTableFile{Url: url}}
locs = append(locs, &remotesapi.UploadLoc{TableFileHash: tmp[:], Location: loc})
locs = append(locs, &remotesapi.UploadLoc{TableFileHash: h[:], Location: loc})
logger(fmt.Sprintf("sending upload location for chunk %s: %s", h.String(), url))
}
@@ -161,8 +180,10 @@ func (rs *RemoteChunkStore) GetUploadLocations(ctx context.Context, req *remotes
return &remotesapi.GetUploadLocsResponse{Locs: locs}, nil
}
func (rs *RemoteChunkStore) getUploadUrl(logger func(string), org, repoName, fileId string) (string, error) {
return fmt.Sprintf("http://%s/%s/%s/%s", rs.HttpHost, org, repoName, fileId), nil
func (rs *RemoteChunkStore) getUploadUrl(logger func(string), org, repoName string, tfd *remotesapi.TableFileDetails) (string, error) {
fileID := hash.New(tfd.Id).String()
expectedFiles[fileID] = *tfd
return fmt.Sprintf("http://%s/%s/%s/%s", rs.HttpHost, org, repoName, fileID), nil
}
func (rs *RemoteChunkStore) Rebase(ctx context.Context, req *remotesapi.RebaseRequest) (*remotesapi.RebaseResponse, error) {

38
go/utils/remotesrv/http.go
View File

@@ -15,6 +15,8 @@
package main
import (
"bytes"
"crypto/md5"
"errors"
"fmt"
"io"
@@ -25,10 +27,14 @@ import (
"strconv"
"strings"
remotesapi "github.com/liquidata-inc/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1"
"github.com/liquidata-inc/dolt/go/libraries/utils/iohelp"
"github.com/liquidata-inc/dolt/go/store/hash"
)
var expectedFiles = make(map[string]remotesapi.TableFileDetails)
func ServeHTTP(respWr http.ResponseWriter, req *http.Request) {
logger := getReqLogger("HTTP_"+req.Method, req.RequestURI)
defer func() { logger("finished") }()
@@ -57,7 +63,7 @@ func ServeHTTP(respWr http.ResponseWriter, req *http.Request) {
}
case http.MethodPost, http.MethodPut:
statusCode = writeChunk(logger, org, repo, hashStr, req)
statusCode = writeTableFile(logger, org, repo, hashStr, req)
}
if statusCode != -1 {
@@ -65,7 +71,7 @@ func ServeHTTP(respWr http.ResponseWriter, req *http.Request) {
}
}
func writeChunk(logger func(string), org, repo, fileId string, request *http.Request) int {
func writeTableFile(logger func(string), org, repo, fileId string, request *http.Request) int {
_, ok := hash.MaybeParse(fileId)
if !ok {
@@ -73,9 +79,37 @@ func writeChunk(logger func(string), org, repo, fileId string, request *http.Req
return http.StatusBadRequest
}
tfd, ok := expectedFiles[fileId]
if !ok {
return http.StatusBadRequest
}
logger(fileId + " is valid")
data, err := ioutil.ReadAll(request.Body)
if tfd.ContentLength != 0 && tfd.ContentLength != uint64(len(data)) {
return http.StatusBadRequest
}
if len(tfd.ContentHash) > 0 {
hasher := md5.New()
n, err := hasher.Write(data)
if err != nil {
return http.StatusInternalServerError
} else if n != len(data) {
return http.StatusInternalServerError
}
actualMD5Bytes := make([]byte, 0, 128)
hasher.Sum(actualMD5Bytes)
if !bytes.Equal(tfd.ContentHash, actualMD5Bytes) {
return http.StatusBadRequest
}
}
if err != nil {
logger("failed to read body " + err.Error())
return http.StatusInternalServerError

9
proto/dolt/services/remotesapi/v1alpha1/chunkstore.proto
View File

@@ -98,9 +98,16 @@ message GetDownloadLocsResponse {
repeated DownloadLoc locs = 1;
}
message TableFileDetails {
bytes id = 1;
uint64 content_length = 2;
bytes content_hash = 3;
}
message GetUploadLocsRequest {
RepoId repo_id = 1;
repeated bytes table_file_hashes = 2;
repeated bytes table_file_hashes = 2 [deprecated = true];
repeated TableFileDetails table_file_details = 3;
}
message GetUploadLocsResponse {