diff --git a/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go b/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go index c8bc765037..16a30fb022 100644 --- a/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go +++ b/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go @@ -585,19 +585,75 @@ func (m *GetDownloadLocsResponse) GetLocs() []*DownloadLoc { return nil } -type GetUploadLocsRequest struct { - RepoId *RepoId `protobuf:"bytes,1,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"` - TableFileHashes [][]byte `protobuf:"bytes,2,rep,name=table_file_hashes,json=tableFileHashes,proto3" json:"table_file_hashes,omitempty"` +type TableFileDetails struct { + Id []byte `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + ContentLength uint64 `protobuf:"varint,2,opt,name=content_length,json=contentLength,proto3" json:"content_length,omitempty"` + ContentHash []byte `protobuf:"bytes,3,opt,name=content_hash,json=contentHash,proto3" json:"content_hash,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } +func (m *TableFileDetails) Reset() { *m = TableFileDetails{} } +func (m *TableFileDetails) String() string { return proto.CompactTextString(m) } +func (*TableFileDetails) ProtoMessage() {} +func (*TableFileDetails) Descriptor() ([]byte, []int) { + return fileDescriptor_702c187af9ca94ec, []int{11} +} + +func (m *TableFileDetails) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TableFileDetails.Unmarshal(m, b) +} +func (m *TableFileDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TableFileDetails.Marshal(b, m, deterministic) +} +func (m *TableFileDetails) XXX_Merge(src proto.Message) { + xxx_messageInfo_TableFileDetails.Merge(m, src) +} +func (m *TableFileDetails) XXX_Size() int { + return xxx_messageInfo_TableFileDetails.Size(m) +} +func (m *TableFileDetails) XXX_DiscardUnknown() { + xxx_messageInfo_TableFileDetails.DiscardUnknown(m) +} + +var xxx_messageInfo_TableFileDetails proto.InternalMessageInfo + +func (m *TableFileDetails) GetId() []byte { + if m != nil { + return m.Id + } + return nil +} + +func (m *TableFileDetails) GetContentLength() uint64 { + if m != nil { + return m.ContentLength + } + return 0 +} + +func (m *TableFileDetails) GetContentHash() []byte { + if m != nil { + return m.ContentHash + } + return nil +} + +type GetUploadLocsRequest struct { + RepoId *RepoId `protobuf:"bytes,1,opt,name=repo_id,json=repoId,proto3" json:"repo_id,omitempty"` + TableFileHashes [][]byte `protobuf:"bytes,2,rep,name=table_file_hashes,json=tableFileHashes,proto3" json:"table_file_hashes,omitempty"` + TableFileDetails []*TableFileDetails `protobuf:"bytes,3,rep,name=table_file_details,json=tableFileDetails,proto3" json:"table_file_details,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + func (m *GetUploadLocsRequest) Reset() { *m = GetUploadLocsRequest{} } func (m *GetUploadLocsRequest) String() string { return proto.CompactTextString(m) } func (*GetUploadLocsRequest) ProtoMessage() {} func (*GetUploadLocsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{11} + return fileDescriptor_702c187af9ca94ec, []int{12} } func (m *GetUploadLocsRequest) XXX_Unmarshal(b []byte) error { @@ -632,6 +688,13 @@ func (m *GetUploadLocsRequest) GetTableFileHashes() [][]byte { return nil } +func (m *GetUploadLocsRequest) GetTableFileDetails() []*TableFileDetails { + if m != nil { + return m.TableFileDetails + } + return nil +} + type GetUploadLocsResponse struct { Locs []*UploadLoc `protobuf:"bytes,1,rep,name=locs,proto3" json:"locs,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -643,7 +706,7 @@ func (m *GetUploadLocsResponse) Reset() { *m = GetUploadLocsResponse{} } func (m *GetUploadLocsResponse) String() string { return proto.CompactTextString(m) } func (*GetUploadLocsResponse) ProtoMessage() {} func (*GetUploadLocsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{12} + return fileDescriptor_702c187af9ca94ec, []int{13} } func (m *GetUploadLocsResponse) XXX_Unmarshal(b []byte) error { @@ -682,7 +745,7 @@ func (m *RebaseRequest) Reset() { *m = RebaseRequest{} } func (m *RebaseRequest) String() string { return proto.CompactTextString(m) } func (*RebaseRequest) ProtoMessage() {} func (*RebaseRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{13} + return fileDescriptor_702c187af9ca94ec, []int{14} } func (m *RebaseRequest) XXX_Unmarshal(b []byte) error { @@ -720,7 +783,7 @@ func (m *RebaseResponse) Reset() { *m = RebaseResponse{} } func (m *RebaseResponse) String() string { return proto.CompactTextString(m) } func (*RebaseResponse) ProtoMessage() {} func (*RebaseResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{14} + return fileDescriptor_702c187af9ca94ec, []int{15} } func (m *RebaseResponse) XXX_Unmarshal(b []byte) error { @@ -752,7 +815,7 @@ func (m *RootRequest) Reset() { *m = RootRequest{} } func (m *RootRequest) String() string { return proto.CompactTextString(m) } func (*RootRequest) ProtoMessage() {} func (*RootRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{15} + return fileDescriptor_702c187af9ca94ec, []int{16} } func (m *RootRequest) XXX_Unmarshal(b []byte) error { @@ -791,7 +854,7 @@ func (m *RootResponse) Reset() { *m = RootResponse{} } func (m *RootResponse) String() string { return proto.CompactTextString(m) } func (*RootResponse) ProtoMessage() {} func (*RootResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{16} + return fileDescriptor_702c187af9ca94ec, []int{17} } func (m *RootResponse) XXX_Unmarshal(b []byte) error { @@ -831,7 +894,7 @@ func (m *ChunkTableInfo) Reset() { *m = ChunkTableInfo{} } func (m *ChunkTableInfo) String() string { return proto.CompactTextString(m) } func (*ChunkTableInfo) ProtoMessage() {} func (*ChunkTableInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{17} + return fileDescriptor_702c187af9ca94ec, []int{18} } func (m *ChunkTableInfo) XXX_Unmarshal(b []byte) error { @@ -881,7 +944,7 @@ func (m *CommitRequest) Reset() { *m = CommitRequest{} } func (m *CommitRequest) String() string { return proto.CompactTextString(m) } func (*CommitRequest) ProtoMessage() {} func (*CommitRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{18} + return fileDescriptor_702c187af9ca94ec, []int{19} } func (m *CommitRequest) XXX_Unmarshal(b []byte) error { @@ -948,7 +1011,7 @@ func (m *CommitResponse) Reset() { *m = CommitResponse{} } func (m *CommitResponse) String() string { return proto.CompactTextString(m) } func (*CommitResponse) ProtoMessage() {} func (*CommitResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{19} + return fileDescriptor_702c187af9ca94ec, []int{20} } func (m *CommitResponse) XXX_Unmarshal(b []byte) error { @@ -988,7 +1051,7 @@ func (m *GetRepoMetadataRequest) Reset() { *m = GetRepoMetadataRequest{} func (m *GetRepoMetadataRequest) String() string { return proto.CompactTextString(m) } func (*GetRepoMetadataRequest) ProtoMessage() {} func (*GetRepoMetadataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{20} + return fileDescriptor_702c187af9ca94ec, []int{21} } func (m *GetRepoMetadataRequest) XXX_Unmarshal(b []byte) error { @@ -1039,7 +1102,7 @@ func (m *GetRepoMetadataResponse) Reset() { *m = GetRepoMetadataResponse func (m *GetRepoMetadataResponse) String() string { return proto.CompactTextString(m) } func (*GetRepoMetadataResponse) ProtoMessage() {} func (*GetRepoMetadataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{21} + return fileDescriptor_702c187af9ca94ec, []int{22} } func (m *GetRepoMetadataResponse) XXX_Unmarshal(b []byte) error { @@ -1086,7 +1149,7 @@ func (m *ClientRepoFormat) Reset() { *m = ClientRepoFormat{} } func (m *ClientRepoFormat) String() string { return proto.CompactTextString(m) } func (*ClientRepoFormat) ProtoMessage() {} func (*ClientRepoFormat) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{22} + return fileDescriptor_702c187af9ca94ec, []int{23} } func (m *ClientRepoFormat) XXX_Unmarshal(b []byte) error { @@ -1132,7 +1195,7 @@ func (m *ListTableFilesRequest) Reset() { *m = ListTableFilesRequest{} } func (m *ListTableFilesRequest) String() string { return proto.CompactTextString(m) } func (*ListTableFilesRequest) ProtoMessage() {} func (*ListTableFilesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{23} + return fileDescriptor_702c187af9ca94ec, []int{24} } func (m *ListTableFilesRequest) XXX_Unmarshal(b []byte) error { @@ -1173,7 +1236,7 @@ func (m *TableFileInfo) Reset() { *m = TableFileInfo{} } func (m *TableFileInfo) String() string { return proto.CompactTextString(m) } func (*TableFileInfo) ProtoMessage() {} func (*TableFileInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{24} + return fileDescriptor_702c187af9ca94ec, []int{25} } func (m *TableFileInfo) XXX_Unmarshal(b []byte) error { @@ -1227,7 +1290,7 @@ func (m *ListTableFilesResponse) Reset() { *m = ListTableFilesResponse{} func (m *ListTableFilesResponse) String() string { return proto.CompactTextString(m) } func (*ListTableFilesResponse) ProtoMessage() {} func (*ListTableFilesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{25} + return fileDescriptor_702c187af9ca94ec, []int{26} } func (m *ListTableFilesResponse) XXX_Unmarshal(b []byte) error { @@ -1275,7 +1338,7 @@ func (m *AddTableFilesRequest) Reset() { *m = AddTableFilesRequest{} } func (m *AddTableFilesRequest) String() string { return proto.CompactTextString(m) } func (*AddTableFilesRequest) ProtoMessage() {} func (*AddTableFilesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{26} + return fileDescriptor_702c187af9ca94ec, []int{27} } func (m *AddTableFilesRequest) XXX_Unmarshal(b []byte) error { @@ -1328,7 +1391,7 @@ func (m *AddTableFilesResponse) Reset() { *m = AddTableFilesResponse{} } func (m *AddTableFilesResponse) String() string { return proto.CompactTextString(m) } func (*AddTableFilesResponse) ProtoMessage() {} func (*AddTableFilesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_702c187af9ca94ec, []int{27} + return fileDescriptor_702c187af9ca94ec, []int{28} } func (m *AddTableFilesResponse) XXX_Unmarshal(b []byte) error { @@ -1368,6 +1431,7 @@ func init() { proto.RegisterType((*UploadLoc)(nil), "dolt.services.remotesapi.v1alpha1.UploadLoc") proto.RegisterType((*GetDownloadLocsRequest)(nil), "dolt.services.remotesapi.v1alpha1.GetDownloadLocsRequest") proto.RegisterType((*GetDownloadLocsResponse)(nil), "dolt.services.remotesapi.v1alpha1.GetDownloadLocsResponse") + proto.RegisterType((*TableFileDetails)(nil), "dolt.services.remotesapi.v1alpha1.TableFileDetails") proto.RegisterType((*GetUploadLocsRequest)(nil), "dolt.services.remotesapi.v1alpha1.GetUploadLocsRequest") proto.RegisterType((*GetUploadLocsResponse)(nil), "dolt.services.remotesapi.v1alpha1.GetUploadLocsResponse") proto.RegisterType((*RebaseRequest)(nil), "dolt.services.remotesapi.v1alpha1.RebaseRequest") @@ -1392,78 +1456,83 @@ func init() { } var fileDescriptor_702c187af9ca94ec = []byte{ - // 1136 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0xdd, 0x6f, 0x1b, 0x45, - 0x10, 0xcf, 0xd9, 0xc1, 0x89, 0xc7, 0x1f, 0x71, 0x4e, 0x4d, 0x6a, 0x19, 0xa1, 0xa6, 0x2b, 0x81, - 0x42, 0x4b, 0x7d, 0x8d, 0x5b, 0xa9, 0x2d, 0xbc, 0x40, 0x42, 0x1b, 0x47, 0x0a, 0xa8, 0x6c, 0x0a, - 0x25, 0x44, 0xc8, 0x3a, 0x9f, 0xd7, 0xf6, 0x89, 0xf3, 0xae, 0x7b, 0xbb, 0x17, 0x1e, 0x91, 0x10, - 0x5f, 0x2f, 0x88, 0x17, 0x9e, 0xf8, 0x67, 0x80, 0xff, 0x0c, 0xed, 0xc7, 0x39, 0x77, 0xae, 0xab, - 0x6c, 0x2a, 0xd3, 0x37, 0xef, 0x64, 0x7f, 0x33, 0xbf, 0xdf, 0xec, 0xce, 0xec, 0xe4, 0xa0, 0x33, - 0x60, 0x91, 0xf0, 0x38, 0x89, 0xcf, 0xc3, 0x80, 0x70, 0x2f, 0x26, 0x13, 0x26, 0x08, 0xf7, 0xa7, - 0xa1, 0x77, 0xbe, 0xe7, 0x47, 0xd3, 0xb1, 0xbf, 0xe7, 0x05, 0xe3, 0x84, 0x7e, 0xc7, 0x05, 0x8b, - 0x49, 0x7b, 0x1a, 0x33, 0xc1, 0xdc, 0x9b, 0x12, 0xd3, 0x4e, 0x31, 0xed, 0x0b, 0x4c, 0x3b, 0xc5, - 0xa0, 0x07, 0x50, 0xc2, 0x64, 0xca, 0x8e, 0x06, 0x6e, 0x03, 0x8a, 0x2c, 0x1e, 0x35, 0x9d, 0x1d, - 0x67, 0xb7, 0x8c, 0xe5, 0x4f, 0xf7, 0x6d, 0x28, 0xc7, 0x64, 0xca, 0x7a, 0xd4, 0x9f, 0x90, 0x66, - 0x41, 0xd9, 0xd7, 0xa5, 0xe1, 0x73, 0x7f, 0x42, 0x10, 0x85, 0x46, 0xd7, 0xe7, 0x07, 0x2a, 0x24, - 0x26, 0x2f, 0x12, 0xc2, 0x85, 0xbb, 0x0f, 0x6b, 0x0a, 0x10, 0x0e, 0x94, 0x9b, 0x4a, 0xe7, 0xfd, - 0xf6, 0xa5, 0x0c, 0xda, 0x3a, 0x3c, 0x2e, 0xc5, 0x9a, 0xc6, 0x36, 0x94, 0xc6, 0x3e, 0x1f, 0x13, - 0xde, 0x2c, 0xec, 0x14, 0x77, 0xab, 0xd8, 0xac, 0xd0, 0x6d, 0xd8, 0xcc, 0xc4, 0xe3, 0x53, 0x46, - 0x39, 0x91, 0x9b, 0xfd, 0x3e, 0x27, 0x54, 0x34, 0x9d, 0x9d, 0xe2, 0xee, 0x5b, 0xd8, 0xac, 0xd0, - 0x43, 0xa8, 0x76, 0x85, 0x98, 0x1e, 0x12, 0xa1, 0x00, 0x52, 0x5b, 0x12, 0x47, 0xa9, 0xb6, 0x24, - 0x8e, 0x5e, 0x19, 0xe6, 0x29, 0x00, 0xf6, 0xe9, 0x88, 0x68, 0x9c, 0x0b, 0xab, 0xd2, 0xae, 0x80, - 0x55, 0xac, 0x7e, 0x4b, 0x24, 0x1b, 0x0e, 0x39, 0x11, 0x2a, 0x25, 0xab, 0xd8, 0xac, 0xa4, 0x3d, - 0x22, 0x74, 0x24, 0xc6, 0xcd, 0xe2, 0x8e, 0xb3, 0x5b, 0xc3, 0x66, 0x85, 0x46, 0x33, 0x2e, 0xca, - 0xf1, 0x02, 0x2e, 0x8f, 0xa1, 0x14, 0xcb, 0x3f, 0x69, 0x2e, 0x95, 0xce, 0x1d, 0x9b, 0xac, 0xcd, - 0x48, 0x62, 0x03, 0x46, 0xff, 0x38, 0x50, 0xf9, 0x94, 0x7d, 0x4f, 0x23, 0xe6, 0x0f, 0x8e, 0x59, - 0xe0, 0x1e, 0xc3, 0xfa, 0x58, 0x88, 0x69, 0x6f, 0x44, 0x84, 0x39, 0x0e, 0xcf, 0xc2, 0x71, 0x36, - 0x6f, 0xdd, 0x15, 0xbc, 0x36, 0xd6, 0x6b, 0xf7, 0x39, 0xd4, 0x53, 0x6f, 0x3d, 0x15, 0x50, 0xc9, - 0xbf, 0x92, 0x4f, 0xc5, 0xb9, 0xbb, 0x82, 0xab, 0xe3, 0xcc, 0x7a, 0x1f, 0x60, 0x3d, 0x62, 0x81, - 0x2f, 0x42, 0x46, 0xd1, 0xbb, 0xb0, 0x29, 0xf7, 0x3e, 0x65, 0x5c, 0x3c, 0xf3, 0xfb, 0x11, 0x79, - 0x12, 0x46, 0x0b, 0x12, 0x86, 0xfe, 0x74, 0xa0, 0xfc, 0xe5, 0x34, 0xd5, 0xf9, 0x1e, 0x6c, 0x08, - 0xb9, 0xb9, 0x37, 0x0c, 0x23, 0xd2, 0xcb, 0x9c, 0x57, 0x4d, 0xa4, 0x3e, 0xba, 0xf2, 0xe0, 0x4e, - 0xa0, 0xac, 0x14, 0x4c, 0x19, 0x17, 0x86, 0xfc, 0x7d, 0x4b, 0xf2, 0x39, 0x42, 0xdd, 0x15, 0xac, - 0x12, 0x2b, 0x8d, 0x39, 0xf6, 0x3f, 0xc0, 0xf6, 0x21, 0x11, 0x99, 0x23, 0x58, 0x6a, 0x61, 0xdc, - 0x84, 0xaa, 0x2a, 0xf0, 0x5e, 0xee, 0xde, 0x56, 0x94, 0xad, 0xab, 0x2f, 0xef, 0xb7, 0x70, 0xfd, - 0x25, 0x02, 0xa6, 0x52, 0xf6, 0x61, 0x35, 0x62, 0x01, 0x57, 0x75, 0x52, 0xe9, 0xb4, 0x2d, 0xc2, - 0x67, 0xdc, 0x60, 0x85, 0x45, 0xbf, 0x38, 0x70, 0xed, 0x90, 0x88, 0x59, 0xe6, 0x97, 0x2a, 0xef, - 0x16, 0x6c, 0xce, 0x9d, 0xe2, 0x4c, 0xe3, 0x46, 0xee, 0x1c, 0x09, 0x47, 0xa7, 0xb0, 0x35, 0xc7, - 0xc3, 0xa8, 0xfc, 0x38, 0xa7, 0xf2, 0x03, 0x0b, 0x16, 0x33, 0x27, 0x46, 0xe3, 0x09, 0xd4, 0x30, - 0xe9, 0xfb, 0x9c, 0x2c, 0x51, 0x1b, 0x6a, 0x40, 0x3d, 0x75, 0xaa, 0x89, 0xa2, 0x2f, 0xa0, 0x82, - 0x19, 0x13, 0xcb, 0x0c, 0x72, 0x1b, 0xaa, 0xda, 0xa5, 0xc9, 0x85, 0xec, 0xde, 0x8c, 0x89, 0x6c, - 0x41, 0xac, 0x4b, 0x83, 0xcc, 0x21, 0x7a, 0x0c, 0x75, 0x55, 0xe1, 0xea, 0x52, 0x1f, 0xd1, 0x21, - 0x5b, 0xd8, 0xea, 0x6e, 0x80, 0xbe, 0x5e, 0xbd, 0x80, 0x25, 0x54, 0xd7, 0x4c, 0x0d, 0x83, 0x32, - 0x1d, 0x48, 0x0b, 0xfa, 0xb7, 0x00, 0xb5, 0x03, 0x36, 0x99, 0x84, 0xcb, 0x54, 0xe2, 0x36, 0x61, - 0x2d, 0x48, 0xe2, 0x98, 0x98, 0x90, 0x55, 0x9c, 0x2e, 0x25, 0xc9, 0xc8, 0xe7, 0x42, 0x75, 0xd8, - 0x2a, 0x56, 0xbf, 0xdd, 0x33, 0x68, 0x68, 0x92, 0xfa, 0xfa, 0x84, 0x74, 0xc8, 0x9a, 0xab, 0xea, - 0xfc, 0xf7, 0x2c, 0x42, 0xe7, 0xb3, 0x80, 0xeb, 0x41, 0x3e, 0x2b, 0x3e, 0xb8, 0x41, 0x14, 0x12, - 0x2a, 0x7a, 0x4a, 0xd5, 0x90, 0xc5, 0x13, 0x5f, 0x34, 0xeb, 0x4a, 0xd9, 0x3d, 0x1b, 0xf7, 0x0a, - 0x2c, 0xf5, 0x3d, 0x51, 0x50, 0xdc, 0x08, 0xe6, 0x2c, 0xe8, 0x16, 0xd4, 0xd3, 0x14, 0x9a, 0x93, - 0x6b, 0xc2, 0x1a, 0x4f, 0x82, 0x80, 0x70, 0xae, 0x72, 0xb8, 0x8e, 0xd3, 0x25, 0xfa, 0xdb, 0x51, - 0x2d, 0x46, 0xa2, 0x3f, 0x23, 0xc2, 0x1f, 0xf8, 0xc2, 0x5f, 0x66, 0xe2, 0xdf, 0x80, 0xda, 0x33, - 0xd5, 0xa2, 0xf2, 0x02, 0x8c, 0xec, 0x1b, 0x50, 0xa1, 0xfd, 0x61, 0xef, 0x9c, 0xc4, 0x3c, 0x64, - 0xd4, 0xf4, 0x7b, 0xa0, 0xfd, 0xe1, 0x57, 0xda, 0xa2, 0x37, 0xf0, 0xd9, 0x86, 0x42, 0xba, 0x81, - 0x9b, 0x0d, 0xe8, 0x19, 0x34, 0xe6, 0x29, 0x2c, 0xc1, 0xeb, 0x19, 0x6c, 0x1d, 0x87, 0x99, 0xfe, - 0xbf, 0xcc, 0xb6, 0x87, 0x4e, 0xa1, 0x36, 0x73, 0xac, 0x6e, 0xdc, 0x75, 0x58, 0x53, 0x1d, 0xd0, - 0x38, 0x2d, 0xe3, 0x92, 0x5c, 0x1e, 0x0d, 0xdc, 0x77, 0x00, 0x68, 0x32, 0xe9, 0xe9, 0x21, 0xcf, - 0xd4, 0x62, 0x99, 0x26, 0x13, 0x3d, 0x12, 0xa5, 0xaf, 0x64, 0xf1, 0xe2, 0x95, 0xfc, 0xc3, 0x81, - 0xed, 0x79, 0xe2, 0x16, 0xbd, 0xc1, 0xfd, 0x3a, 0xf7, 0x9e, 0xaa, 0x7a, 0xd2, 0x73, 0xc9, 0x5d, - 0x0b, 0x79, 0x39, 0x31, 0x99, 0x17, 0x58, 0x2e, 0xd1, 0x5f, 0x05, 0xb8, 0xf6, 0xc9, 0x60, 0xf0, - 0xbf, 0x64, 0xf2, 0x15, 0x97, 0xb7, 0xb0, 0xc4, 0xcb, 0xbb, 0xb0, 0xd5, 0x14, 0x97, 0xd4, 0x6a, - 0xd0, 0x1e, 0x6c, 0xcd, 0xe5, 0xe6, 0xb2, 0x76, 0xd0, 0xf9, 0xa9, 0x0c, 0x9b, 0xca, 0xeb, 0x89, - 0x1c, 0xfa, 0x4f, 0x74, 0x70, 0xf7, 0x37, 0x07, 0x36, 0xe6, 0x6a, 0xcc, 0x7d, 0x64, 0xc1, 0x6f, - 0x71, 0x63, 0x69, 0x7d, 0xf8, 0x3a, 0x50, 0x43, 0xfd, 0x1c, 0xca, 0xb3, 0xa1, 0xdd, 0xb5, 0x39, - 0x84, 0xf9, 0x7f, 0x29, 0x5a, 0xf7, 0xaf, 0x06, 0x32, 0x71, 0x7f, 0xd7, 0x93, 0x4a, 0x66, 0x84, - 0x51, 0x13, 0x1a, 0xb7, 0xcd, 0xc3, 0x82, 0x19, 0xce, 0x36, 0x0f, 0x0b, 0xa7, 0xaf, 0x5f, 0x1d, - 0x70, 0xb3, 0x13, 0x8b, 0x61, 0xf3, 0xc0, 0xce, 0xe5, 0x4b, 0x03, 0x57, 0xeb, 0xe1, 0xd5, 0x81, - 0x86, 0xc9, 0x44, 0xfe, 0xbf, 0x27, 0x47, 0x11, 0xf7, 0xae, 0x55, 0x89, 0x65, 0x46, 0xa1, 0xd6, - 0xde, 0x15, 0x10, 0x26, 0xdc, 0x08, 0x56, 0xe5, 0x50, 0xe2, 0xda, 0x0c, 0x9c, 0x99, 0x81, 0xa8, - 0xe5, 0x59, 0xef, 0xbf, 0xd0, 0xa5, 0x5f, 0x51, 0x2b, 0x5d, 0xb9, 0x99, 0xc5, 0x4a, 0xd7, 0xdc, - 0x13, 0xfd, 0xb3, 0x03, 0xf5, 0x7c, 0x6f, 0x75, 0x6d, 0xce, 0x64, 0xe1, 0x3b, 0xd2, 0x7a, 0xf4, - 0x1a, 0x48, 0xc3, 0xe3, 0x47, 0x07, 0x6a, 0xb9, 0xae, 0x61, 0x75, 0xa7, 0x16, 0xf5, 0x60, 0xab, - 0x3b, 0xb5, 0xb0, 0x41, 0xed, 0x9f, 0x7e, 0xf3, 0x7c, 0x14, 0x8a, 0x71, 0xd2, 0x6f, 0x07, 0x6c, - 0xe2, 0x45, 0xe1, 0x8b, 0x24, 0x94, 0x5d, 0xe0, 0x4e, 0x48, 0x03, 0x4f, 0x7d, 0xb5, 0x18, 0x31, - 0x6f, 0x44, 0xa8, 0xa7, 0x3e, 0x4a, 0x78, 0x97, 0x7e, 0xc7, 0xf8, 0xe8, 0xc2, 0xd6, 0x2f, 0x29, - 0xcc, 0xbd, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0xac, 0x42, 0x71, 0x29, 0xfe, 0x10, 0x00, 0x00, + // 1210 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0xdf, 0x6f, 0x1b, 0xc5, + 0x13, 0xef, 0xd9, 0xf9, 0x3a, 0xf1, 0xf8, 0x47, 0x9d, 0x55, 0x93, 0x5a, 0xfe, 0x0a, 0x35, 0x3d, + 0xa9, 0x28, 0xb4, 0xd4, 0x6e, 0xd2, 0x4a, 0x6d, 0xe1, 0x05, 0x92, 0xb6, 0x71, 0xa4, 0x80, 0xca, + 0xa6, 0x50, 0x42, 0x84, 0xac, 0xf3, 0x79, 0x6d, 0x9f, 0x38, 0xdf, 0xba, 0xb7, 0xeb, 0xf0, 0x88, + 0x84, 0x40, 0xf0, 0x82, 0x78, 0xe1, 0x89, 0x7f, 0x06, 0xf8, 0x73, 0xf8, 0x2f, 0xd0, 0xce, 0xee, + 0x39, 0x77, 0x57, 0x57, 0xd9, 0x54, 0x86, 0xb7, 0xec, 0x64, 0x67, 0xe6, 0x33, 0x9f, 0xf9, 0xb1, + 0x73, 0x86, 0xdd, 0x01, 0x0f, 0x65, 0x47, 0xb0, 0xf8, 0x2c, 0xf0, 0x99, 0xe8, 0xc4, 0x6c, 0xc2, + 0x25, 0x13, 0xde, 0x34, 0xe8, 0x9c, 0xed, 0x78, 0xe1, 0x74, 0xec, 0xed, 0x74, 0xfc, 0xf1, 0x2c, + 0xfa, 0x46, 0x48, 0x1e, 0xb3, 0xf6, 0x34, 0xe6, 0x92, 0x93, 0x9b, 0x4a, 0xa7, 0x9d, 0xe8, 0xb4, + 0xcf, 0x75, 0xda, 0x89, 0x8e, 0xfb, 0x10, 0x4a, 0x94, 0x4d, 0xf9, 0xe1, 0x80, 0x34, 0xa0, 0xc8, + 0xe3, 0x51, 0xd3, 0xd9, 0x72, 0xb6, 0xcb, 0x54, 0xfd, 0x49, 0xfe, 0x0f, 0xe5, 0x98, 0x4d, 0x79, + 0x2f, 0xf2, 0x26, 0xac, 0x59, 0x40, 0xf9, 0x9a, 0x12, 0x7c, 0xea, 0x4d, 0x98, 0x1b, 0x41, 0xa3, + 0xeb, 0x89, 0x7d, 0x74, 0x49, 0xd9, 0xab, 0x19, 0x13, 0x92, 0xec, 0xc1, 0x2a, 0x2a, 0x04, 0x03, + 0x34, 0x53, 0xd9, 0x7d, 0xaf, 0x7d, 0x21, 0x82, 0xb6, 0x76, 0x4f, 0x4b, 0xb1, 0x86, 0xb1, 0x09, + 0xa5, 0xb1, 0x27, 0xc6, 0x4c, 0x34, 0x0b, 0x5b, 0xc5, 0xed, 0x2a, 0x35, 0x27, 0xf7, 0x0e, 0xac, + 0xa7, 0xfc, 0x89, 0x29, 0x8f, 0x04, 0x53, 0x97, 0xbd, 0xbe, 0x60, 0x91, 0x6c, 0x3a, 0x5b, 0xc5, + 0xed, 0xff, 0x51, 0x73, 0x72, 0x1f, 0x41, 0xb5, 0x2b, 0xe5, 0xf4, 0x80, 0x49, 0x54, 0x50, 0xb1, + 0xcd, 0xe2, 0x30, 0x89, 0x6d, 0x16, 0x87, 0x6f, 0x74, 0xf3, 0x1c, 0x80, 0x7a, 0xd1, 0x88, 0x69, + 0x3d, 0x02, 0x2b, 0x4a, 0x8e, 0x8a, 0x55, 0x8a, 0x7f, 0x2b, 0x4d, 0x3e, 0x1c, 0x0a, 0x26, 0x91, + 0x92, 0x15, 0x6a, 0x4e, 0x4a, 0x1e, 0xb2, 0x68, 0x24, 0xc7, 0xcd, 0xe2, 0x96, 0xb3, 0x5d, 0xa3, + 0xe6, 0xe4, 0x8e, 0xe6, 0x58, 0xd0, 0xf0, 0x02, 0x2c, 0x4f, 0xa1, 0x14, 0xab, 0x7f, 0x69, 0x2c, + 0x95, 0xdd, 0xbb, 0x36, 0xac, 0xcd, 0x41, 0x52, 0xa3, 0xec, 0xfe, 0xe9, 0x40, 0xe5, 0x09, 0xff, + 0x36, 0x0a, 0xb9, 0x37, 0x38, 0xe2, 0x3e, 0x39, 0x82, 0xb5, 0xb1, 0x94, 0xd3, 0xde, 0x88, 0x49, + 0x93, 0x8e, 0x8e, 0x85, 0xe1, 0x34, 0x6f, 0xdd, 0x2b, 0x74, 0x75, 0xac, 0xcf, 0xe4, 0x25, 0xd4, + 0x13, 0x6b, 0x3d, 0x74, 0x88, 0xe1, 0x5f, 0xca, 0x26, 0x62, 0xee, 0x5e, 0xa1, 0xd5, 0x71, 0xea, + 0xbc, 0x07, 0xb0, 0x16, 0x72, 0xdf, 0x93, 0x01, 0x8f, 0xdc, 0x5b, 0xb0, 0xae, 0xee, 0x3e, 0xe7, + 0x42, 0xbe, 0xf0, 0xfa, 0x21, 0x7b, 0x16, 0x84, 0x0b, 0x08, 0x73, 0x7f, 0x73, 0xa0, 0xfc, 0xf9, + 0x34, 0x89, 0xf3, 0x5d, 0xb8, 0x2a, 0xd5, 0xe5, 0xde, 0x30, 0x08, 0x59, 0x2f, 0x95, 0xaf, 0x9a, + 0x4c, 0x6c, 0x74, 0x55, 0xe2, 0x8e, 0xa1, 0x8c, 0x11, 0x4c, 0xb9, 0x90, 0x06, 0xfc, 0x03, 0x4b, + 0xf0, 0x19, 0x40, 0xdd, 0x2b, 0x14, 0x89, 0x55, 0xc2, 0x0c, 0xfa, 0xef, 0x60, 0xf3, 0x80, 0xc9, + 0x54, 0x0a, 0x96, 0xda, 0x18, 0x37, 0xa1, 0x8a, 0x0d, 0xde, 0xcb, 0xd4, 0x6d, 0x05, 0x65, 0x5d, + 0x5d, 0xbc, 0x5f, 0xc3, 0xf5, 0xd7, 0x00, 0x98, 0x4e, 0xd9, 0x83, 0x95, 0x90, 0xfb, 0x02, 0xfb, + 0xa4, 0xb2, 0xdb, 0xb6, 0x70, 0x9f, 0x32, 0x43, 0x51, 0xd7, 0x0d, 0xa1, 0x31, 0x27, 0xe1, 0x09, + 0x93, 0x5e, 0x10, 0x0a, 0x52, 0x87, 0x82, 0x09, 0xaa, 0x4a, 0x0b, 0xc1, 0x80, 0xdc, 0x82, 0xba, + 0xcf, 0x23, 0xc9, 0x22, 0xd9, 0x33, 0xdd, 0xa0, 0xbb, 0xa4, 0x66, 0xa4, 0x47, 0x28, 0xc4, 0x60, + 0xcc, 0x35, 0x4c, 0x58, 0x11, 0x0d, 0x54, 0x8c, 0x4c, 0x85, 0xe3, 0xfe, 0xed, 0xc0, 0xb5, 0x03, + 0x26, 0xe7, 0x79, 0x5e, 0x2a, 0x99, 0xb7, 0x61, 0x3d, 0x57, 0x33, 0x73, 0x46, 0xaf, 0x66, 0xaa, + 0x86, 0x09, 0xe2, 0x01, 0x49, 0xdd, 0x1d, 0xe8, 0xc0, 0x9b, 0x45, 0x24, 0xf2, 0xbe, 0x85, 0xeb, + 0x3c, 0x67, 0xb4, 0x21, 0x73, 0x12, 0xf7, 0x04, 0x36, 0x72, 0xa1, 0x9a, 0xb4, 0x7d, 0x94, 0x49, + 0xdb, 0xfb, 0x16, 0xde, 0xe6, 0x46, 0x4c, 0xd2, 0x8e, 0xa1, 0x46, 0x59, 0xdf, 0x13, 0x6c, 0x89, + 0xf4, 0xb9, 0x0d, 0xa8, 0x27, 0x46, 0x35, 0x50, 0xf7, 0x33, 0xa8, 0x50, 0xce, 0xe5, 0x32, 0x9d, + 0xdc, 0x81, 0xaa, 0x36, 0x69, 0xb8, 0x50, 0xcf, 0x11, 0xe7, 0x32, 0xdd, 0xe1, 0x6b, 0x4a, 0x80, + 0xd5, 0xf2, 0x14, 0xea, 0x38, 0xb2, 0x90, 0xec, 0xc3, 0x68, 0xc8, 0x17, 0xce, 0xee, 0x1b, 0xa0, + 0xfb, 0xa5, 0xe7, 0xf3, 0x59, 0xa4, 0x87, 0x40, 0x8d, 0x02, 0x8a, 0xf6, 0x95, 0xc4, 0xfd, 0xab, + 0x00, 0xb5, 0x7d, 0x3e, 0x99, 0x04, 0xcb, 0x8c, 0x84, 0x34, 0x61, 0xd5, 0x9f, 0xc5, 0x31, 0x33, + 0x2e, 0xab, 0x34, 0x39, 0x2a, 0x90, 0xa1, 0x27, 0xa4, 0xa9, 0x7f, 0xfc, 0x9b, 0x9c, 0x42, 0x43, + 0x83, 0xd4, 0x55, 0x17, 0x44, 0x43, 0xde, 0x5c, 0xc1, 0xfc, 0xef, 0x58, 0xb8, 0xce, 0xb2, 0x40, + 0xeb, 0x7e, 0x96, 0x15, 0x0f, 0x88, 0x1f, 0x06, 0xaa, 0xef, 0x30, 0xaa, 0x21, 0x8f, 0x27, 0x9e, + 0x6c, 0xd6, 0x31, 0x32, 0x9b, 0x62, 0xde, 0x47, 0x65, 0x15, 0xdf, 0x33, 0x54, 0xa5, 0x0d, 0x3f, + 0x27, 0x71, 0x6f, 0x43, 0x3d, 0xa1, 0xd0, 0x64, 0xae, 0x09, 0xab, 0x62, 0xe6, 0xfb, 0x4c, 0x08, + 0xe4, 0x70, 0x8d, 0x26, 0x47, 0xf7, 0x0f, 0x07, 0x67, 0xa6, 0xd2, 0xfe, 0x84, 0x49, 0x6f, 0xe0, + 0x49, 0x6f, 0x99, 0xc4, 0xff, 0x07, 0xd1, 0x9e, 0xe2, 0xcc, 0xcd, 0x06, 0x60, 0xc2, 0xbe, 0x01, + 0x95, 0xa8, 0x3f, 0xec, 0x9d, 0xb1, 0x58, 0x04, 0x3c, 0x32, 0x0f, 0x18, 0x44, 0xfd, 0xe1, 0x17, + 0x5a, 0xa2, 0x2f, 0x88, 0xf9, 0x85, 0x42, 0x72, 0x41, 0x98, 0x0b, 0xee, 0x0b, 0x68, 0xe4, 0x21, + 0x2c, 0xc1, 0xea, 0x29, 0x6c, 0x1c, 0x05, 0xa9, 0x07, 0x6d, 0x99, 0x93, 0xd5, 0x3d, 0x81, 0xda, + 0xdc, 0x30, 0x56, 0xdc, 0x75, 0x58, 0xc5, 0xc1, 0x69, 0x8c, 0x96, 0x69, 0x49, 0x1d, 0x0f, 0x07, + 0xe4, 0x1d, 0x80, 0x68, 0x36, 0xe9, 0xe9, 0xad, 0xd5, 0xf4, 0x62, 0x39, 0x9a, 0x4d, 0xf4, 0x8e, + 0x97, 0x3c, 0xfb, 0xc5, 0xf3, 0x67, 0xff, 0x57, 0x07, 0x36, 0xf3, 0xc0, 0x2d, 0x66, 0x03, 0xf9, + 0x32, 0xb3, 0x20, 0x60, 0x3f, 0xe9, 0x45, 0xeb, 0xde, 0x65, 0xa6, 0x37, 0xb6, 0xd3, 0xf9, 0x4a, + 0xa1, 0x8e, 0xee, 0xef, 0x05, 0xb8, 0xf6, 0xf1, 0x60, 0xf0, 0xaf, 0x30, 0xf9, 0x86, 0xe2, 0x2d, + 0x2c, 0xb1, 0x78, 0x17, 0x8e, 0x9a, 0xe2, 0x92, 0x46, 0x8d, 0xbb, 0x03, 0x1b, 0x39, 0x6e, 0x2e, + 0x1a, 0x07, 0xbb, 0x3f, 0x94, 0x61, 0x1d, 0xad, 0x1e, 0xab, 0xaf, 0x98, 0x63, 0xed, 0x9c, 0xfc, + 0xec, 0xc0, 0xd5, 0x5c, 0x8f, 0x91, 0xc7, 0x16, 0xf8, 0x16, 0x0f, 0x96, 0xd6, 0x07, 0x6f, 0xa3, + 0x6a, 0xa0, 0x9f, 0x41, 0x79, 0xfe, 0x15, 0x42, 0x6c, 0x92, 0x90, 0xff, 0x46, 0x6a, 0x3d, 0xb8, + 0x9c, 0x92, 0xf1, 0xfb, 0x8b, 0x5e, 0x86, 0x52, 0x3b, 0x19, 0xae, 0x9c, 0xc2, 0x96, 0x87, 0x05, + 0x4b, 0xa9, 0x2d, 0x0f, 0x0b, 0xd7, 0xc9, 0x9f, 0x1c, 0x20, 0xe9, 0x8d, 0xc5, 0xa0, 0x79, 0x68, + 0x67, 0xf2, 0xb5, 0x9d, 0xae, 0xf5, 0xe8, 0xf2, 0x8a, 0x06, 0xc9, 0x44, 0x7d, 0xc0, 0xaa, 0x55, + 0x84, 0xdc, 0xb3, 0x6a, 0xb1, 0xd4, 0x2a, 0xd4, 0xda, 0xb9, 0x84, 0x86, 0x71, 0x37, 0x82, 0x15, + 0xb5, 0x94, 0x10, 0x9b, 0x0d, 0x3a, 0xb5, 0x10, 0xb5, 0x3a, 0xd6, 0xf7, 0xcf, 0xe3, 0xd2, 0xaf, + 0xa8, 0x55, 0x5c, 0x99, 0x9d, 0xc5, 0x2a, 0xae, 0xdc, 0x13, 0xfd, 0xa3, 0x03, 0xf5, 0xec, 0x6c, + 0x25, 0x36, 0x39, 0x59, 0xf8, 0x8e, 0xb4, 0x1e, 0xbf, 0x85, 0xa6, 0xc1, 0xf1, 0xbd, 0x03, 0xb5, + 0xcc, 0xd4, 0xb0, 0xaa, 0xa9, 0x45, 0x33, 0xd8, 0xaa, 0xa6, 0x16, 0x0e, 0xa8, 0xbd, 0x93, 0xaf, + 0x5e, 0x8e, 0x02, 0x39, 0x9e, 0xf5, 0xdb, 0x3e, 0x9f, 0x74, 0xc2, 0xe0, 0xd5, 0x2c, 0x50, 0x53, + 0xe0, 0x6e, 0x10, 0xf9, 0x1d, 0xfc, 0x19, 0x66, 0xc4, 0x3b, 0x23, 0x16, 0x75, 0xf0, 0x57, 0x96, + 0xce, 0x85, 0x3f, 0xcc, 0x7c, 0x78, 0x2e, 0xeb, 0x97, 0x50, 0xe7, 0xfe, 0x3f, 0x01, 0x00, 0x00, + 0xff, 0xff, 0x9b, 0x26, 0xa3, 0x10, 0xcf, 0x11, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/go.sum b/go/go.sum index c49906ae00..38d95bf5dc 100644 --- a/go/go.sum +++ b/go/go.sum @@ -196,6 +196,8 @@ github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/liquidata-inc/go-mysql-server v0.4.1-0.20191003230430-3a1a09c79ada h1:2D7VUQT4JHEB1N3xUG/MqULhDfWWg65LmPDBjXWBZcs= github.com/liquidata-inc/go-mysql-server v0.4.1-0.20191003230430-3a1a09c79ada/go.mod h1:DdWE0ku/mNfuLsRJIrHeHpDtB7am+6oopxEsQKmVkx8= +github.com/liquidata-inc/go-mysql-server v0.4.1-0.20191017183442-4b9329eafa5b h1:fdht6iaUhHYJEfSUY1m1KAghyi6zfyfv+12gaSsaQ+o= +github.com/liquidata-inc/go-mysql-server v0.4.1-0.20191017183442-4b9329eafa5b/go.mod h1:DdWE0ku/mNfuLsRJIrHeHpDtB7am+6oopxEsQKmVkx8= github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0 h1:phMgajKClMUiIr+hF2LGt8KRuUa2Vd2GI1sNgHgSXoU= github.com/liquidata-inc/ishell v0.0.0-20190514193646-693241f1f2a0/go.mod h1:YC1rI9k5gx8D02ljlbxDfZe80s/iq8bGvaaQsvR+qxs= github.com/liquidata-inc/mmap-go v1.0.3 h1:2LndAeAtup9rpvUmu4wZSYCsjCQ0Zpc+NqE+6+PnT7g= diff --git a/go/libraries/doltcore/remotestorage/chunk_store.go b/go/libraries/doltcore/remotestorage/chunk_store.go index 6878ff9166..c4452df27c 100644 --- a/go/libraries/doltcore/remotestorage/chunk_store.go +++ b/go/libraries/doltcore/remotestorage/chunk_store.go @@ -893,9 +893,20 @@ func (dcs *DoltChunkStore) getDownloadWorkForLoc(ctx context.Context, getRange * } // WriteTableFile reads a table file from the provided reader and writes it to the chunk store. -func (dcs *DoltChunkStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader) error { +func (dcs *DoltChunkStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader, contentLength uint64, contentHash []byte) error { fileIdBytes := hash.Parse(fileId) - req := &remotesapi.GetUploadLocsRequest{RepoId: dcs.getRepoId(), TableFileHashes: [][]byte{fileIdBytes[:]}} + tfd := &remotesapi.TableFileDetails{ + Id: fileIdBytes[:], + ContentLength: contentLength, + ContentHash: contentHash, + } + req := &remotesapi.GetUploadLocsRequest{ + RepoId: dcs.getRepoId(), + TableFileDetails: []*remotesapi.TableFileDetails{tfd}, + + // redundant and deprecated. Still setting for compatibility, but will remove promptly. + TableFileHashes: [][]byte{fileIdBytes[:]}, + } resp, err := dcs.csClient.GetUploadLocations(ctx, req) if err != nil { diff --git a/go/store/datas/pull.go b/go/store/datas/pull.go index 1550013df4..ff96b14c02 100644 --- a/go/store/datas/pull.go +++ b/go/store/datas/pull.go @@ -132,7 +132,7 @@ func clone(ctx context.Context, srcTS, sinkTS nbs.TableFileStore, eventCh chan<- } }() - err = sinkTS.WriteTableFile(ctx, tblFile.FileID(), tblFile.NumChunks(), rd) + err = sinkTS.WriteTableFile(ctx, tblFile.FileID(), tblFile.NumChunks(), rd, 0, nil) if err != nil { if eventCh != nil { diff --git a/go/store/datas/pull_test.go b/go/store/datas/pull_test.go index 7dd102bed3..c9209e9048 100644 --- a/go/store/datas/pull_test.go +++ b/go/store/datas/pull_test.go @@ -421,7 +421,7 @@ func (ttfs *TestTableFileStore) Sources(ctx context.Context) (hash.Hash, []nbs.T return ttfs.root, tblFiles, nil } -func (ttfs *TestTableFileStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader) error { +func (ttfs *TestTableFileStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader, contentLength uint64, contentHash []byte) error { tblFile := &TestTableFileWriter{fileId, numChunks, bytes.NewBuffer(nil), ttfs} _, err := io.Copy(tblFile, rd) diff --git a/go/store/datas/puller.go b/go/store/datas/puller.go index b43f5e1e14..40ebf3fb38 100644 --- a/go/store/datas/puller.go +++ b/go/store/datas/puller.go @@ -182,9 +182,11 @@ func NewPuller(ctx context.Context, tempDir string, chunksPerTF int, srcDB, sink func (p *Puller) processCompletedTables(ctx context.Context, ae *atomicerr.AtomicError, completedTables <-chan FilledWriters) { type tempTblFile struct { - id string - path string - numChunks int + id string + path string + numChunks int + contentLen uint64 + contentHash []byte } var tblFiles []tempTblFile @@ -209,7 +211,13 @@ func (p *Puller) processCompletedTables(ctx context.Context, ae *atomicerr.Atomi continue } - tblFiles = append(tblFiles, tempTblFile{id, path, tblFile.wr.Size()}) + tblFiles = append(tblFiles, tempTblFile{ + id: id, + path: path, + numChunks: tblFile.wr.Size(), + contentLen: tblFile.wr.ContentLength(), + contentHash: tblFile.wr.GetMD5(), + }) } if ae.IsSet() { @@ -239,7 +247,7 @@ func (p *Puller) processCompletedTables(ctx context.Context, ae *atomicerr.Atomi p.eventCh <- NewTFPullerEvent(StartUploadTableFile, details) fWithSize := FileReaderWithSize{f, fi.Size()} - err = p.sinkDB.chunkStore().(nbs.TableFileStore).WriteTableFile(ctx, tmpTblFile.id, tmpTblFile.numChunks, fWithSize) + err = p.sinkDB.chunkStore().(nbs.TableFileStore).WriteTableFile(ctx, tmpTblFile.id, tmpTblFile.numChunks, fWithSize, tmpTblFile.contentLen, tmpTblFile.contentHash) go func() { _ = os.Remove(tmpTblFile.path) diff --git a/go/store/nbs/byte_sink.go b/go/store/nbs/byte_sink.go index 052fe10476..71f716ce50 100644 --- a/go/store/nbs/byte_sink.go +++ b/go/store/nbs/byte_sink.go @@ -15,7 +15,9 @@ package nbs import ( + "crypto/md5" "errors" + "hash" "io" "os" "path/filepath" @@ -287,3 +289,58 @@ func (sink *BufferedFileByteSink) FlushToFile(path string) (err error) { return os.Rename(sink.path, path) } + +// HashingByteSink is a ByteSink that keeps an md5 hash of all the data written to it. +type HashingByteSink struct { + backingSink ByteSink + hasher hash.Hash + size uint64 +} + +func NewHashingByteSink(backingSink ByteSink) *HashingByteSink { + return &HashingByteSink{backingSink: backingSink, hasher: md5.New(), size: 0} +} + +// Write writes a byte array to the sink. +func (sink *HashingByteSink) Write(src []byte) (int, error) { + nWritten, err := sink.backingSink.Write(src) + + if err != nil { + return 0, err + } + + nHashed, err := sink.hasher.Write(src[:nWritten]) + + if err != nil { + return 0, err + } else if nWritten != nHashed { + return 0, errors.New("failed to hash all the data that was written to the byte sink.") + } + + sink.size += uint64(nWritten) + + return nWritten, nil +} + +// Flush writes all the data that was written to the ByteSink to the supplied writer +func (sink *HashingByteSink) Flush(wr io.Writer) error { + return sink.backingSink.Flush(wr) +} + +// FlushToFile writes all the data that was written to the ByteSink to a file at the given path +func (sink *HashingByteSink) FlushToFile(path string) error { + return sink.backingSink.FlushToFile(path) +} + +// GetMD5 gets the MD5 hash of all the bytes written to the sink +func (sink *HashingByteSink) GetMD5() []byte { + result := make([]byte, 0, 128) + sink.hasher.Sum(result) + + return result +} + +// Size gets the number of bytes written to the sink +func (sink *HashingByteSink) Size() uint64 { + return sink.size +} diff --git a/go/store/nbs/cmp_chunk_table_writer.go b/go/store/nbs/cmp_chunk_table_writer.go index 312c02dee3..0abe44dbcd 100644 --- a/go/store/nbs/cmp_chunk_table_writer.go +++ b/go/store/nbs/cmp_chunk_table_writer.go @@ -36,7 +36,7 @@ var ErrAlreadyFinished = errors.New("already Finished") // CmpChunkTableWriter writes CompressedChunks to a table file type CmpChunkTableWriter struct { - sink ByteSink + sink *HashingByteSink totalCompressedData uint64 totalUncompressedData uint64 prefixes prefixIndexSlice // TODO: This is in danger of exploding memory @@ -51,7 +51,7 @@ func NewCmpChunkTableWriter() (*CmpChunkTableWriter, error) { return nil, err } - return &CmpChunkTableWriter{s, 0, 0, nil, nil}, nil + return &CmpChunkTableWriter{NewHashingByteSink(s), 0, 0, nil, nil}, nil } // Size returns the number of compressed chunks that have been added @@ -59,6 +59,16 @@ func (tw *CmpChunkTableWriter) Size() int { return len(tw.prefixes) } +// Gets the size of the entire table file in bytes +func (tw *CmpChunkTableWriter) ContentLength() uint64 { + return tw.sink.Size() +} + +// Gets the MD5 of the entire table file +func (tw *CmpChunkTableWriter) GetMD5() []byte { + return tw.sink.GetMD5() +} + // AddCmpChunk adds a compressed chunk func (tw *CmpChunkTableWriter) AddCmpChunk(c CompressedChunk) error { if len(c.CompressedData) == 0 { diff --git a/go/store/nbs/store.go b/go/store/nbs/store.go index a677fb5d62..9a7d005210 100644 --- a/go/store/nbs/store.go +++ b/go/store/nbs/store.go @@ -933,7 +933,7 @@ func (nbs *NomsBlockStore) Sources(ctx context.Context) (hash.Hash, []TableFile, } // WriteTableFile will read a table file from the provided reader and write it to the TableFileStore -func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader) error { +func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader, contentLength uint64, contentHash []byte) error { fsPersister, ok := nbs.p.(*fsTablePersister) if !ok { diff --git a/go/store/nbs/store_test.go b/go/store/nbs/store_test.go index 2fe67d5d25..2e8809acd7 100644 --- a/go/store/nbs/store_test.go +++ b/go/store/nbs/store_test.go @@ -51,7 +51,7 @@ func TestNBSAsTableFileStore(t *testing.T) { data, addr, err := buildTable(chunkData) fileID := addr.String() fileToData[fileID] = data - err = st.WriteTableFile(ctx, fileID, i+1, bytes.NewReader(data)) + err = st.WriteTableFile(ctx, fileID, i+1, bytes.NewReader(data), 0, nil) require.NoError(t, err) } diff --git a/go/store/nbs/table.go b/go/store/nbs/table.go index a56ec2ad61..42613cfe9a 100644 --- a/go/store/nbs/table.go +++ b/go/store/nbs/table.go @@ -286,7 +286,7 @@ type TableFileStore interface { Sources(ctx context.Context) (hash.Hash, []TableFile, error) // WriteTableFile will read a table file from the provided reader and write it to the TableFileStore - WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader) error + WriteTableFile(ctx context.Context, fileId string, numChunks int, rd io.Reader, contentLength uint64, contentHash []byte) error // SetRootChunk changes the root chunk hash from the previous value to the new root. SetRootChunk(ctx context.Context, root, previous hash.Hash) error diff --git a/go/utils/remotesrv/grpc.go b/go/utils/remotesrv/grpc.go index a8c233f4e0..a28faa553f 100644 --- a/go/utils/remotesrv/grpc.go +++ b/go/utils/remotesrv/grpc.go @@ -127,6 +127,25 @@ func (rs *RemoteChunkStore) getDownloadUrl(logger func(string), org, repoName, f return fmt.Sprintf("http://%s/%s/%s/%s", rs.HttpHost, org, repoName, fileId), nil } +func parseTableFileDetails(req *remotesapi.GetUploadLocsRequest) []*remotesapi.TableFileDetails { + tfd := req.GetTableFileDetails() + + if len(tfd) == 0 { + _, hashToIdx := remotestorage.ParseByteSlices(req.TableFileHashes) + + tfd = make([]*remotesapi.TableFileDetails, len(hashToIdx)) + for h, i := range hashToIdx { + tfd[i] = &remotesapi.TableFileDetails{ + Id: h[:], + ContentLength: 0, + ContentHash: nil, + } + } + } + + return tfd +} + func (rs *RemoteChunkStore) GetUploadLocations(ctx context.Context, req *remotesapi.GetUploadLocsRequest) (*remotesapi.GetUploadLocsResponse, error) { logger := getReqLogger("GRPC", "GetUploadLocations") defer func() { logger("finished") }() @@ -141,19 +160,19 @@ func (rs *RemoteChunkStore) GetUploadLocations(ctx context.Context, req *remotes org := req.RepoId.Org repoName := req.RepoId.RepoName - hashes, _ := remotestorage.ParseByteSlices(req.TableFileHashes) + tfds := parseTableFileDetails(req) var locs []*remotesapi.UploadLoc - for h := range hashes { - tmp := h - url, err := rs.getUploadUrl(logger, org, repoName, h.String()) + for _, tfd := range tfds { + h := hash.New(tfd.Id) + url, err := rs.getUploadUrl(logger, org, repoName, tfd) if err != nil { return nil, status.Error(codes.Internal, "Failed to get upload Url.") } loc := &remotesapi.UploadLoc_HttpPost{HttpPost: &remotesapi.HttpPostTableFile{Url: url}} - locs = append(locs, &remotesapi.UploadLoc{TableFileHash: tmp[:], Location: loc}) + locs = append(locs, &remotesapi.UploadLoc{TableFileHash: h[:], Location: loc}) logger(fmt.Sprintf("sending upload location for chunk %s: %s", h.String(), url)) } @@ -161,8 +180,10 @@ func (rs *RemoteChunkStore) GetUploadLocations(ctx context.Context, req *remotes return &remotesapi.GetUploadLocsResponse{Locs: locs}, nil } -func (rs *RemoteChunkStore) getUploadUrl(logger func(string), org, repoName, fileId string) (string, error) { - return fmt.Sprintf("http://%s/%s/%s/%s", rs.HttpHost, org, repoName, fileId), nil +func (rs *RemoteChunkStore) getUploadUrl(logger func(string), org, repoName string, tfd *remotesapi.TableFileDetails) (string, error) { + fileID := hash.New(tfd.Id).String() + expectedFiles[fileID] = *tfd + return fmt.Sprintf("http://%s/%s/%s/%s", rs.HttpHost, org, repoName, fileID), nil } func (rs *RemoteChunkStore) Rebase(ctx context.Context, req *remotesapi.RebaseRequest) (*remotesapi.RebaseResponse, error) { diff --git a/go/utils/remotesrv/http.go b/go/utils/remotesrv/http.go index 119e75b750..4d023a6968 100644 --- a/go/utils/remotesrv/http.go +++ b/go/utils/remotesrv/http.go @@ -15,6 +15,8 @@ package main import ( + "bytes" + "crypto/md5" "errors" "fmt" "io" @@ -25,10 +27,14 @@ import ( "strconv" "strings" + remotesapi "github.com/liquidata-inc/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1" + "github.com/liquidata-inc/dolt/go/libraries/utils/iohelp" "github.com/liquidata-inc/dolt/go/store/hash" ) +var expectedFiles = make(map[string]remotesapi.TableFileDetails) + func ServeHTTP(respWr http.ResponseWriter, req *http.Request) { logger := getReqLogger("HTTP_"+req.Method, req.RequestURI) defer func() { logger("finished") }() @@ -57,7 +63,7 @@ func ServeHTTP(respWr http.ResponseWriter, req *http.Request) { } case http.MethodPost, http.MethodPut: - statusCode = writeChunk(logger, org, repo, hashStr, req) + statusCode = writeTableFile(logger, org, repo, hashStr, req) } if statusCode != -1 { @@ -65,7 +71,7 @@ func ServeHTTP(respWr http.ResponseWriter, req *http.Request) { } } -func writeChunk(logger func(string), org, repo, fileId string, request *http.Request) int { +func writeTableFile(logger func(string), org, repo, fileId string, request *http.Request) int { _, ok := hash.MaybeParse(fileId) if !ok { @@ -73,9 +79,37 @@ func writeChunk(logger func(string), org, repo, fileId string, request *http.Req return http.StatusBadRequest } + tfd, ok := expectedFiles[fileId] + + if !ok { + return http.StatusBadRequest + } + logger(fileId + " is valid") data, err := ioutil.ReadAll(request.Body) + if tfd.ContentLength != 0 && tfd.ContentLength != uint64(len(data)) { + return http.StatusBadRequest + } + + if len(tfd.ContentHash) > 0 { + hasher := md5.New() + n, err := hasher.Write(data) + + if err != nil { + return http.StatusInternalServerError + } else if n != len(data) { + return http.StatusInternalServerError + } + + actualMD5Bytes := make([]byte, 0, 128) + hasher.Sum(actualMD5Bytes) + + if !bytes.Equal(tfd.ContentHash, actualMD5Bytes) { + return http.StatusBadRequest + } + } + if err != nil { logger("failed to read body " + err.Error()) return http.StatusInternalServerError diff --git a/proto/dolt/services/remotesapi/v1alpha1/chunkstore.proto b/proto/dolt/services/remotesapi/v1alpha1/chunkstore.proto index 4a4879fc8b..490e43c124 100644 --- a/proto/dolt/services/remotesapi/v1alpha1/chunkstore.proto +++ b/proto/dolt/services/remotesapi/v1alpha1/chunkstore.proto @@ -98,9 +98,16 @@ message GetDownloadLocsResponse { repeated DownloadLoc locs = 1; } +message TableFileDetails { + bytes id = 1; + uint64 content_length = 2; + bytes content_hash = 3; +} + message GetUploadLocsRequest { RepoId repo_id = 1; - repeated bytes table_file_hashes = 2; + repeated bytes table_file_hashes = 2 [deprecated = true]; + repeated TableFileDetails table_file_details = 3; } message GetUploadLocsResponse {