Merge remote-tracking branch 'origin/main' into aaron/autogc

This commit is contained in:
Aaron Son
2025-02-14 13:46:58 -08:00
55 changed files with 1397 additions and 812 deletions

View File

@@ -402,9 +402,11 @@ type RangeChunk struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"`
Offset uint64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"`
Length uint32 `protobuf:"varint,3,opt,name=length,proto3" json:"length,omitempty"`
Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"`
Offset uint64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"`
Length uint32 `protobuf:"varint,3,opt,name=length,proto3" json:"length,omitempty"`
DictionaryOffset uint64 `protobuf:"varint,4,opt,name=dictionary_offset,json=dictionaryOffset,proto3" json:"dictionary_offset,omitempty"`
DictionaryLength uint32 `protobuf:"varint,5,opt,name=dictionary_length,json=dictionaryLength,proto3" json:"dictionary_length,omitempty"`
}
func (x *RangeChunk) Reset() {
@@ -460,6 +462,20 @@ func (x *RangeChunk) GetLength() uint32 {
return 0
}
func (x *RangeChunk) GetDictionaryOffset() uint64 {
if x != nil {
return x.DictionaryOffset
}
return 0
}
func (x *RangeChunk) GetDictionaryLength() uint32 {
if x != nil {
return x.DictionaryLength
}
return 0
}
type HttpGetRange struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -2233,418 +2249,424 @@ var file_dolt_services_remotesapi_v1alpha1_chunkstore_proto_rawDesc = []byte{
0x74, 0x70, 0x47, 0x65, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72,
0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x16, 0x0a, 0x06,
0x68, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x68, 0x61,
0x73, 0x68, 0x65, 0x73, 0x22, 0x50, 0x0a, 0x0a, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x43, 0x68, 0x75,
0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c,
0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74,
0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x16,
0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06,
0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x67, 0x0a, 0x0c, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65,
0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x45, 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x67,
0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x61, 0x6e,
0x67, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22,
0xe9, 0x02, 0x0a, 0x0b, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x12,
0x4c, 0x0a, 0x08, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61,
0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x43, 0x68, 0x75,
0x6e, 0x6b, 0x48, 0x00, 0x52, 0x07, 0x68, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x12, 0x57, 0x0a,
0x0e, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x67, 0x65, 0x74, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18,
0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72,
0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69,
0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65,
0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x48, 0x00, 0x52, 0x0c, 0x68, 0x74, 0x74, 0x70, 0x47, 0x65,
0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73,
0x68, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e,
0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e,
0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65,
0x73, 0x68, 0x41, 0x66, 0x74, 0x65, 0x72, 0x12, 0x66, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65,
0x73, 0x68, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x3d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x73, 0x68, 0x65, 0x73, 0x22, 0xaa, 0x01, 0x0a, 0x0a, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x43, 0x68,
0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28,
0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65,
0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12,
0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52,
0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x2b, 0x0a, 0x11, 0x64, 0x69, 0x63, 0x74, 0x69,
0x6f, 0x6e, 0x61, 0x72, 0x79, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01,
0x28, 0x04, 0x52, 0x10, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x4f, 0x66,
0x66, 0x73, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61,
0x72, 0x79, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52,
0x10, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x4c, 0x65, 0x6e, 0x67, 0x74,
0x68, 0x22, 0x67, 0x0a, 0x0c, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x52, 0x61, 0x6e, 0x67,
0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
0x75, 0x72, 0x6c, 0x12, 0x45, 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x02, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76,
0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x43, 0x68, 0x75,
0x6e, 0x6b, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0xe9, 0x02, 0x0a, 0x0b, 0x44,
0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x12, 0x4c, 0x0a, 0x08, 0x68, 0x74,
0x74, 0x70, 0x5f, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x64,
0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d,
0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31,
0x2e, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x48, 0x00, 0x52,
0x07, 0x68, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x12, 0x57, 0x0a, 0x0e, 0x68, 0x74, 0x74, 0x70,
0x5f, 0x67, 0x65, 0x74, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c,
0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c,
0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52,
0x0e, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x42,
0x0a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x25, 0x0a, 0x11, 0x48,
0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65,
0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75,
0x72, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x09, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63,
0x12, 0x26, 0x0a, 0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x68,
0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c, 0x65,
0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x12, 0x53, 0x0a, 0x09, 0x68, 0x74, 0x74, 0x70,
0x5f, 0x70, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x64, 0x6f,
0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x52, 0x61, 0x6e, 0x67,
0x65, 0x48, 0x00, 0x52, 0x0c, 0x68, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x52, 0x61, 0x6e, 0x67,
0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x61, 0x66, 0x74,
0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73,
0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x41, 0x66, 0x74,
0x65, 0x72, 0x12, 0x66, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x72, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x64, 0x6f,
0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e,
0x48, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c,
0x65, 0x48, 0x00, 0x52, 0x08, 0x68, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x42, 0x0a, 0x0a,
0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbb, 0x01, 0x0a, 0x16, 0x47, 0x65,
0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18,
0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72,
0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69,
0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64,
0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x68, 0x75, 0x6e,
0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0b,
0x63, 0x68, 0x75, 0x6e, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72,
0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65,
0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72,
0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x7c, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x44, 0x6f,
0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b,
0x32, 0x2e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c,
0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63,
0x52, 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74,
0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f,
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8b, 0x01, 0x0a, 0x10, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46,
0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64,
0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f,
0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01,
0x28, 0x04, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4c, 0x65, 0x6e, 0x67, 0x74,
0x68, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x68, 0x61, 0x73,
0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74,
0x48, 0x61, 0x73, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x68, 0x75, 0x6e,
0x6b, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x43, 0x68, 0x75,
0x6e, 0x6b, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61,
0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07,
0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e,
0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61,
0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64,
0x12, 0x2e, 0x0a, 0x11, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x68,
0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52,
0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73,
0x12, 0x61, 0x0a, 0x12, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64,
0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64,
0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d,
0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31,
0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c,
0x73, 0x52, 0x10, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61,
0x69, 0x6c, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65,
0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b,
0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18,
0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22,
0x78, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x04, 0x6c, 0x6f, 0x63, 0x73,
0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65,
0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65,
0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x72,
0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x42, 0x0a, 0x0a, 0x08, 0x6c, 0x6f,
0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x25, 0x0a, 0x11, 0x48, 0x74, 0x74, 0x70, 0x50, 0x6f,
0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75,
0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x94, 0x01,
0x0a, 0x09, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x12, 0x26, 0x0a, 0x0f, 0x74,
0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48,
0x61, 0x73, 0x68, 0x12, 0x53, 0x0a, 0x09, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x73, 0x74,
0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70,
0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61,
0x64, 0x4c, 0x6f, 0x63, 0x52, 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65,
0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x50,
0x6f, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x00, 0x52, 0x08,
0x68, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x42, 0x0a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbb, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e,
0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c,
0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70,
0x6f, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x68, 0x61, 0x73,
0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x68, 0x75, 0x6e, 0x6b,
0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74,
0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f,
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61,
0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61,
0x74, 0x68, 0x22, 0x7c, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61,
0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a,
0x04, 0x6c, 0x6f, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x64, 0x6f,
0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e,
0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x52, 0x04, 0x6c, 0x6f, 0x63,
0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
0x22, 0x8b, 0x01, 0x0a, 0x10, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65,
0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x0c, 0x52, 0x02, 0x69, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74,
0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x63,
0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x21, 0x0a, 0x0c,
0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01,
0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x48, 0x61, 0x73, 0x68, 0x12,
0x1d, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20,
0x01, 0x28, 0x04, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x22, 0xa9,
0x02, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70,
0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x2e, 0x0a, 0x11, 0x74,
0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73,
0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0f, 0x74, 0x61, 0x62, 0x6c,
0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x61, 0x0a, 0x12, 0x74,
0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c,
0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61,
0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x54, 0x61, 0x62, 0x6c,
0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x10, 0x74, 0x61,
0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x1d,
0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01,
0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a,
0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09,
0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x78, 0x0a, 0x15, 0x47, 0x65,
0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
0x0b, 0x32, 0x2c, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61,
0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x52,
0x04, 0x6c, 0x6f, 0x63, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f,
0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54,
0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8f, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x62, 0x61, 0x73, 0x65, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69,
0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61,
0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f,
0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65,
0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8f, 0x01, 0x0a, 0x0d, 0x52, 0x65,
0x62, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72,
0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64,
0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d,
0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31,
0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12,
0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20,
0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b,
0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28,
0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x2f, 0x0a, 0x0e, 0x52,
0x65, 0x62, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a,
0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8d, 0x01, 0x0a,
0x0b, 0x52, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07,
0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70,
0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65,
0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x2f, 0x0a, 0x0e, 0x52, 0x65, 0x62, 0x61, 0x73, 0x65,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f,
0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65,
0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8d, 0x01, 0x0a, 0x0b, 0x52, 0x6f, 0x6f, 0x74,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70,
0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72,
0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65,
0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72,
0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x4a, 0x0a, 0x0c, 0x52, 0x6f, 0x6f, 0x74, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x6f, 0x6f, 0x74, 0x5f,
0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x6f, 0x6f, 0x74,
0x48, 0x61, 0x73, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b,
0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f,
0x6b, 0x65, 0x6e, 0x22, 0x45, 0x0a, 0x0e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c,
0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20,
0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x68, 0x75,
0x6e, 0x6b, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a,
0x63, 0x68, 0x75, 0x6e, 0x6b, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xde, 0x02, 0x0a, 0x0d, 0x43,
0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07,
0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e,
0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61,
0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64,
0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02,
0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12,
0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01,
0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x4a, 0x0a, 0x0c,
0x52, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09,
0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
0x08, 0x72, 0x6f, 0x6f, 0x74, 0x48, 0x61, 0x73, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70,
0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72,
0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x45, 0x0a, 0x0e, 0x43, 0x68, 0x75, 0x6e,
0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61,
0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x1f,
0x0a, 0x0b, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20,
0x01, 0x28, 0x0d, 0x52, 0x0a, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22,
0xde, 0x02, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
0x12, 0x18, 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28,
0x0c, 0x52, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61,
0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6c, 0x61, 0x73, 0x74, 0x12, 0x5b,
0x0a, 0x10, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x6e,
0x66, 0x6f, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x68, 0x75,
0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0e, 0x63, 0x68, 0x75,
0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x61, 0x0a, 0x12, 0x63,
0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61,
0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61,
0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65,
0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x10, 0x63, 0x6c,
0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1b,
0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28,
0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x2a, 0x0a, 0x0e, 0x43,
0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a,
0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07,
0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0xfb, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x52,
0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76,
0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06,
0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x61, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x0e, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31,
0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72,
0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74,
0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12,
0x12, 0x0a, 0x04, 0x6c, 0x61, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6c,
0x61, 0x73, 0x74, 0x12, 0x5b, 0x0a, 0x10, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x74, 0x61, 0x62,
0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e,
0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61,
0x31, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f,
0x52, 0x0e, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f,
0x12, 0x61, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f,
0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64,
0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d,
0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31,
0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61,
0x74, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72,
0x6d, 0x61, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68,
0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68,
0x22, 0x2a, 0x0a, 0x0e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20,
0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0xfb, 0x01, 0x0a,
0x16, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70,
0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52,
0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70,
0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72,
0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f,
0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70,
0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x92, 0x02, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70,
0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x66, 0x56, 0x65, 0x72, 0x73, 0x69,
0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x73, 0x56, 0x65, 0x72, 0x73,
0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73,
0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61,
0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74,
0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f,
0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x73, 0x0a, 0x18, 0x70, 0x75, 0x73, 0x68, 0x5f, 0x63, 0x6f,
0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f,
0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61,
0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x75, 0x73, 0x68,
0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72,
0x6f, 0x6c, 0x52, 0x16, 0x70, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65,
0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x22, 0x54, 0x0a, 0x10, 0x43, 0x6c,
0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1f,
0x0a, 0x0b, 0x6e, 0x62, 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x66, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12,
0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02,
0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
0x22, 0xc0, 0x01, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69,
0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65,
0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f,
0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e,
0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x27,
0x0a, 0x0d, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18,
0x02, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0c, 0x61, 0x70, 0x70, 0x65, 0x6e,
0x64, 0x69, 0x78, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f,
0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70,
0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70,
0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50,
0x61, 0x74, 0x68, 0x22, 0x82, 0x02, 0x0a, 0x0d, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c,
0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d,
0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01,
0x28, 0x0d, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x10, 0x0a,
0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12,
0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72,
0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61,
0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x41, 0x66, 0x74, 0x65, 0x72,
0x12, 0x66, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x72, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x64, 0x6f, 0x6c, 0x74,
0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65,
0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65,
0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72,
0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73,
0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb5, 0x01, 0x0a, 0x1a, 0x52, 0x65, 0x66,
0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70,
0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x66,
0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69,
0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b,
0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f,
0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68,
0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68,
0x22, 0x8f, 0x01, 0x0a, 0x1b, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c,
0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75,
0x72, 0x6c, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x61, 0x66,
0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x41, 0x66,
0x74, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65,
0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b,
0x65, 0x6e, 0x22, 0x99, 0x02, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65,
0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a,
0x09, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c,
0x52, 0x08, 0x72, 0x6f, 0x6f, 0x74, 0x48, 0x61, 0x73, 0x68, 0x12, 0x58, 0x0a, 0x0f, 0x74, 0x61,
0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76,
0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c,
0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65,
0x49, 0x6e, 0x66, 0x6f, 0x12, 0x69, 0x0a, 0x18, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78,
0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f,
0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70,
0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65,
0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x15, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64,
0x69, 0x78, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12,
0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20,
0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xba,
0x03, 0x0a, 0x14, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70,
0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x61, 0x0a, 0x12, 0x63,
0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61,
0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73,
0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61,
0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65,
0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x10, 0x63, 0x6c,
0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1d,
0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x5b,
0x0a, 0x10, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x6e,
0x66, 0x6f, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x68, 0x75,
0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0e, 0x63, 0x68, 0x75,
0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x62, 0x0a, 0x0f, 0x61,
0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04,
0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76,
0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e,
0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73,
0x74, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52,
0x0e, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12,
0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20,
0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b,
0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28,
0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x50, 0x0a, 0x15, 0x41,
0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18,
0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1d,
0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01,
0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a,
0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x92, 0x02, 0x0a, 0x17, 0x47,
0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x66, 0x5f, 0x76, 0x65,
0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x66,
0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x73, 0x5f, 0x76,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62,
0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72,
0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b,
0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72,
0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x73, 0x0a, 0x18, 0x70, 0x75,
0x73, 0x68, 0x5f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x63,
0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x64,
0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d,
0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31,
0x2e, 0x50, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79,
0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x52, 0x16, 0x70, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e,
0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x22,
0x54, 0x0a, 0x10, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72,
0x6d, 0x61, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69,
0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x66, 0x56, 0x65, 0x72,
0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73,
0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x73, 0x56, 0x65,
0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xc0, 0x01, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61,
0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c,
0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70,
0x6f, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0d, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f,
0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0c,
0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x1d, 0x0a, 0x0a,
0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72,
0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08,
0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x82, 0x02, 0x0a, 0x0d, 0x54, 0x61, 0x62,
0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x69,
0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c,
0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b,
0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x43, 0x68, 0x75, 0x6e,
0x6b, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52,
0x03, 0x75, 0x72, 0x6c, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f,
0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f,
0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68,
0x41, 0x66, 0x74, 0x65, 0x72, 0x12, 0x66, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68,
0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d,
0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72,
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68,
0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46,
0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0e, 0x72,
0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb5, 0x01,
0x0a, 0x1a, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69,
0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07,
0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e,
0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61,
0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64,
0x12, 0x17, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70,
0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72,
0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f,
0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70,
0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x8f, 0x01, 0x0a, 0x1b, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73,
0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65,
0x73, 0x68, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72,
0x65, 0x73, 0x68, 0x41, 0x66, 0x74, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f,
0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65,
0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x99, 0x02, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74,
0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18,
0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x6f, 0x6f, 0x74, 0x48, 0x61, 0x73, 0x68, 0x12,
0x58, 0x0a, 0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x6e,
0x66, 0x6f, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x54, 0x61, 0x62,
0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c,
0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x69, 0x0a, 0x18, 0x61, 0x70, 0x70,
0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65,
0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x64, 0x6f,
0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e,
0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x15, 0x61,
0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65,
0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b,
0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f,
0x6b, 0x65, 0x6e, 0x22, 0xba, 0x03, 0x0a, 0x14, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65,
0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07,
0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e,
0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61,
0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64,
0x12, 0x61, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f,
0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64,
0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d,
0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31,
0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61,
0x74, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72,
0x6d, 0x61, 0x74, 0x12, 0x5b, 0x0a, 0x10, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x74, 0x61, 0x62,
0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e,
0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61,
0x31, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f,
0x52, 0x0e, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f,
0x12, 0x62, 0x0a, 0x0f, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f, 0x6f, 0x70, 0x74,
0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74,
0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65,
0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4d, 0x61,
0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70,
0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70,
0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b,
0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f,
0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68,
0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68,
0x22, 0x50, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63,
0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63,
0x65, 0x73, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65,
0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b,
0x65, 0x6e, 0x2a, 0xa4, 0x01, 0x0a, 0x16, 0x50, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x63, 0x75,
0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x28, 0x0a,
0x24, 0x50, 0x55, 0x53, 0x48, 0x5f, 0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43,
0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43,
0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x2f, 0x0a, 0x2b, 0x50, 0x55, 0x53, 0x48, 0x5f,
0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54,
0x52, 0x4f, 0x4c, 0x5f, 0x49, 0x47, 0x4e, 0x4f, 0x52, 0x45, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x49,
0x4e, 0x47, 0x5f, 0x53, 0x45, 0x54, 0x10, 0x01, 0x12, 0x2f, 0x0a, 0x2b, 0x50, 0x55, 0x53, 0x48,
0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x2a, 0xa4, 0x01,
0x0a, 0x16, 0x50, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63,
0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x28, 0x0a, 0x24, 0x50, 0x55, 0x53, 0x48,
0x5f, 0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x43, 0x4f, 0x4e,
0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, 0x5f, 0x57, 0x4f, 0x52, 0x4b,
0x49, 0x4e, 0x47, 0x5f, 0x53, 0x45, 0x54, 0x10, 0x02, 0x2a, 0x89, 0x01, 0x0a, 0x16, 0x4d, 0x61,
0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70,
0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x24, 0x4d, 0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54,
0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x58, 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e,
0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x20,
0x0a, 0x1c, 0x4d, 0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e,
0x44, 0x49, 0x58, 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x45, 0x54, 0x10, 0x01,
0x12, 0x23, 0x0a, 0x1f, 0x4d, 0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50,
0x45, 0x4e, 0x44, 0x49, 0x58, 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x50, 0x50,
0x45, 0x4e, 0x44, 0x10, 0x02, 0x32, 0xb2, 0x0b, 0x0a, 0x11, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x53,
0x74, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x88, 0x01, 0x0a, 0x0f,
0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12,
0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e,
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70,
0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64,
0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x64, 0x6f, 0x6c,
0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44,
0x10, 0x00, 0x12, 0x2f, 0x0a, 0x2b, 0x50, 0x55, 0x53, 0x48, 0x5f, 0x43, 0x4f, 0x4e, 0x43, 0x55,
0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x49,
0x47, 0x4e, 0x4f, 0x52, 0x45, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x45,
0x54, 0x10, 0x01, 0x12, 0x2f, 0x0a, 0x2b, 0x50, 0x55, 0x53, 0x48, 0x5f, 0x43, 0x4f, 0x4e, 0x43,
0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f,
0x41, 0x53, 0x53, 0x45, 0x52, 0x54, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x49, 0x4e, 0x47, 0x5f, 0x53,
0x45, 0x54, 0x10, 0x02, 0x2a, 0x89, 0x01, 0x0a, 0x16, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73,
0x74, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12,
0x28, 0x0a, 0x24, 0x4d, 0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50, 0x45,
0x4e, 0x44, 0x49, 0x58, 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50,
0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x4d, 0x41, 0x4e,
0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x58, 0x5f, 0x4f,
0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x45, 0x54, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x4d,
0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x58,
0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x02,
0x32, 0xb2, 0x0b, 0x0a, 0x11, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x88, 0x01, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x52, 0x65,
0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c,
0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74,
0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47,
0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x76, 0x0a, 0x09, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75,
0x6e, 0x6b, 0x73, 0x12, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76,
0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b,
0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e,
0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73,
0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x61, 0x73,
0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8d,
0x01, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f,
0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61,
0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44,
0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31,
0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f,
0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x94,
0x01, 0x0a, 0x17, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61,
0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c,
0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74,
0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47,
0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72,
0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69,
0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70,
0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x12, 0x76, 0x0a, 0x09, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x33,
0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72,
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68,
0x61, 0x31, 0x2e, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76,
0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8d, 0x01, 0x0a, 0x14, 0x47, 0x65,
0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f,
0x6e, 0x73, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31,
0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f,
0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e,
0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61,
0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x94, 0x01, 0x0a, 0x17, 0x53, 0x74,
0x72, 0x65, 0x61, 0x6d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72,
0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69,
0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77,
0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x87, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c,
0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x37, 0x2e, 0x64,
0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d,
0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31,
0x2e, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72,
0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69,
0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c,
0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
0x6d, 0x0a, 0x06, 0x52, 0x65, 0x62, 0x61, 0x73, 0x65, 0x12, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74,
0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65,
0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65,
0x62, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x64, 0x6f,
0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e,
0x52, 0x65, 0x62, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x67,
0x0a, 0x04, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70,
0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x6f, 0x6f, 0x74, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70,
0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x6f, 0x6f, 0x74, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6d, 0x0a, 0x06, 0x43, 0x6f, 0x6d, 0x6d, 0x69,
0x74, 0x12, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61,
0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76,
0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x85, 0x01, 0x0a, 0x0e, 0x4c, 0x69, 0x73, 0x74, 0x54,
0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x38, 0x2e, 0x64, 0x6f, 0x6c, 0x74,
0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65,
0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4c, 0x69,
0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76,
0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c,
0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x94,
0x01, 0x0a, 0x13, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46,
0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x3d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70,
0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65,
0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72,
0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69,
0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73,
0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x82, 0x01, 0x0a, 0x0d, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62,
0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x37, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73,
0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x3a, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c,
0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64,
0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01,
0x12, 0x87, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f,
0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x37, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61,
0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x54,
0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55,
0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x38, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c,
0x70, 0x68, 0x61, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c,
0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x53, 0x5a, 0x51, 0x67, 0x69,
0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x6f, 0x6c, 0x74, 0x68, 0x75, 0x62,
0x2f, 0x64, 0x6f, 0x6c, 0x74, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x2f, 0x64, 0x6f, 0x6c, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x61, 0x6c,
0x70, 0x68, 0x61, 0x31, 0x3b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f,
0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6d, 0x0a, 0x06, 0x52, 0x65,
0x62, 0x61, 0x73, 0x65, 0x12, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76,
0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e,
0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x62, 0x61, 0x73, 0x65, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70,
0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x62, 0x61, 0x73,
0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x67, 0x0a, 0x04, 0x52, 0x6f, 0x6f,
0x74, 0x12, 0x2e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61,
0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61,
0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x12, 0x6d, 0x0a, 0x06, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x30, 0x2e, 0x64,
0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d,
0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31,
0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31,
0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72,
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68,
0x61, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x12, 0x85, 0x01, 0x0a, 0x0e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46,
0x69, 0x6c, 0x65, 0x73, 0x12, 0x38, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76,
0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e,
0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62,
0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x39,
0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72,
0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68,
0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65,
0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x94, 0x01, 0x0a, 0x13, 0x52, 0x65,
0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72,
0x6c, 0x12, 0x3d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61,
0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62,
0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x3e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73,
0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c,
0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c,
0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x12, 0x82, 0x01, 0x0a, 0x0d, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c,
0x65, 0x73, 0x12, 0x37, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63,
0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31,
0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46,
0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x64, 0x6f,
0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e,
0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x53, 0x5a, 0x51, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x6f, 0x6c, 0x74, 0x68, 0x75, 0x62, 0x2f, 0x64, 0x6f, 0x6c, 0x74,
0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x64, 0x6f,
0x6c, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3b,
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (

View File

@@ -56,7 +56,7 @@ require (
github.com/cespare/xxhash/v2 v2.2.0
github.com/creasty/defaults v1.6.0
github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
github.com/dolthub/go-mysql-server v0.19.1-0.20250207201905-b3a4c87c4fdc
github.com/dolthub/go-mysql-server v0.19.1-0.20250214204118-1e0e5e8f244a
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
github.com/dolthub/swiss v0.1.0
github.com/esote/minmaxheap v1.0.0

View File

@@ -179,8 +179,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90 h1:Sni8jrP0sy/w9ZYXoff4g/ixe+7bFCZlfCqXKJSU+zM=
github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA=
github.com/dolthub/go-mysql-server v0.19.1-0.20250207201905-b3a4c87c4fdc h1:SdN7GRPtaqmLwfi6cVcyF4Oc8FbFUJ+mwsFRV++6iH4=
github.com/dolthub/go-mysql-server v0.19.1-0.20250207201905-b3a4c87c4fdc/go.mod h1:QQxZvPHOtycbC2bVmqmT6/Fov2g1/T1Rtm76wLd/Y1E=
github.com/dolthub/go-mysql-server v0.19.1-0.20250214204118-1e0e5e8f244a h1:3aeB6IsvQKOTxSRDeDuBjvOkHBolqYrhLUXa0UmgWjc=
github.com/dolthub/go-mysql-server v0.19.1-0.20250214204118-1e0e5e8f244a/go.mod h1:QQxZvPHOtycbC2bVmqmT6/Fov2g1/T1Rtm76wLd/Y1E=
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=

View File

@@ -295,7 +295,12 @@ func (rs *RemoteChunkStore) StreamDownloadLocations(stream remotesapi.ChunkStore
var ranges []*remotesapi.RangeChunk
for h, r := range hashToRange {
hCpy := h
ranges = append(ranges, &remotesapi.RangeChunk{Hash: hCpy[:], Offset: r.Offset, Length: r.Length})
ranges = append(ranges, &remotesapi.RangeChunk{
Hash: hCpy[:],
Offset: r.Offset,
Length: r.Length,
DictionaryOffset: r.DictOffset,
DictionaryLength: r.DictLength})
}
url := rs.getDownloadUrl(md, prefix+"/"+loc)
@@ -606,7 +611,7 @@ func getTableFileInfo(
}
appendixTableFileInfo := make([]*remotesapi.TableFileInfo, 0)
for _, t := range tableList {
url := rs.getDownloadUrl(md, prefix+"/"+t.LocationPrefix()+t.FileID())
url := rs.getDownloadUrl(md, prefix+"/"+t.LocationPrefix()+t.FileID()+t.LocationSuffix())
url, err = rs.sealer.Seal(url)
if err != nil {
return nil, status.Error(codes.Internal, "failed to get seal download url for "+t.FileID())

View File

@@ -35,6 +35,7 @@ import (
"github.com/dolthub/dolt/go/libraries/utils/filesys"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/nbs"
"github.com/dolthub/dolt/go/store/types"
)
@@ -94,12 +95,18 @@ func (fh filehandler) ServeHTTP(respWr http.ResponseWriter, req *http.Request) {
respWr.WriteHeader(http.StatusBadRequest)
return
}
_, ok := hash.MaybeParse(path[i+1:])
fileName := path[i+1:]
if strings.HasSuffix(fileName, nbs.ArchiveFileSuffix) {
fileName = fileName[:len(fileName)-len(nbs.ArchiveFileSuffix)]
}
_, ok := hash.MaybeParse(fileName)
if !ok {
logger.WithField("last_path_component", path[i+1:]).Warn("bad request with unparseable last path component")
logger.WithField("last_path_component", fileName).Warn("bad request with unparseable last path component")
respWr.WriteHeader(http.StatusBadRequest)
return
}
abs, err := fh.fs.Abs(path)
if err != nil {
logger.WithError(err).Error("could not get absolute path")

View File

@@ -17,7 +17,6 @@ package remotesrv
import (
"context"
"crypto/tls"
"errors"
"net"
"net/http"
"strings"
@@ -29,7 +28,6 @@ import (
"google.golang.org/grpc"
remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1"
"github.com/dolthub/dolt/go/libraries/doltcore/env"
"github.com/dolthub/dolt/go/libraries/utils/filesys"
)
@@ -80,14 +78,6 @@ func NewServer(args ServerArgs) (*Server, error) {
args.Logger = logrus.NewEntry(logrus.StandardLogger())
}
storageMetadata, err := env.GetMultiEnvStorageMetadata(args.FS)
if err != nil {
return nil, err
}
if storageMetadata.ArchiveFilesPresent() {
return nil, errors.New("archive files present. Please run `dolt archive --revert` before running the server.")
}
s := new(Server)
s.stopChan = make(chan struct{})

View File

@@ -21,21 +21,20 @@ import (
// ChunkCache is an interface used for caching chunks
type ChunkCache interface {
// Put puts a slice of chunks into the cache.
Put(c []nbs.CompressedChunk) bool
// Put puts a slice of chunks into the cache. Error returned if the cache capacity has been exceeded.
Put(c []nbs.ToChunker) error
// Get gets a map of hash to chunk for a set of hashes. In the event that a chunk is not in the cache, chunks.Empty.
// is put in it's place
Get(h hash.HashSet) map[hash.Hash]nbs.CompressedChunk
Get(h hash.HashSet) map[hash.Hash]nbs.ToChunker
// Has takes a set of hashes and returns the set of hashes that the cache currently does not have in it.
Has(h hash.HashSet) (absent hash.HashSet)
// PutChunk puts a single chunk in the cache. true returns in the event that the chunk was cached successfully
// and false is returned if that chunk is already is the cache.
PutChunk(chunk nbs.CompressedChunk) bool
// PutChunk puts a single chunk in the cache. Returns an error if the cache capacity has been exceeded.
PutChunk(chunk nbs.ToChunker) error
// GetAndClearChunksToFlush gets a map of hash to chunk which includes all the chunks that were put in the cache
// between the last time GetAndClearChunksToFlush was called and now.
GetAndClearChunksToFlush() map[hash.Hash]nbs.CompressedChunk
GetAndClearChunksToFlush() map[hash.Hash]nbs.ToChunker
}

View File

@@ -18,9 +18,11 @@ import (
"context"
"errors"
"io"
"sync"
"sync/atomic"
"time"
"github.com/dolthub/gozstd"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc"
@@ -48,8 +50,14 @@ type ChunkFetcher struct {
eg *errgroup.Group
egCtx context.Context
// toGetCh is the channel used to request chunks. This will be initially given a root,
// and as refs are found, they will be added to the channel for workers to batch and request.
toGetCh chan hash.HashSet
resCh chan nbs.CompressedChunk
// resCh is the results channel for the fetcher. It is used both to return
// chunks themselves, and to indicate which chunks were requested but missing
// by having a Hash, but are empty.
resCh chan nbs.ToChunker
abortCh chan struct{}
stats StatsRecorder
@@ -69,7 +77,7 @@ func NewChunkFetcher(ctx context.Context, dcs *DoltChunkStore) *ChunkFetcher {
egCtx: ctx,
toGetCh: make(chan hash.HashSet),
resCh: make(chan nbs.CompressedChunk),
resCh: make(chan nbs.ToChunker),
abortCh: make(chan struct{}),
stats: StatsFactory(),
@@ -123,7 +131,7 @@ func (f *ChunkFetcher) CloseSend() error {
// by |Get|. Returns |io.EOF| after |CloseSend| is called and all requested
// chunks have been successfully received. Returns an error if this
// |ChunkFetcher| is terminally failed or if the supplied |ctx| is |Done|.
func (f *ChunkFetcher) Recv(ctx context.Context) (nbs.CompressedChunk, error) {
func (f *ChunkFetcher) Recv(ctx context.Context) (nbs.ToChunker, error) {
select {
case <-ctx.Done():
return nbs.CompressedChunk{}, context.Cause(ctx)
@@ -219,7 +227,7 @@ func fetcherHashSetToGetDlLocsReqsThread(ctx context.Context, reqCh chan hash.Ha
// delivered in |reqCh|, and they will be delivered in order.
//
// This function handles backoff and retries for the underlying streaming RPC.
func fetcherRPCDownloadLocsThread(ctx context.Context, reqCh chan *remotesapi.GetDownloadLocsRequest, resCh chan []*remotesapi.DownloadLoc, client remotesapi.ChunkStoreServiceClient, storeRepoToken func(string), missingChunkCh chan nbs.CompressedChunk, host string) error {
func fetcherRPCDownloadLocsThread(ctx context.Context, reqCh chan *remotesapi.GetDownloadLocsRequest, resCh chan []*remotesapi.DownloadLoc, client remotesapi.ChunkStoreServiceClient, storeRepoToken func(string), missingChunkCh chan nbs.ToChunker, host string) error {
stream, err := reliable.MakeCall[*remotesapi.GetDownloadLocsRequest, *remotesapi.GetDownloadLocsResponse](
ctx,
reliable.CallOptions[*remotesapi.GetDownloadLocsRequest, *remotesapi.GetDownloadLocsResponse]{
@@ -327,33 +335,53 @@ func getMissingChunks(req *remotesapi.GetDownloadLocsRequest, resp *remotesapi.G
return requested, nil
}
// Delivered to a fetching thread by the download-coalescing thread, requests for the
// fetching thread to download all the entries in the |GetRange| and deliver them to
// the appropriate places. For |rangeType| Chunk, it delivers them to the fetched
// chunk channel. For |rangeType| Dictionary, it delivers them by calling |set| on the
// dictionaryCache with the fetched dictionary.
type fetchResp struct {
get *GetRange
refresh func(ctx context.Context, err error, client remotesapi.ChunkStoreServiceClient) (string, error)
get *GetRange
refresh func(ctx context.Context, err error, client remotesapi.ChunkStoreServiceClient) (string, error)
rangeType rangeType
dictCache *dictionaryCache
path string
}
type fetchReq struct {
respCh chan fetchResp
cancelCh chan struct{}
respCh chan fetchResp
}
// A simple structure to keep track of *GetRange requests along with
// |locationRefreshes| for the URL paths we have seen.
type downloads struct {
ranges *ranges.Tree
// This Tree exclusively holds Chunk fetch ranges.
chunkRanges *ranges.Tree
// This Tree exclusively holds Dictionary fetch ranges. Every
// entry that we create in |dictionaryCache| which needs to be
// populated goes in here. These ranges must be fetched before
// (or concurrently with) any chunkRanges, since the chunk
// range fetches will block the fetching thread on the
// population of the dictionary cache entry.
dictRanges *ranges.Tree
// Holds all pending and fetched dictionaries for any chunk
// ranges that have gone into |chunkRanges|.
dictCache *dictionaryCache
refreshes map[string]*locationRefresh
}
func newDownloads() downloads {
return downloads{
ranges: ranges.NewTree(chunkAggDistance),
refreshes: make(map[string]*locationRefresh),
chunkRanges: ranges.NewTree(chunkAggDistance),
dictRanges: ranges.NewTree(chunkAggDistance),
dictCache: &dictionaryCache{},
refreshes: make(map[string]*locationRefresh),
}
}
func (d downloads) Add(resp *remotesapi.DownloadLoc) {
gr := (*GetRange)(resp.Location.(*remotesapi.DownloadLoc_HttpGetRange).HttpGetRange)
path := gr.ResourcePath()
hgr := resp.Location.(*remotesapi.DownloadLoc_HttpGetRange).HttpGetRange
path := ResourcePath(hgr.Url)
if v, ok := d.refreshes[path]; ok {
v.Add(resp)
} else {
@@ -361,8 +389,14 @@ func (d downloads) Add(resp *remotesapi.DownloadLoc) {
refresh.Add(resp)
d.refreshes[path] = refresh
}
for _, r := range gr.Ranges {
d.ranges.Insert(gr.Url, r.Hash, r.Offset, r.Length)
for _, r := range hgr.Ranges {
if r.DictionaryLength != 0 {
_, has := d.dictCache.getOrCreate(path, r.DictionaryOffset, r.DictionaryLength)
if !has {
d.dictRanges.Insert(hgr.Url, nil, r.DictionaryOffset, r.DictionaryLength, 0, 0)
}
}
d.chunkRanges.Insert(hgr.Url, r.Hash[:], r.Offset, r.Length, r.DictionaryOffset, r.DictionaryLength)
}
}
@@ -370,84 +404,41 @@ func toGetRange(rs []*ranges.GetRange) *GetRange {
ret := new(GetRange)
for _, r := range rs {
ret.Url = r.Url
ret.Ranges = append(ret.Ranges, &remotesapi.RangeChunk{
Hash: r.Hash,
Offset: r.Offset,
Length: r.Length,
ret.Ranges = append(ret.Ranges, &Range{
Hash: r.Hash,
Offset: r.Offset,
Length: r.Length,
DictOffset: r.DictOffset,
DictLength: r.DictLength,
})
}
return ret
}
type rangeType int
const (
rangeType_Chunk rangeType = iota
rangeType_Dictionary
)
// Reads off |locCh| and assembles DownloadLocs into download ranges.
func fetcherDownloadRangesThread(ctx context.Context, locCh chan []*remotesapi.DownloadLoc, fetchReqCh chan fetchReq, doneCh chan struct{}) error {
downloads := newDownloads()
pending := make([]fetchReq, 0)
var toSend *GetRange
for {
// pending is our slice of request threads that showed up
// asking for a download. We range through it and try to send
// them any work we have available.
for j := range pending {
// |toSend| could have come from a previous iteration
// of this loop or the outer loop. If it's |nil|, we
// can get the next range to download from
// |downloads.ranges|.
if toSend == nil {
max := downloads.ranges.DeleteMaxRegion()
if len(max) == 0 {
break
}
toSend = toGetRange(max)
}
path := toSend.ResourcePath()
refresh := downloads.refreshes[path]
resp := fetchResp{
get: toSend,
refresh: func(ctx context.Context, err error, client remotesapi.ChunkStoreServiceClient) (string, error) {
return refresh.GetURL(ctx, err, client)
},
}
select {
case pending[j].respCh <- resp:
toSend = nil
case <-pending[j].cancelCh:
// Because of dynamic thread pool sizing, a
// request thread could have been canceled and
// it has now gone away. If this happens, its
// respCh will be set to |nil| below and we
// will remove it from our |pending| set. But
// we need to hold onto |toSend| so that we do
// send it to a request thread eventually.
case <-ctx.Done():
return context.Cause(ctx)
}
pending[j].respCh = nil
}
// Remove anything from |pending| that was actually delivered
// to. We use |respCh == nil| to indicate that the above loop
// delivered to the download thread.
newpending := make([]fetchReq, 0)
for i := range pending {
if pending[i].respCh != nil {
newpending = append(newpending, pending[i])
}
}
pending = newpending
// Once |locCh| closes, we set |locCh| to nil. If |locCh| is
// nil and our ranges Tree is empty, then we have delivered
// every download we will ever see to a download thread. We can
// close |doneCh| and return nil.
if locCh == nil && downloads.ranges.Len() == 0 {
hasWork := downloads.dictRanges.Len() > 0 || downloads.chunkRanges.Len() > 0
if !hasWork && locCh == nil {
// Once |locCh| closes, we sit it to |nil|. If
// our range trees are empty then we have
// already delivered every download we will
// ever see to a download thread.
close(doneCh)
return nil
}
var reqCh chan fetchReq
if hasWork {
reqCh = fetchReqCh
}
select {
case req, ok := <-locCh:
if !ok {
@@ -457,8 +448,32 @@ func fetcherDownloadRangesThread(ctx context.Context, locCh chan []*remotesapi.D
downloads.Add(loc)
}
}
case req := <-fetchReqCh:
pending = append(pending, req)
case req := <-reqCh:
var toSend *GetRange
var toSendType rangeType
if downloads.dictRanges.Len() > 0 {
max := downloads.dictRanges.DeleteMaxRegion()
toSend, toSendType = toGetRange(max), rangeType_Dictionary
} else {
// Necessarily non-empty, since |hasWork| is true...
max := downloads.chunkRanges.DeleteMaxRegion()
toSend, toSendType = toGetRange(max), rangeType_Chunk
}
path := toSend.ResourcePath()
refresh := downloads.refreshes[path]
resp := fetchResp{
get: toSend,
refresh: func(ctx context.Context, err error, client remotesapi.ChunkStoreServiceClient) (string, error) {
return refresh.GetURL(ctx, err, client)
},
rangeType: toSendType,
path: path,
dictCache: downloads.dictCache,
}
// Requester should deliver an exclusive,
// buffered channel where this deliver always
// succeeds.
req.respCh <- resp
case <-ctx.Done():
return context.Cause(ctx)
}
@@ -527,7 +542,7 @@ func (cc *ConcurrencyControl) Run(ctx context.Context, done <-chan struct{}, ss
}
}
func fetcherDownloadURLThreads(ctx context.Context, fetchReqCh chan fetchReq, doneCh chan struct{}, chunkCh chan nbs.CompressedChunk, client remotesapi.ChunkStoreServiceClient, stats StatsRecorder, fetcher HTTPFetcher, params NetworkRequestParams) error {
func fetcherDownloadURLThreads(ctx context.Context, fetchReqCh chan fetchReq, doneCh chan struct{}, chunkCh chan nbs.ToChunker, client remotesapi.ChunkStoreServiceClient, stats StatsRecorder, fetcher HTTPFetcher, params NetworkRequestParams) error {
eg, ctx := errgroup.WithContext(ctx)
cc := &ConcurrencyControl{
MaxConcurrency: params.MaximumConcurrentDownloads,
@@ -559,24 +574,68 @@ func fetcherDownloadURLThreads(ctx context.Context, fetchReqCh chan fetchReq, do
return nil
}
func fetcherDownloadURLThread(ctx context.Context, fetchReqCh chan fetchReq, doneCh <-chan struct{}, chunkCh chan nbs.CompressedChunk, client remotesapi.ChunkStoreServiceClient, stats StatsRecorder, health reliable.HealthRecorder, fetcher HTTPFetcher, params NetworkRequestParams) error {
respCh := make(chan fetchResp)
cancelCh := make(chan struct{})
func deliverChunkCallback(chunkCh chan nbs.ToChunker, path string, dictCache *dictionaryCache) func(context.Context, []byte, *Range) error {
return func(ctx context.Context, bs []byte, rang *Range) error {
h := hash.New(rang.Hash[:])
var cc nbs.ToChunker
if rang.DictLength != 0 {
payload, _ := dictCache.getOrCreate(path, rang.DictOffset, rang.DictLength)
dictRes, err := payload.Get()
if err != nil {
return err
}
cc = nbs.NewArchiveToChunker(h, dictRes.(*gozstd.DDict), bs)
} else {
var err error
cc, err = nbs.NewCompressedChunk(h, bs)
if err != nil {
return err
}
}
select {
case chunkCh <- cc:
case <-ctx.Done():
return context.Cause(ctx)
}
return nil
}
}
func setDictionaryCallback(dictCache *dictionaryCache, path string) func(context.Context, []byte, *Range) error {
return func(ctx context.Context, bs []byte, rang *Range) error {
var ddict *gozstd.DDict
decompressed, err := gozstd.Decompress(nil, bs)
if err == nil {
ddict, err = gozstd.NewDDict(decompressed)
}
payload, _ := dictCache.getOrCreate(path, rang.Offset, rang.Length)
payload.Set(ddict, err)
// XXX: For now, we fail here on any error, instead of when we try to use the dictionary...
// For now, the record in the cache will be terminally failed and is never removed.
return err
}
}
func fetcherDownloadURLThread(ctx context.Context, fetchReqCh chan fetchReq, doneCh <-chan struct{}, chunkCh chan nbs.ToChunker, client remotesapi.ChunkStoreServiceClient, stats StatsRecorder, health reliable.HealthRecorder, fetcher HTTPFetcher, params NetworkRequestParams) error {
respCh := make(chan fetchResp, 1)
for {
select {
case <-ctx.Done():
return context.Cause(ctx)
case <-doneCh:
return nil
case fetchReqCh <- fetchReq{respCh: respCh, cancelCh: cancelCh}:
case fetchReqCh <- fetchReq{respCh: respCh}:
select {
case <-doneCh:
close(cancelCh)
return nil
case <-ctx.Done():
return context.Cause(ctx)
case fetchResp := <-respCh:
f := fetchResp.get.GetDownloadFunc(ctx, stats, health, fetcher, params, chunkCh, func(ctx context.Context, lastError error, resourcePath string) (string, error) {
var cb func(context.Context, []byte, *Range) error
if fetchResp.rangeType == rangeType_Chunk {
cb = deliverChunkCallback(chunkCh, fetchResp.path, fetchResp.dictCache)
} else {
cb = setDictionaryCallback(fetchResp.dictCache, fetchResp.path)
}
f := fetchResp.get.GetDownloadFunc(ctx, stats, health, fetcher, params, cb, func(ctx context.Context, lastError error, resourcePath string) (string, error) {
return fetchResp.refresh(ctx, lastError, client)
})
err := f()
@@ -587,3 +646,48 @@ func fetcherDownloadURLThread(ctx context.Context, fetchReqCh chan fetchReq, don
}
}
}
// A dictionaryCache provides a rendezvous point for chunk fetches
// which have data dependencies on dictionary fetches. It stores a
// single record per |path|,|offset| tuple we see, and that record
// will be populated with the |*gozstd.DDict| that results from
// fetching those contents. Every |GetRange| that has a dictionary
// dependency gets the record out of the dictionary cache. The first
// time the cache entry is created, the download thread also schedules
// the dictionary itself to be fetched and populated through |Set|.
type dictionaryCache struct {
dictionaries sync.Map
}
// DictionaryKey is the a globaly unique identifier for an archive dictionary.
type DictionaryKey struct {
// This is the short url to the resource, not including the query parameters - which are provided by the
// locationRefresher.
path string
off uint64
len uint32
}
type DictionaryPayload struct {
done chan struct{}
res any
err error
}
func (p *DictionaryPayload) Get() (any, error) {
<-p.done
return p.res, p.err
}
func (p *DictionaryPayload) Set(res any, err error) {
p.res = res
p.err = err
close(p.done)
}
func (dc *dictionaryCache) getOrCreate(path string, offset uint64, length uint32) (*DictionaryPayload, bool) {
key := DictionaryKey{path, offset, length}
entry, loaded := dc.dictionaries.LoadOrStore(key, &DictionaryPayload{done: make(chan struct{})})
payload := entry.(*DictionaryPayload)
return payload, loaded
}

View File

@@ -317,7 +317,7 @@ func (dcs *DoltChunkStore) Get(ctx context.Context, h hash.Hash) (chunks.Chunk,
func (dcs *DoltChunkStore) GetMany(ctx context.Context, hashes hash.HashSet, found func(context.Context, *chunks.Chunk)) error {
ae := atomicerr.New()
decompressedSize := uint64(0)
err := dcs.GetManyCompressed(ctx, hashes, func(ctx context.Context, cc nbs.CompressedChunk) {
err := dcs.GetManyCompressed(ctx, hashes, func(ctx context.Context, cc nbs.ToChunker) {
if ae.IsSet() {
return
}
@@ -340,7 +340,7 @@ func (dcs *DoltChunkStore) GetMany(ctx context.Context, hashes hash.HashSet, fou
// GetMany gets the Chunks with |hashes| from the store. On return, |foundChunks| will have been fully sent all chunks
// which have been found. Any non-present chunks will silently be ignored.
func (dcs *DoltChunkStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error {
func (dcs *DoltChunkStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error {
ctx, span := tracer.Start(ctx, "remotestorage.GetManyCompressed")
defer span.End()
@@ -353,7 +353,7 @@ func (dcs *DoltChunkStore) GetManyCompressed(ctx context.Context, hashes hash.Ha
for h := range hashes {
c := hashToChunk[h]
if c.IsEmpty() {
if c == nil || c.IsEmpty() {
notCached = append(notCached, h)
} else {
found(ctx, c)
@@ -371,11 +371,28 @@ func (dcs *DoltChunkStore) GetManyCompressed(ctx context.Context, hashes hash.Ha
return nil
}
type GetRange remotesapi.HttpGetRange
// GetRange is structurally similar to remotesapi.HttpGetRange, but
// with added functions.
type GetRange struct {
Url string
Ranges []*Range
}
type Range struct {
Hash []byte
Offset uint64
Length uint32
DictOffset uint64
DictLength uint32
}
func ResourcePath(urlS string) string {
u, _ := url.Parse(urlS)
return fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Path)
}
func (gr *GetRange) ResourcePath() string {
u, _ := url.Parse(gr.Url)
return fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Path)
return ResourcePath(gr.Url)
}
func (gr *GetRange) Append(other *GetRange) {
@@ -432,10 +449,11 @@ func sortRangesBySize(ranges []*GetRange) {
type resourcePathToUrlFunc func(ctx context.Context, lastError error, resourcePath string) (url string, err error)
func (gr *GetRange) GetDownloadFunc(ctx context.Context, stats StatsRecorder, health reliable.HealthRecorder, fetcher HTTPFetcher, params NetworkRequestParams, chunkChan chan nbs.CompressedChunk, pathToUrl resourcePathToUrlFunc) func() error {
func (gr *GetRange) GetDownloadFunc(ctx context.Context, stats StatsRecorder, health reliable.HealthRecorder, fetcher HTTPFetcher, params NetworkRequestParams, resCb func(context.Context, []byte, *Range) error, pathToUrl resourcePathToUrlFunc) func() error {
if len(gr.Ranges) == 0 {
return func() error { return nil }
}
return func() error {
urlF := func(lastError error) (string, error) {
url, err := pathToUrl(ctx, lastError, gr.ResourcePath())
@@ -466,55 +484,66 @@ func (gr *GetRange) GetDownloadFunc(ctx context.Context, stats StatsRecorder, he
RespHeadersTimeout: params.RespHeadersTimeout,
})
defer resp.Close()
reader := &RangeChunkReader{GetRange: gr, Reader: resp.Body}
reader := &RangeReader{GetRange: gr, Reader: resp.Body}
for {
cc, err := reader.ReadChunk()
bs, rang, err := reader.ReadNextRange()
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
select {
case chunkChan <- cc:
case <-ctx.Done():
return context.Cause(ctx)
err = resCb(ctx, bs, rang)
if err != nil {
return err
}
}
}
}
type RangeChunkReader struct {
type RangeReader struct {
GetRange *GetRange
Reader io.Reader
i int
skip int
// The range we currently reading.
i int
// The |skip|, from the last range we read
// to the current range, which we need to
// exexcute before on the next call to
// |ReadNextRange|
skip int
}
func (r *RangeChunkReader) ReadChunk() (nbs.CompressedChunk, error) {
func (r *RangeReader) ReadNextRange() ([]byte, *Range, error) {
if r.skip > 0 {
_, err := io.CopyN(io.Discard, r.Reader, int64(r.skip))
if err != nil {
return nbs.CompressedChunk{}, err
return nil, nil, err
}
r.skip = 0
}
if r.i >= len(r.GetRange.Ranges) {
return nbs.CompressedChunk{}, io.EOF
}
if r.i < len(r.GetRange.Ranges)-1 {
r.skip = int(r.GetRange.GapBetween(r.i, r.i+1))
}
l := r.GetRange.Ranges[r.i].Length
h := hash.New(r.GetRange.Ranges[r.i].Hash)
idx := r.i
r.i += 1
if idx >= len(r.GetRange.Ranges) {
return nil, nil, io.EOF
}
if idx < len(r.GetRange.Ranges)-1 {
// If this isn't the last range, calculate and
// store the skip that will be necessary after
// we read this range.
r.skip = int(r.GetRange.GapBetween(idx, idx+1))
}
rang := r.GetRange.Ranges[idx]
l := rang.Length
buf := make([]byte, l)
_, err := io.ReadFull(r.Reader, buf)
if err != nil {
return nbs.CompressedChunk{}, err
} else {
return nbs.NewCompressedChunk(h, buf)
return nil, nil, err
}
return buf, rang, nil
}
type locationRefresh struct {
@@ -574,7 +603,7 @@ type RepoRequest interface {
SetRepoPath(string)
}
func (dcs *DoltChunkStore) readChunksAndCache(ctx context.Context, hashes []hash.Hash, found func(context.Context, nbs.CompressedChunk)) (err error) {
func (dcs *DoltChunkStore) readChunksAndCache(ctx context.Context, hashes []hash.Hash, found func(context.Context, nbs.ToChunker)) (err error) {
toSend := hash.NewHashSet(hashes...)
fetcher := dcs.ChunkFetcher(ctx)
@@ -603,9 +632,9 @@ func (dcs *DoltChunkStore) readChunksAndCache(ctx context.Context, hashes []hash
return err
}
// Don't forward on empty/not found chunks.
if len(cc.CompressedData) > 0 {
if dcs.cache.PutChunk(cc) {
return ErrCacheCapacityExceeded
if !cc.IsEmpty() {
if err := dcs.cache.PutChunk(cc); err != nil {
return err
}
found(egCtx, cc)
}
@@ -644,7 +673,7 @@ func (dcs *DoltChunkStore) HasMany(ctx context.Context, hashes hash.HashSet) (ha
hashSl, byteSl := HashSetToSlices(notCached)
absent := make(hash.HashSet)
var found []nbs.CompressedChunk
var found []nbs.ToChunker
var err error
batchItr(len(hashSl), maxHasManyBatchSize, func(st, end int) (stop bool) {
@@ -702,8 +731,8 @@ func (dcs *DoltChunkStore) HasMany(ctx context.Context, hashes hash.HashSet) (ha
}
if len(found) > 0 {
if dcs.cache.Put(found) {
return hash.HashSet{}, ErrCacheCapacityExceeded
if err := dcs.cache.Put(found); err != nil {
return hash.HashSet{}, err
}
}
@@ -738,8 +767,8 @@ func (dcs *DoltChunkStore) Put(ctx context.Context, c chunks.Chunk, getAddrs chu
}
cc := nbs.ChunkToCompressedChunk(c)
if dcs.cache.Put([]nbs.CompressedChunk{cc}) {
return ErrCacheCapacityExceeded
if err := dcs.cache.Put([]nbs.ToChunker{cc}); err != nil {
return err
}
return nil
}
@@ -1051,6 +1080,11 @@ func (dcs *DoltChunkStore) SupportedOperations() chunks.TableFileStoreOps {
// WriteTableFile reads a table file from the provided reader and writes it to the chunk store.
func (dcs *DoltChunkStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) error {
// Err if the suffix is an archive file
if strings.HasSuffix(fileId, nbs.ArchiveFileSuffix) {
return errors.New("cannot write archive file ids currently.")
}
fileIdBytes := hash.Parse(fileId)
err := dcs.uploadTableFileWithRetries(ctx, fileIdBytes, uint64(numChunks), contentHash, getRd)
if err != nil {
@@ -1152,6 +1186,15 @@ func (drtf DoltRemoteTableFile) LocationPrefix() string {
return ""
}
func (drtf DoltRemoteTableFile) LocationSuffix() string {
u, _ := url.Parse(drtf.info.Url)
if strings.HasSuffix(u.Path, nbs.ArchiveFileSuffix) {
return nbs.ArchiveFileSuffix
}
return ""
}
// FileID gets the id of the file
func (drtf DoltRemoteTableFile) FileID() string {
id := drtf.info.FileId

View File

@@ -21,18 +21,27 @@ import (
"github.com/google/btree"
)
// GetRange represents a way to get the contents for a Chunk from a given Url
// with an HTTP Range request. The chunk with hash |Hash| can be fetched using
// GetRange represents a range of remote data that has semantic meaning to the
// ChunkFetcher. These ranges are currently either Chunks, or Dictionaries.
// They can be fetched from the remote URL with an HTTP Range request.
// For a chunk range, the chunk with hash |Hash| can be fetched using
// the |Url| with a Range request starting at |Offset| and reading |Length|
// bytes.
// bytes. A Dictionary does not have a meaningful Hash, but its identity is
// unique for a Url and Offset.
//
// A |GetRange| struct is a member of a |Region| in the |RegionHeap|.
//
// Chunk |GetRange|s which depend on Dictionaries can be constructed with
// some state which allows them to fetch those dictionaries from a shared
// chache when they need them. That is their GetDict callback.
type GetRange struct {
Url string
Hash []byte
Offset uint64
Length uint32
Region *Region
Url string
Hash []byte
Offset uint64
Length uint32
DictOffset uint64
DictLength uint32
Region *Region
}
// A |Region| represents a continuous range of bytes within in a Url.
@@ -145,12 +154,14 @@ func (t *Tree) Len() int {
return t.t.Len()
}
func (t *Tree) Insert(url string, hash []byte, offset uint64, length uint32) {
func (t *Tree) Insert(url string, hash []byte, offset uint64, length uint32, dictOffset uint64, dictLength uint32) {
ins := &GetRange{
Url: t.intern(url),
Hash: hash,
Offset: offset,
Length: length,
Url: t.intern(url),
Hash: hash,
Offset: offset,
Length: length,
DictOffset: dictOffset,
DictLength: dictLength,
}
t.t.ReplaceOrInsert(ins)

View File

@@ -77,11 +77,11 @@ func TestTree(t *testing.T) {
tree := NewTree(8 * 1024)
// Insert 1KB ranges every 16 KB.
for i, j := 0, 0; i < 16; i, j = i+1, j+16*1024 {
tree.Insert("A", []byte{}, uint64(j), 1024)
tree.Insert("A", []byte{}, uint64(j), 1024, 0, 0)
}
// Insert 1KB ranges every 16 KB, offset by 8KB.
for i := 15*16*1024 + 8*1024; i >= 0; i -= 16 * 1024 {
tree.Insert("A", []byte{}, uint64(i), 1024)
tree.Insert("A", []byte{}, uint64(i), 1024, 0, 0)
}
assertTree(t, tree)
})
@@ -89,11 +89,11 @@ func TestTree(t *testing.T) {
tree := NewTree(8 * 1024)
// Insert 1KB ranges every 16 KB, offset by 8KB.
for i := 15*16*1024 + 8*1024; i >= 0; i -= 16 * 1024 {
tree.Insert("A", []byte{}, uint64(i), 1024)
tree.Insert("A", []byte{}, uint64(i), 1024, 0, 0)
}
// Insert 1KB ranges every 16 KB.
for i, j := 0, 0; i < 16; i, j = i+1, j+16*1024 {
tree.Insert("A", []byte{}, uint64(j), 1024)
tree.Insert("A", []byte{}, uint64(j), 1024, 0, 0)
}
assertTree(t, tree)
})
@@ -111,7 +111,7 @@ func TestTree(t *testing.T) {
})
tree := NewTree(8 * 1024)
for _, offset := range entries {
tree.Insert("A", []byte{}, offset, 1024)
tree.Insert("A", []byte{}, offset, 1024, 0, 0)
}
assertTree(t, tree)
}
@@ -126,7 +126,7 @@ func TestTree(t *testing.T) {
"B", "A", "9", "8",
}
for i, j := 0, 0; i < 16; i, j = i+1, j+1024 {
tree.Insert(files[i], []byte{}, uint64(j), 1024)
tree.Insert(files[i], []byte{}, uint64(j), 1024, 0, 0)
}
assert.Equal(t, 16, tree.regions.Len())
assert.Equal(t, 16, tree.t.Len())
@@ -134,17 +134,17 @@ func TestTree(t *testing.T) {
t.Run("MergeInMiddle", func(t *testing.T) {
tree := NewTree(8 * 1024)
// 1KB chunk at byte 0
tree.Insert("A", []byte{}, 0, 1024)
tree.Insert("A", []byte{}, 0, 1024, 0, 0)
// 1KB chunk at byte 16KB
tree.Insert("A", []byte{}, 16384, 1024)
tree.Insert("A", []byte{}, 16384, 1024, 0, 0)
assert.Equal(t, 2, tree.regions.Len())
assert.Equal(t, 2, tree.t.Len())
// 1KB chunk at byte 8KB
tree.Insert("A", []byte{}, 8192, 1024)
tree.Insert("A", []byte{}, 8192, 1024, 0, 0)
assert.Equal(t, 1, tree.regions.Len())
assert.Equal(t, 3, tree.t.Len())
tree.Insert("A", []byte{}, 4096, 1024)
tree.Insert("A", []byte{}, 12228, 1024)
tree.Insert("A", []byte{}, 4096, 1024, 0, 0)
tree.Insert("A", []byte{}, 12228, 1024, 0, 0)
assert.Equal(t, 1, tree.regions.Len())
assert.Equal(t, 5, tree.t.Len())
e, _ := tree.t.Min()
@@ -184,7 +184,7 @@ func TestTree(t *testing.T) {
t.Run("InsertAscending", func(t *testing.T) {
tree := NewTree(4 * 1024)
for _, e := range entries {
tree.Insert(e.url, []byte{e.id}, e.offset, e.length)
tree.Insert(e.url, []byte{e.id}, e.offset, e.length, 0, 0)
}
assertTree(t, tree)
})
@@ -192,7 +192,7 @@ func TestTree(t *testing.T) {
tree := NewTree(4 * 1024)
for i := len(entries) - 1; i >= 0; i-- {
e := entries[i]
tree.Insert(e.url, []byte{e.id}, e.offset, e.length)
tree.Insert(e.url, []byte{e.id}, e.offset, e.length, 0, 0)
}
assertTree(t, tree)
})
@@ -205,7 +205,7 @@ func TestTree(t *testing.T) {
})
tree := NewTree(4 * 1024)
for _, e := range entries {
tree.Insert(e.url, []byte{e.id}, e.offset, e.length)
tree.Insert(e.url, []byte{e.id}, e.offset, e.length, 0, 0)
}
assertTree(t, tree)
}

View File

@@ -21,19 +21,24 @@ import (
"github.com/dolthub/dolt/go/store/nbs"
)
const (
// averageChunkSize is used to estimate the size of chunk for purposes of avoiding excessive memory usage
averageChunkSize = 1 << 12
)
// mapChunkCache is a ChunkCache implementation that stores everything in an in memory map.
type mapChunkCache struct {
mu *sync.Mutex
hashToChunk map[hash.Hash]nbs.CompressedChunk
toFlush map[hash.Hash]nbs.CompressedChunk
hashToChunk map[hash.Hash]nbs.ToChunker
toFlush map[hash.Hash]nbs.ToChunker
cm CapacityMonitor
}
func newMapChunkCache() *mapChunkCache {
return &mapChunkCache{
&sync.Mutex{},
make(map[hash.Hash]nbs.CompressedChunk),
make(map[hash.Hash]nbs.CompressedChunk),
make(map[hash.Hash]nbs.ToChunker),
make(map[hash.Hash]nbs.ToChunker),
NewUncappedCapacityMonitor(),
}
}
@@ -42,14 +47,14 @@ func newMapChunkCache() *mapChunkCache {
func NewMapChunkCacheWithMaxCapacity(maxCapacity int64) *mapChunkCache {
return &mapChunkCache{
&sync.Mutex{},
make(map[hash.Hash]nbs.CompressedChunk),
make(map[hash.Hash]nbs.CompressedChunk),
make(map[hash.Hash]nbs.ToChunker),
make(map[hash.Hash]nbs.ToChunker),
NewFixedCapacityMonitor(maxCapacity),
}
}
// Put puts a slice of chunks into the cache.
func (mcc *mapChunkCache) Put(chnks []nbs.CompressedChunk) bool {
// Put puts a slice of chunks into the cache. Returns an error if the cache capacity has been exceeded.
func (mcc *mapChunkCache) Put(chnks []nbs.ToChunker) error {
mcc.mu.Lock()
defer mcc.mu.Unlock()
@@ -63,8 +68,8 @@ func (mcc *mapChunkCache) Put(chnks []nbs.CompressedChunk) bool {
}
}
if mcc.cm.CapacityExceeded(len(c.FullCompressedChunk)) {
return true
if mcc.cm.CapacityExceeded(averageChunkSize) {
return ErrCacheCapacityExceeded
}
mcc.hashToChunk[h] = c
@@ -74,13 +79,13 @@ func (mcc *mapChunkCache) Put(chnks []nbs.CompressedChunk) bool {
}
}
return false
return nil
}
// Get gets a map of hash to chunk for a set of hashes. In the event that a chunk is not in the cache, chunks.Empty.
// is put in it's place
func (mcc *mapChunkCache) Get(hashes hash.HashSet) map[hash.Hash]nbs.CompressedChunk {
hashToChunk := make(map[hash.Hash]nbs.CompressedChunk)
func (mcc *mapChunkCache) Get(hashes hash.HashSet) map[hash.Hash]nbs.ToChunker {
hashToChunk := make(map[hash.Hash]nbs.ToChunker)
mcc.mu.Lock()
defer mcc.mu.Unlock()
@@ -112,26 +117,26 @@ func (mcc *mapChunkCache) Has(hashes hash.HashSet) (absent hash.HashSet) {
return absent
}
func (mcc *mapChunkCache) PutChunk(ch nbs.CompressedChunk) bool {
func (mcc *mapChunkCache) PutChunk(ch nbs.ToChunker) error {
mcc.mu.Lock()
defer mcc.mu.Unlock()
h := ch.Hash()
if existing, ok := mcc.hashToChunk[h]; !ok || existing.IsEmpty() {
if mcc.cm.CapacityExceeded(len(ch.FullCompressedChunk)) {
return true
if mcc.cm.CapacityExceeded(averageChunkSize) {
return ErrCacheCapacityExceeded
}
mcc.hashToChunk[h] = ch
mcc.toFlush[h] = ch
}
return false
return nil
}
// GetAndClearChunksToFlush gets a map of hash to chunk which includes all the chunks that were put in the cache
// between the last time GetAndClearChunksToFlush was called and now.
func (mcc *mapChunkCache) GetAndClearChunksToFlush() map[hash.Hash]nbs.CompressedChunk {
newToFlush := make(map[hash.Hash]nbs.CompressedChunk)
func (mcc *mapChunkCache) GetAndClearChunksToFlush() map[hash.Hash]nbs.ToChunker {
newToFlush := make(map[hash.Hash]nbs.ToChunker)
mcc.mu.Lock()
defer mcc.mu.Unlock()

View File

@@ -27,8 +27,8 @@ import (
"github.com/dolthub/dolt/go/store/nbs"
)
func genRandomChunks(rng *rand.Rand, n int) (hash.HashSet, []nbs.CompressedChunk) {
chks := make([]nbs.CompressedChunk, n)
func genRandomChunks(rng *rand.Rand, n int) (hash.HashSet, []nbs.ToChunker) {
chks := make([]nbs.ToChunker, n)
hashes := make(hash.HashSet)
for i := 0; i < n; i++ {
size := int(rng.Int31n(99) + 1)
@@ -88,7 +88,7 @@ func TestMapChunkCache(t *testing.T) {
toFlush = mapChunkCache.GetAndClearChunksToFlush()
expected := map[hash.Hash]nbs.CompressedChunk{moreChks[0].Hash(): moreChks[0]}
expected := map[hash.Hash]nbs.ToChunker{moreChks[0].Hash(): moreChks[0]}
eq := reflect.DeepEqual(toFlush, expected)
assert.True(t, eq, "Missing or unexpected chunks to flush (seed %d)", seed)
}

View File

@@ -29,22 +29,22 @@ var noopChunkCache = &noopChunkCacheImpl{}
type noopChunkCacheImpl struct {
}
func (*noopChunkCacheImpl) Put(chnks []nbs.CompressedChunk) bool {
return false
func (*noopChunkCacheImpl) Put(chnks []nbs.ToChunker) error {
return nil
}
func (*noopChunkCacheImpl) Get(hashes hash.HashSet) map[hash.Hash]nbs.CompressedChunk {
return make(map[hash.Hash]nbs.CompressedChunk)
func (*noopChunkCacheImpl) Get(hashes hash.HashSet) map[hash.Hash]nbs.ToChunker {
return make(map[hash.Hash]nbs.ToChunker)
}
func (*noopChunkCacheImpl) Has(hashes hash.HashSet) (absent hash.HashSet) {
return hashes
}
func (*noopChunkCacheImpl) PutChunk(ch nbs.CompressedChunk) bool {
return false
func (*noopChunkCacheImpl) PutChunk(ch nbs.ToChunker) error {
return nil
}
func (*noopChunkCacheImpl) GetAndClearChunksToFlush() map[hash.Hash]nbs.CompressedChunk {
func (*noopChunkCacheImpl) GetAndClearChunksToFlush() map[hash.Hash]nbs.ToChunker {
panic("noopChunkCache does not support GetAndClearChunksToFlush().")
}

View File

@@ -31,6 +31,10 @@ type TableFile interface {
// LocationPrefix
LocationPrefix() string
// Used in conjuction with the FileID to create file paths to table files. Currently archive files are the only
// that take advantage of this, using .darc as the file suffix.
LocationSuffix() string
// NumChunks returns the number of chunks in a table file
NumChunks() int

View File

@@ -81,9 +81,11 @@ func mapTableFiles(tblFiles []chunks.TableFile) ([]string, map[string]chunks.Tab
fileIDtoNumChunks := make(map[string]int)
for i, tblFile := range tblFiles {
fileIDtoTblFile[tblFile.FileID()] = tblFile
fileIds[i] = tblFile.FileID()
fileIDtoNumChunks[tblFile.FileID()] = tblFile.NumChunks()
fileId := tblFile.FileID()
fileIDtoTblFile[fileId] = tblFile
fileIds[i] = fileId
fileIDtoNumChunks[fileId] = tblFile.NumChunks()
}
return fileIds, fileIDtoTblFile, fileIDtoNumChunks
@@ -136,7 +138,7 @@ func clone(ctx context.Context, srcTS, sinkTS chunks.TableFileStore, sinkCS chun
}
report(TableFileEvent{EventType: DownloadStart, TableFiles: []chunks.TableFile{tblFile}})
err = sinkTS.WriteTableFile(ctx, tblFile.FileID(), tblFile.NumChunks(), nil, func() (io.ReadCloser, uint64, error) {
err = sinkTS.WriteTableFile(ctx, tblFile.FileID()+tblFile.LocationSuffix(), tblFile.NumChunks(), nil, func() (io.ReadCloser, uint64, error) {
rd, contentLength, err := tblFile.Open(ctx)
if err != nil {
return nil, 0, err

View File

@@ -26,7 +26,7 @@ import (
)
type GetManyer interface {
GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error
GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error
}
type ChunkFetcherable interface {
@@ -52,7 +52,7 @@ type PullChunkFetcher struct {
batchCh chan hash.HashSet
doneCh chan struct{}
resCh chan nbs.CompressedChunk
resCh chan nbs.ToChunker
}
func NewPullChunkFetcher(ctx context.Context, getter GetManyer) *PullChunkFetcher {
@@ -63,7 +63,7 @@ func NewPullChunkFetcher(ctx context.Context, getter GetManyer) *PullChunkFetche
getter: getter,
batchCh: make(chan hash.HashSet),
doneCh: make(chan struct{}),
resCh: make(chan nbs.CompressedChunk),
resCh: make(chan nbs.ToChunker),
}
ret.eg.Go(func() error {
return ret.fetcherThread(func() {
@@ -86,9 +86,9 @@ func (f *PullChunkFetcher) fetcherThread(finalize func()) error {
missing := batch.Copy()
// Blocking get, no concurrency, only one fetcher.
err := f.getter.GetManyCompressed(f.ctx, batch, func(ctx context.Context, chk nbs.CompressedChunk) {
err := f.getter.GetManyCompressed(f.ctx, batch, func(ctx context.Context, chk nbs.ToChunker) {
mu.Lock()
missing.Remove(chk.H)
missing.Remove(chk.Hash())
mu.Unlock()
select {
case <-ctx.Done():
@@ -139,7 +139,7 @@ func (f *PullChunkFetcher) Close() error {
return f.eg.Wait()
}
func (f *PullChunkFetcher) Recv(ctx context.Context) (nbs.CompressedChunk, error) {
func (f *PullChunkFetcher) Recv(ctx context.Context) (nbs.ToChunker, error) {
select {
case res, ok := <-f.resCh:
if !ok {

View File

@@ -70,8 +70,12 @@ func TestPullChunkFetcher(t *testing.T) {
defer wg.Done()
cmp, err := f.Recv(context.Background())
assert.NoError(t, err)
assert.Equal(t, cmp.H, gm.C.H)
assert.Equal(t, cmp.FullCompressedChunk, gm.C.FullCompressedChunk)
assert.Equal(t, cmp.Hash(), gm.C.H)
cc, ok := cmp.(nbs.CompressedChunk)
assert.True(t, ok)
assert.Equal(t, cc.FullCompressedChunk, gm.C.FullCompressedChunk)
_, err = f.Recv(context.Background())
assert.ErrorIs(t, err, io.EOF)
assert.NoError(t, f.Close())
@@ -92,8 +96,11 @@ func TestPullChunkFetcher(t *testing.T) {
defer wg.Done()
cmp, err := f.Recv(context.Background())
assert.NoError(t, err)
assert.Equal(t, cmp.H, h)
assert.Nil(t, cmp.FullCompressedChunk)
assert.Equal(t, cmp.Hash(), h)
cc, ok := cmp.(nbs.CompressedChunk)
assert.True(t, ok)
assert.Nil(t, cc.FullCompressedChunk)
_, err = f.Recv(context.Background())
assert.ErrorIs(t, err, io.EOF)
assert.NoError(t, f.Close())
@@ -136,7 +143,7 @@ func TestPullChunkFetcher(t *testing.T) {
type emptyGetManyer struct {
}
func (emptyGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error {
func (emptyGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error {
return nil
}
@@ -144,7 +151,7 @@ type deliveringGetManyer struct {
C nbs.CompressedChunk
}
func (d deliveringGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error {
func (d deliveringGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error {
for _ = range hashes {
found(ctx, d.C)
}
@@ -155,7 +162,7 @@ type blockingGetManyer struct {
block chan struct{}
}
func (b blockingGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error {
func (b blockingGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error {
<-b.block
return nil
}
@@ -165,6 +172,6 @@ type errorGetManyer struct {
var getManyerErr = fmt.Errorf("always return an error")
func (errorGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error {
func (errorGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error {
return getManyerErr
}

View File

@@ -359,17 +359,19 @@ func (p *Puller) Pull(ctx context.Context) error {
if cChk.IsGhost() {
return fmt.Errorf("attempted to push or pull ghost chunk: %w", nbs.ErrGhostChunkRequested)
}
if len(cChk.FullCompressedChunk) == 0 {
if cChk.IsEmpty() {
return errors.New("failed to get all chunks.")
}
atomic.AddUint64(&p.stats.fetchedSourceBytes, uint64(len(cChk.FullCompressedChunk)))
atomic.AddUint64(&p.stats.fetchedSourceChunks, uint64(1))
chnk, err := cChk.ToChunk()
if err != nil {
return err
}
atomic.AddUint64(&p.stats.fetchedSourceBytes, uint64(len(chnk.Data())))
err = p.waf(chnk, func(h hash.Hash, _ bool) error {
tracker.Seen(h)
return nil
@@ -379,9 +381,19 @@ func (p *Puller) Pull(ctx context.Context) error {
}
tracker.TickProcessed()
err = p.wr.AddCompressedChunk(ctx, cChk)
if err != nil {
return err
if compressedChunk, ok := cChk.(nbs.CompressedChunk); ok {
err = p.wr.AddCompressedChunk(ctx, compressedChunk)
if err != nil {
return err
}
} else if _, ok := cChk.(nbs.ArchiveToChunker); ok {
// NM4 - Until we can write quickly to archives.....
cc := nbs.ChunkToCompressedChunk(chnk)
err = p.wr.AddCompressedChunk(ctx, cc)
if err != nil {
return err
}
}
}
})

View File

@@ -27,7 +27,7 @@ Chunks from the Archive.
ByteSpans are arbitrary offset/lengths into the file which store (1) zstd dictionary data, and (2) compressed chunk
data. Each Chunk is stored as a pair of ByteSpans (dict,data). Dictionary ByteSpans can (should) be used by multiple
Chunks, so there are more ByteSpans than Chunks. The Index is used to map Chunks to ByteSpan pairs. These pairs are
called ChunkRefs, and were store them as [uint32,uint32] on disk. This allows us to quickly find the ByteSpans for a
called ChunkRefs, and we store them as [uint32,uint32] on disk. This allows us to quickly find the ByteSpans for a
given Chunk with minimal processing at load time.
A Dolt Archive file follows the following format:
@@ -162,7 +162,7 @@ const (
archiveCheckSumSize +
1 + // version byte
archiveFileSigSize
archiveFileSuffix = ".darc"
ArchiveFileSuffix = ".darc"
)
/*

View File

@@ -57,7 +57,7 @@ func UnArchive(ctx context.Context, cs chunks.ChunkStore, smd StorageMetadata, p
return err
}
if exists {
// We have a fast path to follow because oritinal table file is still on disk.
// We have a fast path to follow because original table file is still on disk.
swapMap[arc.hash()] = orginTfId
} else {
// We don't have the original table file id, so we have to create a new one.

View File

@@ -36,7 +36,7 @@ type archiveChunkSource struct {
var _ chunkSource = &archiveChunkSource{}
func newArchiveChunkSource(ctx context.Context, dir string, h hash.Hash, chunkCount uint32, q MemoryQuotaProvider) (archiveChunkSource, error) {
archiveFile := filepath.Join(dir, h.String()+archiveFileSuffix)
archiveFile := filepath.Join(dir, h.String()+ArchiveFileSuffix)
file, size, err := openReader(archiveFile)
if err != nil {
@@ -141,6 +141,10 @@ func (acs archiveChunkSource) hash() hash.Hash {
return acs.aRdr.footer.hash
}
func (acs archiveChunkSource) name() string {
return acs.hash().String() + ArchiveFileSuffix
}
func (acs archiveChunkSource) currentSize() uint64 {
return acs.aRdr.footer.fileSize
}
@@ -167,14 +171,54 @@ func (acs archiveChunkSource) clone() (chunkSource, error) {
return archiveChunkSource{acs.file, rdr}, nil
}
func (acs archiveChunkSource) getRecordRanges(_ context.Context, _ []getRecord, _ keeperF) (map[hash.Hash]Range, gcBehavior, error) {
return nil, gcBehavior_Continue, errors.New("Archive chunk source does not support getRecordRanges")
func (acs archiveChunkSource) getRecordRanges(_ context.Context, requests []getRecord, keeper keeperF) (map[hash.Hash]Range, gcBehavior, error) {
result := make(map[hash.Hash]Range, len(requests))
for _, req := range requests {
hAddr := *req.a
idx := acs.aRdr.search(hAddr)
if idx < 0 {
// Chunk not found.
continue
}
if keeper != nil && keeper(hAddr) {
return nil, gcBehavior_Block, nil
}
dictId, dataId := acs.aRdr.getChunkRef(idx)
dataSpan := acs.aRdr.getByteSpanByID(dataId)
dictSpan := acs.aRdr.getByteSpanByID(dictId)
rng := Range{
Offset: dataSpan.offset,
Length: uint32(dataSpan.length),
DictOffset: dictSpan.offset,
DictLength: uint32(dictSpan.length),
}
result[hAddr] = rng
}
return result, gcBehavior_Continue, nil
}
func (acs archiveChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
return acs.getMany(ctx, eg, reqs, func(ctx context.Context, chk *chunks.Chunk) {
found(ctx, ChunkToCompressedChunk(*chk))
}, keeper, stats)
func (acs archiveChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
foundAll := true
for i, req := range reqs {
h := *req.a
toChk, err := acs.aRdr.getAsToChunker(h)
if err != nil {
return true, gcBehavior_Continue, err
}
if toChk == nil || toChk.IsEmpty() {
foundAll = false
} else {
if keeper != nil && keeper(h) {
return true, gcBehavior_Block, nil
}
found(ctx, toChk)
reqs[i].found = true
}
}
return !foundAll, gcBehavior_Continue, nil
}
func (acs archiveChunkSource) iterateAllChunks(ctx context.Context, cb func(chunks.Chunk), _ *Stats) error {

View File

@@ -0,0 +1,65 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nbs
import (
"github.com/dolthub/gozstd"
"github.com/dolthub/dolt/go/store/chunks"
"github.com/dolthub/dolt/go/store/hash"
)
type ArchiveToChunker struct {
h hash.Hash
dictionary *gozstd.DDict
chunkData []byte
}
var _ ToChunker = (*ArchiveToChunker)(nil)
func NewArchiveToChunker(h hash.Hash, dict *gozstd.DDict, chunkData []byte) ToChunker {
return ArchiveToChunker{h: h, dictionary: dict, chunkData: chunkData}
}
func (a ArchiveToChunker) Hash() hash.Hash {
return a.h
}
func (a ArchiveToChunker) ToChunk() (chunks.Chunk, error) {
dict := a.dictionary
data := a.chunkData
rawChunk, err := gozstd.DecompressDict(nil, data, dict)
if err != nil {
return chunks.EmptyChunk, err
}
newChunk := chunks.NewChunk(rawChunk)
// TODO: remove this once we have confidence in archives.
if newChunk.Hash() != a.h {
panic("Hash Mismatch!!")
}
return newChunk, err
}
func (a ArchiveToChunker) IsEmpty() bool {
return len(a.chunkData) == 0
}
func (a ArchiveToChunker) IsGhost() bool {
// archives are never ghosts. They are only instantiated when the chunk is found.
return false
}

View File

@@ -19,6 +19,7 @@ import (
"crypto/sha512"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"io"
"math/bits"
@@ -271,19 +272,33 @@ func (ar archiveReader) get(hash hash.Hash) ([]byte, error) {
if err != nil || data == nil {
return nil, err
}
if dict == nil {
return nil, errors.New("runtime error: unable to get archived chunk. dictionary is nil")
}
var result []byte
if dict == nil {
result, err = gozstd.Decompress(nil, data)
} else {
result, err = gozstd.DecompressDict(nil, data, dict)
}
result, err = gozstd.DecompressDict(nil, data, dict)
if err != nil {
return nil, err
}
return result, nil
}
// getAsToChunker returns the chunk which is has not been decompressed. Similar to get, but with a different return type.
// If the hash is not found, a ToChunker instance with IsEmpty() == true is returned (no error)
func (ar archiveReader) getAsToChunker(h hash.Hash) (ToChunker, error) {
dict, data, err := ar.getRaw(h)
if err != nil {
return nil, err
}
if data == nil {
return ArchiveToChunker{h, nil, []byte{}}, nil
}
return ArchiveToChunker{h, dict, data}, nil
}
func (ar archiveReader) count() uint32 {
return ar.footer.chunkCount
}
@@ -308,7 +323,7 @@ func (ar archiveReader) readByteSpan(bs byteSpan) ([]byte, error) {
// getRaw returns the raw data for the given hash. If the hash is not found, nil is returned for both slices. Also,
// no error is returned in this case. Errors will only be returned if there is an io error.
//
// The data returned is still compressed, regardless of the dictionary being present or not.
// The data returned is still compressed, and the DDict is required to decompress it.
func (ar archiveReader) getRaw(hash hash.Hash) (dict *gozstd.DDict, data []byte, err error) {
idx := ar.search(hash)
if idx < 0 {

View File

@@ -115,41 +115,50 @@ func TestArchiveSingleChunkWithDictionary(t *testing.T) {
func TestArchiverMultipleChunksMultipleDictionaries(t *testing.T) {
writer := NewFixedBufferByteSink(make([]byte, 4096))
aw := newArchiveWriterWithSink(writer)
data1 := []byte{11, 11, 11, 11, 11, 11, 11, 11, 11, 11} // span 1
dict1, _ := generateDictionary(1) // span 2
data2 := []byte{22, 22, 22, 22, 22, 22, 22, 22, 22, 22} // span 3
data3 := []byte{33, 33, 33, 33, 33, 33, 33, 33, 33, 33} // span 4
data4 := []byte{44, 44, 44, 44, 44, 44, 44, 44, 44, 44} // span 5
dict2, _ := generateDictionary(2) // span 6
defDict, _ := generateTerribleDefaultDictionary() // span 1
data1 := []byte{11, 11, 11, 11, 11, 11, 11, 11, 11, 11} // span 2
dict1, _ := generateDictionary(1) // span 3
data2 := []byte{22, 22, 22, 22, 22, 22, 22, 22, 22, 22} // span 4
data3 := []byte{33, 33, 33, 33, 33, 33, 33, 33, 33, 33} // span 5
data4 := []byte{44, 44, 44, 44, 44, 44, 44, 44, 44, 44} // span 6
dict2, _ := generateDictionary(2) // span 7
id, _ := aw.writeByteSpan(defDict)
assert.Equal(t, uint32(1), id)
h1 := hashWithPrefix(t, 42)
id, _ := aw.writeByteSpan(data1)
assert.Equal(t, uint32(1), id)
_ = aw.stageChunk(h1, 0, 1)
id, _ = aw.writeByteSpan(data1)
assert.Equal(t, uint32(2), id)
_ = aw.stageChunk(h1, 1, 2)
h2 := hashWithPrefix(t, 42)
_, _ = aw.writeByteSpan(dict1)
_, _ = aw.writeByteSpan(data2)
_ = aw.stageChunk(h2, 2, 3)
id, _ = aw.writeByteSpan(dict1)
assert.Equal(t, uint32(3), id)
id, _ = aw.writeByteSpan(data2)
assert.Equal(t, uint32(4), id)
_ = aw.stageChunk(h2, 3, 4)
h3 := hashWithPrefix(t, 42)
_, _ = aw.writeByteSpan(data3)
_ = aw.stageChunk(h3, 2, 4)
id, _ = aw.writeByteSpan(data3)
assert.Equal(t, uint32(5), id)
_ = aw.stageChunk(h3, 3, 5)
h4 := hashWithPrefix(t, 81)
_, _ = aw.writeByteSpan(data4)
_ = aw.stageChunk(h4, 0, 5)
id, _ = aw.writeByteSpan(data4)
assert.Equal(t, uint32(6), id)
_ = aw.stageChunk(h4, 0, 6)
h5 := hashWithPrefix(t, 21)
id, _ = aw.writeByteSpan(dict2)
assert.Equal(t, uint32(6), id)
_ = aw.stageChunk(h5, 6, 1)
assert.Equal(t, uint32(7), id)
_ = aw.stageChunk(h5, 7, 2)
h6 := hashWithPrefix(t, 88)
_ = aw.stageChunk(h6, 6, 1)
_ = aw.stageChunk(h6, 7, 2)
h7 := hashWithPrefix(t, 42)
_ = aw.stageChunk(h7, 2, 4)
_ = aw.stageChunk(h7, 3, 5)
_ = aw.finalizeByteSpans()
_ = aw.writeIndex()
@@ -175,7 +184,7 @@ func TestArchiverMultipleChunksMultipleDictionaries(t *testing.T) {
assert.False(t, aIdx.has(hashWithPrefix(t, 55)))
dict, data, _ := aIdx.getRaw(h1)
assert.Nil(t, dict)
assert.NotNil(t, dict)
assert.Equal(t, data1, data)
dict, data, _ = aIdx.getRaw(h2)
@@ -280,13 +289,17 @@ func TestMetadata(t *testing.T) {
// zStd has a CRC check built into it, and it will get triggered when we
// attempt to decompress a corrupted chunk.
func TestArchiveChunkCorruption(t *testing.T) {
writer := NewFixedBufferByteSink(make([]byte, 1024))
writer := NewFixedBufferByteSink(make([]byte, 4096))
aw := newArchiveWriterWithSink(writer)
defDict, _ := generateTerribleDefaultDictionary()
_, _ = aw.writeByteSpan(defDict)
testBlob := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
_, _ = aw.writeByteSpan(testBlob)
h := hashWithPrefix(t, 23)
_ = aw.stageChunk(h, 0, 1)
_ = aw.stageChunk(h, 1, 2)
_ = aw.finalizeByteSpans()
_ = aw.writeIndex()
_ = aw.writeMetadata(nil)
@@ -299,7 +312,7 @@ func TestArchiveChunkCorruption(t *testing.T) {
assert.NoError(t, err)
// Corrupt the data
writer.buff[3] = writer.buff[3] + 1
writer.buff[len(defDict)+3] = writer.buff[len(defDict)+3] + 1
data, err := idx.get(h)
assert.ErrorContains(t, err, "cannot decompress invalid src")
@@ -592,7 +605,7 @@ func hashWithPrefix(t *testing.T, prefix uint64) hash.Hash {
return hash.Hash(randomBytes)
}
// For tests which need a dictionary, we generate a terrible one because we don't care about the actual compression.
// Most tests need a test dictionary. We generate a terrible one because we don't care about the actual compression.
// We return both the raw form and the CDict form.
func generateTerribleDefaultDictionary() ([]byte, *gozstd.CDict) {
return generateDictionary(1977)
@@ -676,7 +689,7 @@ func (tcs *testChunkSource) getMany(ctx context.Context, eg *errgroup.Group, req
panic("never used")
}
func (tcs *testChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
func (tcs *testChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
panic("never used")
}
@@ -696,6 +709,10 @@ func (tcs *testChunkSource) hash() hash.Hash {
panic("never used")
}
func (tcs *testChunkSource) name() string {
panic("never used")
}
func (tcs *testChunkSource) reader(ctx context.Context) (io.ReadCloser, uint64, error) {
panic("never used")
}

View File

@@ -426,7 +426,7 @@ func (aw *archiveWriter) genFileName(path string) (string, error) {
return "", err
}
fileName := fmt.Sprintf("%s%s", h.String(), archiveFileSuffix)
fileName := fmt.Sprintf("%s%s", h.String(), ArchiveFileSuffix)
fullPath := filepath.Join(path, fileName)
return fullPath, nil
}

View File

@@ -46,7 +46,7 @@ type ChunkFetcher interface {
CloseSend() error
Recv(context.Context) (CompressedChunk, error)
Recv(context.Context) (ToChunker, error)
Close() error
}

View File

@@ -29,6 +29,10 @@ func (csa chunkSourceAdapter) hash() hash.Hash {
return csa.h
}
func (csa chunkSourceAdapter) name() string {
return csa.h.String()
}
func newReaderFromIndexData(ctx context.Context, q MemoryQuotaProvider, idxData []byte, name hash.Hash, tra tableReaderAt, blockSize uint64) (cs chunkSource, err error) {
index, err := parseTableIndexByCopy(ctx, idxData, q)
if err != nil {

View File

@@ -75,17 +75,22 @@ func (tw *CmpChunkTableWriter) GetMD5() []byte {
}
// AddCmpChunk adds a compressed chunk
func (tw *CmpChunkTableWriter) AddCmpChunk(c CompressedChunk) error {
if c.IsGhost() {
func (tw *CmpChunkTableWriter) AddCmpChunk(tc ToChunker) error {
if tc.IsGhost() {
// Ghost chunks cannot be written to a table file. They should
// always be filtered by the write processes before landing
// here.
return ErrGhostChunkRequested
}
if len(c.CompressedData) == 0 {
if tc.IsEmpty() {
panic("NBS blocks cannot be zero length")
}
c, ok := tc.(CompressedChunk)
if !ok {
panic("runtime error: Require a CompressedChunk instance")
}
uncmpLen, err := snappy.DecodedLen(c.CompressedData)
if err != nil {

View File

@@ -48,10 +48,10 @@ func TestCmpChunkTableWriter(t *testing.T) {
}
reqs := toGetRecords(hashes)
found := make([]CompressedChunk, 0)
found := make([]ToChunker, 0)
eg, egCtx := errgroup.WithContext(ctx)
_, _, err = tr.getManyCompressed(egCtx, eg, reqs, func(ctx context.Context, c CompressedChunk) { found = append(found, c) }, nil, &Stats{})
_, _, err = tr.getManyCompressed(egCtx, eg, reqs, func(ctx context.Context, c ToChunker) { found = append(found, c) }, nil, &Stats{})
require.NoError(t, err)
require.NoError(t, eg.Wait())

View File

@@ -50,7 +50,7 @@ func (ecs emptyChunkSource) getMany(ctx context.Context, eg *errgroup.Group, req
return true, gcBehavior_Continue, nil
}
func (ecs emptyChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
func (ecs emptyChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
return true, gcBehavior_Continue, nil
}
@@ -66,6 +66,10 @@ func (ecs emptyChunkSource) hash() hash.Hash {
return hash.Hash{}
}
func (ecs emptyChunkSource) name() string {
return ecs.hash().String()
}
func (ecs emptyChunkSource) index() (tableIndex, error) {
return onHeapTableIndex{}, nil
}

View File

@@ -54,7 +54,7 @@ func tableFileExists(ctx context.Context, dir string, h hash.Hash) (bool, error)
}
func archiveFileExists(ctx context.Context, dir string, h hash.Hash) (bool, error) {
darc := fmt.Sprintf("%s%s", h.String(), archiveFileSuffix)
darc := fmt.Sprintf("%s%s", h.String(), ArchiveFileSuffix)
path := filepath.Join(dir, darc)
_, err := os.Stat(path)
@@ -169,6 +169,10 @@ func (ftr *fileTableReader) hash() hash.Hash {
return ftr.h
}
func (ftr *fileTableReader) name() string {
return ftr.h.String()
}
func (ftr *fileTableReader) Close() error {
return ftr.tableReader.close()
}

View File

@@ -56,7 +56,7 @@ func newGarbageCollectionCopier(tfp tableFilePersister) (*gcCopier, error) {
return &gcCopier{writer, tfp}, nil
}
func (gcc *gcCopier) addChunk(ctx context.Context, c CompressedChunk) error {
func (gcc *gcCopier) addChunk(ctx context.Context, c ToChunker) error {
return gcc.writer.AddCmpChunk(c)
}

View File

@@ -28,6 +28,7 @@ import (
)
var _ chunks.ChunkStore = (*GenerationalNBS)(nil)
var _ chunks.GenerationalCS = (*GenerationalNBS)(nil)
var _ chunks.TableFileStore = (*GenerationalNBS)(nil)
var _ chunks.GenerationalCS = (*GenerationalNBS)(nil)
var _ chunks.ChunkStoreGarbageCollector = (*GenerationalNBS)(nil)
@@ -144,14 +145,14 @@ func (gcs *GenerationalNBS) GetMany(ctx context.Context, hashes hash.HashSet, fo
return gcs.ghostGen.GetMany(ctx, notFound, found)
}
func (gcs *GenerationalNBS) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk)) error {
func (gcs *GenerationalNBS) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker)) error {
return gcs.getManyCompressed(ctx, hashes, found, gcDependencyMode_TakeDependency)
}
func (gcs *GenerationalNBS) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk), gcDepMode gcDependencyMode) error {
func (gcs *GenerationalNBS) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker), gcDepMode gcDependencyMode) error {
var mu sync.Mutex
notInOldGen := hashes.Copy()
err := gcs.oldGen.getManyCompressed(ctx, hashes, func(ctx context.Context, chunk CompressedChunk) {
err := gcs.oldGen.getManyCompressed(ctx, hashes, func(ctx context.Context, chunk ToChunker) {
mu.Lock()
delete(notInOldGen, chunk.Hash())
mu.Unlock()
@@ -165,7 +166,7 @@ func (gcs *GenerationalNBS) getManyCompressed(ctx context.Context, hashes hash.H
}
notFound := notInOldGen.Copy()
err = gcs.newGen.getManyCompressed(ctx, notInOldGen, func(ctx context.Context, chunk CompressedChunk) {
err = gcs.newGen.getManyCompressed(ctx, notInOldGen, func(ctx context.Context, chunk ToChunker) {
mu.Lock()
delete(notFound, chunk.Hash())
mu.Unlock()

View File

@@ -90,11 +90,11 @@ func (g GhostBlockStore) GetMany(ctx context.Context, hashes hash.HashSet, found
return nil
}
func (g GhostBlockStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk)) error {
func (g GhostBlockStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker)) error {
return g.getManyCompressed(ctx, hashes, found, gcDependencyMode_TakeDependency)
}
func (g GhostBlockStore) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk), gcDepMode gcDependencyMode) error {
func (g GhostBlockStore) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker), gcDepMode gcDependencyMode) error {
for h := range hashes {
if g.skippedRefs.Has(h) {
found(ctx, NewGhostCompressedChunk(h))

View File

@@ -74,8 +74,8 @@ func TestGhostBlockStore(t *testing.T) {
require.Equal(t, ghost, got[0].Hash())
})
t.Run("GetManyCompressed", func(t *testing.T) {
var got []CompressedChunk
err := bs.GetManyCompressed(ctx, hash.NewHashSet(absent, ghost), func(_ context.Context, c CompressedChunk) {
var got []ToChunker
err := bs.GetManyCompressed(ctx, hash.NewHashSet(absent, ghost), func(_ context.Context, c ToChunker) {
got = append(got, c)
})
require.NoError(t, err)

View File

@@ -96,7 +96,7 @@ type journalRecord struct {
}
func (s journalChunkSource) getMany(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, *chunks.Chunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
return s.getManyCompressed(ctx, eg, reqs, func(ctx context.Context, cc CompressedChunk) {
return s.getManyCompressed(ctx, eg, reqs, func(ctx context.Context, cc ToChunker) {
ch, err := cc.ToChunk()
if err != nil {
eg.Go(func() error {
@@ -115,7 +115,7 @@ func (s journalChunkSource) getMany(ctx context.Context, eg *errgroup.Group, req
// and then (4) asynchronously perform reads. We release the journal read
// lock after returning when all reads are completed, which can be after the
// function returns.
func (s journalChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
func (s journalChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
defer trace.StartRegion(ctx, "journalChunkSource.getManyCompressed").End()
var remaining bool
@@ -185,6 +185,10 @@ func (s journalChunkSource) hash() hash.Hash {
return journalAddr
}
func (s journalChunkSource) name() string {
return s.hash().String()
}
// reader implements chunkSource.
func (s journalChunkSource) reader(ctx context.Context) (io.ReadCloser, uint64, error) {
rdr, sz, err := s.journal.snapshot(ctx)

View File

@@ -193,7 +193,7 @@ func (mt *memTable) getMany(ctx context.Context, eg *errgroup.Group, reqs []getR
return remaining, gcBehavior_Continue, nil
}
func (mt *memTable) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
func (mt *memTable) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
var remaining bool
for i, r := range reqs {
data := mt.chunks[*r.a]

View File

@@ -306,7 +306,7 @@ func (crg chunkReaderGroup) getMany(ctx context.Context, eg *errgroup.Group, req
return true, gcBehavior_Continue, nil
}
func (crg chunkReaderGroup) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
func (crg chunkReaderGroup) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
for _, haver := range crg {
remaining, gcb, err := haver.getManyCompressed(ctx, eg, reqs, found, keeper, stats)
if err != nil {

View File

@@ -15,6 +15,7 @@
package nbs
import (
"errors"
"os"
"path/filepath"
@@ -22,34 +23,38 @@ import (
"github.com/dolthub/dolt/go/store/hash"
)
type StorageType int
const (
Journal StorageType = iota
TableFileNewGen
TableFileOldGen
Archive
)
type ArchiveMetadata struct {
originalTableFileId string
}
type TableFileFormat int
const (
TypeNoms TableFileFormat = iota
TypeArchive
)
type StorageArtifact struct {
id hash.Hash
path string
storageType StorageType
// ID of the storage artifact. This is uses in the manifest to identify the artifact, but it is not the file name.
// as archives has a suffix.
id hash.Hash
// path to the storage artifact.
path string
// storageType is the type of the storage artifact.
storageType TableFileFormat
// arcMetadata is additional metadata for archive files. it is only set for storageType == TypeArchive.
arcMetadata *ArchiveMetadata
}
type StorageMetadata struct {
// root is the path to storage. Specifically, it contains a .dolt directory.
root string
artifacts []StorageArtifact
}
func (sm *StorageMetadata) ArchiveFilesPresent() bool {
for _, artifact := range sm.artifacts {
if artifact.storageType == Archive {
if artifact.storageType == TypeArchive {
return true
}
}
@@ -60,7 +65,7 @@ func (sm *StorageMetadata) ArchiveFilesPresent() bool {
func (sm *StorageMetadata) RevertMap() map[hash.Hash]hash.Hash {
revertMap := make(map[hash.Hash]hash.Hash)
for _, artifact := range sm.artifacts {
if artifact.storageType == Archive {
if artifact.storageType == TypeArchive {
md := artifact.arcMetadata
revertMap[artifact.id] = hash.Parse(md.originalTableFileId)
}
@@ -68,6 +73,8 @@ func (sm *StorageMetadata) RevertMap() map[hash.Hash]hash.Hash {
return revertMap
}
// oldGenTableExists returns true if the table file exists in the oldgen directory. This is a file system check for
// a table file we have no record of, but may be useful in the process of reverting an archive operation.
func (sm *StorageMetadata) oldGenTableExists(id hash.Hash) (bool, error) {
path := filepath.Join(sm.root, ".dolt", "noms", "oldgen", id.String())
_, err := os.Stat(path)
@@ -88,20 +95,9 @@ func GetStorageMetadata(path string) (StorageMetadata, error) {
return StorageMetadata{}, err
}
// TODO: new gen and journal information in storage metadata will be useful in the future.
// newGen := filepath.Join(path, ".dolt", "noms")
// newgenManifest := filepath.Join(newGen, "manifest")
oldgen := filepath.Join(path, ".dolt", "noms", "oldgen")
oldgenManifest := filepath.Join(oldgen, "manifest")
// If there is not oldgen manifest, then GC has never been run. Which is fine. We just don't have any oldgen.
if _, err := os.Stat(oldgenManifest); err != nil {
return StorageMetadata{}, nil
}
// create a io.Reader for the manifest file
manifestReader, err := os.Open(oldgenManifest)
newGen := filepath.Join(path, ".dolt", "noms")
newgenManifest := filepath.Join(newGen, "manifest")
manifestReader, err := os.Open(newgenManifest)
if err != nil {
return StorageMetadata{}, err
}
@@ -116,53 +112,90 @@ func GetStorageMetadata(path string) (StorageMetadata, error) {
// for each table in the manifest, get the table spec
for i := 0; i < manifest.NumTableSpecs(); i++ {
tableSpecInfo := manifest.GetTableSpecInfo(i)
// If the oldgen/name exists, it's not an archive. If it exists with a .darc suffix, then it's an archive.
tfName := tableSpecInfo.GetName()
fullPath := filepath.Join(oldgen, tfName)
_, err := os.Stat(fullPath)
if err == nil {
// exists. Not an archive.
artifacts = append(artifacts, StorageArtifact{
id: hash.Parse(tfName),
path: fullPath,
storageType: TableFileOldGen,
})
} else if os.IsNotExist(err) {
arcName := tfName + ".darc"
arcPath := filepath.Join(oldgen, arcName)
_, err := os.Stat(arcPath)
if err == nil {
// reader for the path. State. call
reader, fileSize, err := openReader(arcPath)
if err != nil {
return StorageMetadata{}, err
}
arcMetadata, err := newArchiveMetadata(reader, fileSize)
if err != nil {
return StorageMetadata{}, err
}
artifacts = append(artifacts, StorageArtifact{
id: hash.Parse(tfName),
path: arcPath,
storageType: Archive,
arcMetadata: arcMetadata,
})
} else {
// any error is bad here. If the files don't exist, then the manifest is no good.
return StorageMetadata{}, err
}
} else {
// some other error.
artifact, err := buildArtifact(tableSpecInfo, newGen)
if err != nil {
return StorageMetadata{}, err
}
artifacts = append(artifacts, artifact)
}
oldgen := filepath.Join(newGen, "oldgen")
oldgenManifest := filepath.Join(oldgen, "manifest")
// If there is no oldgen manifest, then GC has never been run. Which is fine. We just don't have any oldgen.
if _, err := os.Stat(oldgenManifest); err != nil {
return StorageMetadata{path, artifacts}, nil
}
manifestReader, err = os.Open(oldgenManifest)
if err != nil {
return StorageMetadata{}, err
}
manifest, err = ParseManifest(manifestReader)
if err != nil {
return StorageMetadata{}, err
}
for i := 0; i < manifest.NumTableSpecs(); i++ {
tableSpecInfo := manifest.GetTableSpecInfo(i)
artifact, err := buildArtifact(tableSpecInfo, oldgen)
if err != nil {
return StorageMetadata{}, err
}
artifacts = append(artifacts, artifact)
}
return StorageMetadata{path, artifacts}, nil
}
func buildArtifact(info TableSpecInfo, genPath string) (StorageArtifact, error) {
tfName := info.GetName()
// This code is going to be removed as soon as backup supports archives.
archive := false
fullPath := filepath.Join(genPath, tfName)
_, err := os.Stat(fullPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
fullPath = filepath.Join(genPath, tfName+ArchiveFileSuffix)
} else {
return StorageArtifact{}, err
}
_, err = os.Stat(fullPath)
if err != nil {
return StorageArtifact{}, err
}
archive = true
}
if !archive {
return StorageArtifact{
id: hash.Parse(tfName),
path: fullPath,
storageType: TypeNoms,
}, nil
} else {
reader, fileSize, err := openReader(fullPath)
if err != nil {
return StorageArtifact{}, err
}
arcMetadata, err := newArchiveMetadata(reader, fileSize)
if err != nil {
return StorageArtifact{}, err
}
return StorageArtifact{
id: hash.Parse(tfName),
path: fullPath,
storageType: TypeArchive,
arcMetadata: arcMetadata,
}, nil
}
}
func validateDir(path string) error {
info, err := os.Stat(path)

View File

@@ -99,7 +99,7 @@ func (nbsMW *NBSMetricWrapper) PruneTableFiles(ctx context.Context) error {
// GetManyCompressed gets the compressed Chunks with |hashes| from the store. On return,
// |found| will have been fully sent all chunks which have been
// found. Any non-present chunks will silently be ignored.
func (nbsMW *NBSMetricWrapper) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk)) error {
func (nbsMW *NBSMetricWrapper) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker)) error {
atomic.AddInt32(&nbsMW.TotalChunkGets, int32(len(hashes)))
return nbsMW.nbs.GetManyCompressed(ctx, hashes, found)
}

View File

@@ -67,7 +67,6 @@ const (
defaultMaxTables = 256
defaultManifestCacheSize = 1 << 23 // 8MB
preflushChunkCount = 8
)
var (
@@ -85,7 +84,7 @@ func makeGlobalCaches() {
type NBSCompressedChunkStore interface {
chunks.ChunkStore
GetManyCompressed(context.Context, hash.HashSet, func(context.Context, CompressedChunk)) error
GetManyCompressed(context.Context, hash.HashSet, func(context.Context, ToChunker)) error
}
type gcDependencyMode int
@@ -96,7 +95,7 @@ const (
)
type CompressedChunkStoreForGC interface {
getManyCompressed(context.Context, hash.HashSet, func(context.Context, CompressedChunk), gcDependencyMode) error
getManyCompressed(context.Context, hash.HashSet, func(context.Context, ToChunker), gcDependencyMode) error
}
type NomsBlockStore struct {
@@ -143,8 +142,10 @@ var _ chunks.ChunkStoreGarbageCollector = &NomsBlockStore{}
const hasCacheSize = 100000
type Range struct {
Offset uint64
Length uint32
Offset uint64
Length uint32
DictOffset uint64
DictLength uint32
}
// ChunkJournal returns the ChunkJournal in use by this NomsBlockStore, or nil if no ChunkJournal is being used.
@@ -156,19 +157,26 @@ func (nbs *NomsBlockStore) ChunkJournal() *ChunkJournal {
}
func (nbs *NomsBlockStore) GetChunkLocationsWithPaths(ctx context.Context, hashes hash.HashSet) (map[string]map[hash.Hash]Range, error) {
locs, err := nbs.GetChunkLocations(ctx, hashes)
sourcesToRanges, err := nbs.getChunkLocations(ctx, hashes)
if err != nil {
return nil, err
}
toret := make(map[string]map[hash.Hash]Range, len(locs))
for k, v := range locs {
toret[k.String()] = v
res := make(map[string]map[hash.Hash]Range, len(sourcesToRanges))
for csP, ranges := range sourcesToRanges {
cs := *csP
suffix := ""
if _, ok := cs.(archiveChunkSource); ok {
suffix = ArchiveFileSuffix
}
res[cs.hash().String()+suffix] = ranges
}
return toret, nil
return res, nil
}
func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.HashSet) (map[hash.Hash]map[hash.Hash]Range, error) {
fn := func(css chunkSourceSet, gr []getRecord, ranges map[hash.Hash]map[hash.Hash]Range, keeper keeperF) (gcBehavior, error) {
func (nbs *NomsBlockStore) getChunkLocations(ctx context.Context, hashes hash.HashSet) (map[*chunkSource]map[hash.Hash]Range, error) {
fn := func(css chunkSourceSet, gr []getRecord, ranges map[*chunkSource]map[hash.Hash]Range, keeper keeperF) (gcBehavior, error) {
for _, cs := range css {
rng, gcb, err := cs.getRecordRanges(ctx, gr, keeper)
if err != nil {
@@ -177,14 +185,16 @@ func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.Ha
if gcb != gcBehavior_Continue {
return gcb, nil
}
if len(rng) == 0 {
continue
}
h := hash.Hash(cs.hash())
if m, ok := ranges[h]; ok {
if m, ok := ranges[&cs]; ok {
for k, v := range rng {
m[k] = v
}
} else {
ranges[h] = rng
ranges[&cs] = rng
}
}
return gcBehavior_Continue, nil
@@ -196,7 +206,7 @@ func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.Ha
nbs.mu.Unlock()
gr := toGetRecords(hashes)
ranges := make(map[hash.Hash]map[hash.Hash]Range)
ranges := make(map[*chunkSource]map[hash.Hash]Range)
gcb, err := fn(tables.upstream, gr, ranges, keeper)
if needsContinue, err := nbs.handleUnlockedRead(ctx, gcb, endRead, err); err != nil {
@@ -214,6 +224,20 @@ func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.Ha
return ranges, nil
}
}
func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.HashSet) (map[hash.Hash]map[hash.Hash]Range, error) {
sourcesToRanges, err := nbs.getChunkLocations(ctx, hashes)
if err != nil {
return nil, err
}
res := make(map[hash.Hash]map[hash.Hash]Range, len(hashes))
for csP, ranges := range sourcesToRanges {
cs := *csP
res[cs.hash()] = ranges
}
return res, nil
}
func (nbs *NomsBlockStore) handleUnlockedRead(ctx context.Context, gcb gcBehavior, endRead func(), err error) (bool, error) {
@@ -901,11 +925,11 @@ func (nbs *NomsBlockStore) GetMany(ctx context.Context, hashes hash.HashSet, fou
)
}
func (nbs *NomsBlockStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk)) error {
func (nbs *NomsBlockStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker)) error {
return nbs.getManyCompressed(ctx, hashes, found, gcDependencyMode_TakeDependency)
}
func (nbs *NomsBlockStore) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk), gcDepMode gcDependencyMode) error {
func (nbs *NomsBlockStore) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker), gcDepMode gcDependencyMode) error {
ctx, span := tracer.Start(ctx, "nbs.GetManyCompressed", trace.WithAttributes(attribute.Int("num_hashes", len(hashes))))
defer span.End()
return nbs.getManyWithFunc(ctx, hashes, gcDepMode,
@@ -1482,8 +1506,9 @@ func (nbs *NomsBlockStore) StatsSummary() string {
// tableFile is our implementation of TableFile.
type tableFile struct {
info TableSpecInfo
open func(ctx context.Context) (io.ReadCloser, uint64, error)
info TableSpecInfo
open func(ctx context.Context) (io.ReadCloser, uint64, error)
suffix string
}
// LocationPrefix
@@ -1491,6 +1516,10 @@ func (tf tableFile) LocationPrefix() string {
return ""
}
func (tf tableFile) LocationSuffix() string {
return tf.suffix
}
// FileID gets the id of the file
func (tf tableFile) FileID() string {
return tf.info.GetName()
@@ -1555,12 +1584,18 @@ func getTableFiles(css map[hash.Hash]chunkSource, contents manifestContents, num
if !ok {
return nil, ErrSpecWithoutChunkSource
}
tableFiles = append(tableFiles, newTableFile(cs, info))
}
return tableFiles, nil
}
func newTableFile(cs chunkSource, info tableSpec) tableFile {
s := ""
if _, ok := cs.(archiveChunkSource); ok {
s = ArchiveFileSuffix
}
return tableFile{
info: info,
open: func(ctx context.Context) (io.ReadCloser, uint64, error) {
@@ -1570,6 +1605,7 @@ func newTableFile(cs chunkSource, info tableSpec) tableFile {
}
return r, s, nil
},
suffix: s,
}
}
@@ -1625,7 +1661,7 @@ func (nbs *NomsBlockStore) Path() (string, bool) {
}
// WriteTableFile will read a table file from the provided reader and write it to the TableFileStore
func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) error {
func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileName string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) error {
tfp, ok := nbs.p.(tableFilePersister)
if !ok {
return errors.New("Not implemented")
@@ -1636,7 +1672,7 @@ func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, nu
return err
}
defer r.Close()
return tfp.CopyTableFile(ctx, r, fileId, sz, uint32(numChunks))
return tfp.CopyTableFile(ctx, r, fileName, sz, uint32(numChunks))
}
// AddTableFilesToManifest adds table files to the manifest
@@ -1982,25 +2018,25 @@ func (i *markAndSweeper) SaveHashes(ctx context.Context, hashes []hash.Hash) err
found := 0
var addErr error
err = i.src.getManyCompressed(ctx, toVisit, func(ctx context.Context, cc CompressedChunk) {
err = i.src.getManyCompressed(ctx, toVisit, func(ctx context.Context, tc ToChunker) {
mu.Lock()
defer mu.Unlock()
if addErr != nil {
return
}
found += 1
if cc.IsGhost() {
if tc.IsGhost() {
// Ghost chunks encountered on the walk can be left alone --- they
// do not bring their dependencies, and because of how generational
// store works, they will still be ghost chunks
// in the store after the GC is finished.
return
}
addErr = i.gcc.addChunk(ctx, cc)
addErr = i.gcc.addChunk(ctx, tc)
if addErr != nil {
return
}
c, err := cc.ToChunk()
c, err := tc.ToChunk()
if err != nil {
addErr = err
return

View File

@@ -187,7 +187,7 @@ type extractRecord struct {
err error
}
// Returned by read methods that take a |keeperFunc|, this lets a
// Returned by read methods that take a |keeperF|, this lets a
// caller know whether the operation was successful or if it needs to
// be retried. It may need to be retried if a GC is in progress but
// the dependencies indicated by the operation cannot be added to the
@@ -202,6 +202,10 @@ const (
gcBehavior_Block = true
)
// keeperF is a function that takes a hash.Hash and returns true if the hash is used by the GC system to indicate
// that the chunk requested may not be present in the future, and therefore |gcBehavior_Block| should be returned. This
// is used to allow read/write ops to the store by non-GC processes while GC is underway. The |keeperF| may be nil,
// in which case GC is not underway. If it's non-nil, and return false, it's ok to proceed with the operation (|gcBehavior_Continue|)
type keeperF func(hash.Hash) bool
type chunkReader interface {
@@ -221,7 +225,7 @@ type chunkReader interface {
// getManyCompressed sets getRecord.found to true, and calls |found| for each present getRecord query.
// It returns true if any getRecord query was not found in this chunkReader.
getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error)
getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error)
// count returns the chunk count for this chunkReader.
count() (uint32, error)
@@ -239,6 +243,10 @@ type chunkSource interface {
// hash returns the hash address of this chunkSource.
hash() hash.Hash
// name is the on disk short name for this chunkSource. Classically, this was a hash. Having files
// with suffixes (eg darc) was useful.
name() string
// opens a Reader to the first byte of the chunkData segment of this table.
reader(context.Context) (io.ReadCloser, uint64, error)

View File

@@ -38,6 +38,13 @@ import (
// Do not read more than 128MB at a time.
const maxReadSize = 128 * 1024 * 1024
type ToChunker interface {
Hash() hash.Hash
ToChunk() (chunks.Chunk, error)
IsEmpty() bool
IsGhost() bool
}
// CompressedChunk represents a chunk of data in a table file which is still compressed via snappy.
type CompressedChunk struct {
// H is the hash of the chunk
@@ -53,6 +60,8 @@ type CompressedChunk struct {
ghost bool
}
var _ ToChunker = CompressedChunk{}
// NewCompressedChunk creates a CompressedChunk
func NewCompressedChunk(h hash.Hash, buff []byte) (CompressedChunk, error) {
dataLen := uint64(len(buff)) - checksumSize
@@ -323,10 +332,10 @@ var _ chunkReader = tableReader{}
func (tr tableReader) readCompressedAtOffsets(
ctx context.Context,
rb readBatch,
found func(context.Context, CompressedChunk),
found func(context.Context, ToChunker),
stats *Stats,
) error {
return tr.readAtOffsetsWithCB(ctx, rb, stats, func(ctx context.Context, cmp CompressedChunk) error {
return tr.readAtOffsetsWithCB(ctx, rb, stats, func(ctx context.Context, cmp ToChunker) error {
found(ctx, cmp)
return nil
})
@@ -338,7 +347,7 @@ func (tr tableReader) readAtOffsets(
found func(context.Context, *chunks.Chunk),
stats *Stats,
) error {
return tr.readAtOffsetsWithCB(ctx, rb, stats, func(ctx context.Context, cmp CompressedChunk) error {
return tr.readAtOffsetsWithCB(ctx, rb, stats, func(ctx context.Context, cmp ToChunker) error {
chk, err := cmp.ToChunk()
if err != nil {
@@ -354,7 +363,7 @@ func (tr tableReader) readAtOffsetsWithCB(
ctx context.Context,
rb readBatch,
stats *Stats,
cb func(ctx context.Context, cmp CompressedChunk) error,
cb func(ctx context.Context, cmp ToChunker) error,
) error {
readLength := rb.End() - rb.Start()
buff := make([]byte, readLength)
@@ -405,7 +414,7 @@ func (tr tableReader) getMany(
err = tr.getManyAtOffsets(ctx, eg, offsetRecords, found, stats)
return remaining, gcBehavior_Continue, err
}
func (tr tableReader) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
func (tr tableReader) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
// Pass #1: Iterate over |reqs| and |tr.prefixes| (both sorted by address) and build the set
// of table locations which must be read in order to satisfy the getMany operation.
offsetRecords, remaining, gcb, err := tr.findOffsets(reqs, keeper)
@@ -419,7 +428,7 @@ func (tr tableReader) getManyCompressed(ctx context.Context, eg *errgroup.Group,
return remaining, gcBehavior_Continue, err
}
func (tr tableReader) getManyCompressedAtOffsets(ctx context.Context, eg *errgroup.Group, offsetRecords offsetRecSlice, found func(context.Context, CompressedChunk), stats *Stats) error {
func (tr tableReader) getManyCompressedAtOffsets(ctx context.Context, eg *errgroup.Group, offsetRecords offsetRecSlice, found func(context.Context, ToChunker), stats *Stats) error {
return tr.getManyAtOffsetsWithReadFunc(ctx, eg, offsetRecords, stats, func(
ctx context.Context,
rb readBatch,

View File

@@ -228,7 +228,7 @@ func (ts tableSet) getMany(ctx context.Context, eg *errgroup.Group, reqs []getRe
return f(ts.upstream)
}
func (ts tableSet) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
func (ts tableSet) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) {
f := func(css chunkSourceSet) (bool, gcBehavior, error) {
for _, haver := range css {
remaining, gcb, err := haver.getManyCompressed(ctx, eg, reqs, found, keeper, stats)
@@ -544,7 +544,7 @@ func (ts tableSet) rebase(ctx context.Context, specs []tableSpec, srcs chunkSour
} else if existing, ok := srcs[spec.name]; ok {
cs, err = existing.clone()
} else {
cs, err = ts.p.Open(ctx, spec.name, spec.chunkCount, stats) // NM4 - spec.name is the tf/arch name.
cs, err = ts.p.Open(ctx, spec.name, spec.chunkCount, stats)
}
if err != nil {
return err

View File

@@ -0,0 +1 @@
5:__DOLT__:6eseiohaelofp485e95ti1pgqe5qrseg:att0kn4lt0uqce5mdqf01ni9ediahoo4:6eseiohaelofp485e95ti1pgqe5qrseg:8p5e2m6skovfdjlh4jg3llr8sfvu384l:2

View File

@@ -0,0 +1 @@
5:__DOLT__:c3qnrebcud1nuibb18lmitlc442ql6me:00000000000000000000000000000000:c3qnrebcud1nuibb18lmitlc442ql6me:29o8a3uevcpr15tilcemb3s438edmoog:125:dnu4lr5j8sstbj5usbld7alsnuj5nf23:139

View File

@@ -0,0 +1,6 @@
{
"head": "refs/heads/main",
"remotes": {},
"backups": {},
"branches": {}
}

View File

@@ -11,6 +11,12 @@ setup() {
}
teardown() {
if [ -n "$remotesrv_pid" ]; then
kill "$remotesrv_pid"
wait "$remotesrv_pid" || :
remotesrv_pid=""
fi
assert_feature_version
teardown_common
}
@@ -109,19 +115,6 @@ mutations_and_gc_statement() {
[ "$files" -eq "2" ]
}
@test "archive: archive with remotesrv no go" {
dolt sql -q "$(mutations_and_gc_statement)"
dolt archive
run dolt sql-server --remotesapi-port=12321
[ "$status" -eq 1 ]
[[ "$output" =~ "archive files present" ]] || false
run remotesrv --repo-mode
[ "$status" -eq 1 ]
[[ "$output" =~ "archive files present" ]] || false
}
@test "archive: archive --revert (fast)" {
dolt sql -q "$(mutations_and_gc_statement)"
dolt archive
@@ -143,19 +136,136 @@ mutations_and_gc_statement() {
[ "$commits" -eq "66" ]
}
@test "archive: archive backup no go" {
dolt sql -q "$(mutations_and_gc_statement)"
dolt archive
@test "archive: can clone archived repository" {
mkdir -p remote/.dolt
mkdir cloned
dolt backup add bac1 file://../bac1
run dolt backup sync bac1
# Copy the archive test repo to remote directory
cp -R $BATS_TEST_DIRNAME/archive-test-repo/* remote/.dolt
cd remote
[ "$status" -eq 1 ]
[[ "$output" =~ "archive files present" ]] || false
port=$( definePORT )
# currently the cli and stored procedures are different code paths.
run dolt sql -q "call dolt_backup('sync', 'bac1')"
[ "$status" -eq 1 ]
# NM4 - TODO. This message is cryptic, but plumbing the error through is awkward.
[[ "$output" =~ "Archive chunk source" ]] || false
remotesrv --http-port $port --grpc-port $port --repo-mode &
remotesrv_pid=$!
[[ "$remotesrv_pid" -gt 0 ]] || false
cd ../cloned
run dolt clone http://localhost:$port/test-org/test-repo repo1
[ "$status" -eq 0 ]
cd repo1
# Verify we can read data
run dolt sql -q 'select sum(i) from tbl;'
[[ "$status" -eq 0 ]] || false
[[ "$output" =~ "138075" ]] || false # i = 1 - 525, sum is 138075
kill $remotesrv_pid
wait $remotesrv_pid || :
remotesrv_pid=""
## The above test is the setup for the next test - so we'll stick both in here.
## This tests cloning from a clone. Archive files are generally in oldgen, but not the case with a fresh clone.
cd ../../
mkdir clone2
cd cloned/repo1 # start the server using the clone from above.
port=$( definePORT )
remotesrv --http-port $port --grpc-port $port --repo-mode &
remotesrv_pid=$!
[[ "$remotesrv_pid" -gt 0 ]] || false
cd ../../clone2
run dolt clone http://localhost:$port/test-org/test-repo repo2
[ "$status" -eq 0 ]
cd repo2
run dolt sql -q 'select sum(i) from tbl;'
[[ "$status" -eq 0 ]] || false
[[ "$output" =~ "138075" ]] || false # i = 1 - 525, sum is 138075
## Temporary check. We want to ensure that backup will give an error, even when
## there are archives in newgen.
mkdir ../backup
dolt backup add bac1 file://../backup
run dolt backup sync bac1
[ "$status" -eq 1 ]
[[ "$output" =~ "error: archive files present" ]] || false
}
@test "archive: can clone respiratory with mixed types" {
mkdir -p remote/.dolt
mkdir cloned
# Copy the archive test repo to remote directory
cp -R $BATS_TEST_DIRNAME/archive-test-repo/* remote/.dolt
cd remote
# Insert data (commits automatically), but don't gc/archive yet. Want to make sure we can still clone it.
dolt sql -q "$(insert_statement)"
port=$( definePORT )
remotesrv --http-port $port --grpc-port $port --repo-mode &
remotesrv_pid=$!
[[ "$remotesrv_pid" -gt 0 ]] || false
cd ../cloned
run dolt clone http://localhost:$port/test-org/test-repo repo1
[ "$status" -eq 0 ]
cd repo1
# verify new data is there.
run dolt sql -q 'select sum(i) from tbl;'
[[ "$status" -eq 0 ]] || false
[[ "$output" =~ "151525" ]] || false # i = 1 - 550, sum is 151525
}
@test "archive: can fetch chunks from an archived repo" {
mkdir -p remote/.dolt
mkdir cloned
# Copy the archive test repo to remote directory
cp -R $BATS_TEST_DIRNAME/archive-test-repo/* remote/.dolt
cd remote
port=$( definePORT )
remotesrv --http-port $port --grpc-port $port --repo-mode &
remotesrv_pid=$!
[[ "$remotesrv_pid" -gt 0 ]] || false
cd ../cloned
dolt clone http://localhost:$port/test-org/test-repo repo1
# Fetch when there are no changes.
cd repo1
dolt fetch
## update the remote repo directly. Need to run the archive command when the server is stopped.
## This will result in archived files on the remote, which we will need to read chunks from when we fetch.
cd ../../remote
kill $remotesrv_pid
wait $remotesrv_pid || :
remotesrv_pid=""
dolt sql -q "$(mutations_and_gc_statement)"
dolt archive
remotesrv --http-port $port --grpc-port $port --repo-mode &
remotesrv_pid=$!
[[ "$remotesrv_pid" -gt 0 ]] || false
cd ../cloned/repo1
dolt fetch
run dolt status
[ "$status" -eq 0 ]
[[ "$output" =~ "Your branch is behind 'origin/main' by 20 commits, and can be fast-forwarded" ]] || false
# Verify the repo has integrity.
dolt fsck
}

View File

@@ -89,6 +89,13 @@ message RangeChunk {
bytes hash = 1;
uint64 offset = 2;
uint32 length = 3;
// Archive zStd dictionary spans are indicated with these fields. On the consumer side, the existence
// of a non-zero dictionary length indicates that the chunk is compressed using zStd. This implies that
// the chunkSource is an archive source. If the dictionary length is zero, the chunk is compressed using snappy
// compression (classic NOMS table file or journal chunk use snappy compression).
uint64 dictionary_offset = 4;
uint32 dictionary_length = 5;
}
message HttpGetRange {