diff --git a/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go b/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go index 4fb6fcd3d9..1f2b6e2477 100644 --- a/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go +++ b/go/gen/proto/dolt/services/remotesapi/v1alpha1/chunkstore.pb.go @@ -402,9 +402,11 @@ type RangeChunk struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` - Offset uint64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` - Length uint32 `protobuf:"varint,3,opt,name=length,proto3" json:"length,omitempty"` + Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` + Offset uint64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` + Length uint32 `protobuf:"varint,3,opt,name=length,proto3" json:"length,omitempty"` + DictionaryOffset uint64 `protobuf:"varint,4,opt,name=dictionary_offset,json=dictionaryOffset,proto3" json:"dictionary_offset,omitempty"` + DictionaryLength uint32 `protobuf:"varint,5,opt,name=dictionary_length,json=dictionaryLength,proto3" json:"dictionary_length,omitempty"` } func (x *RangeChunk) Reset() { @@ -460,6 +462,20 @@ func (x *RangeChunk) GetLength() uint32 { return 0 } +func (x *RangeChunk) GetDictionaryOffset() uint64 { + if x != nil { + return x.DictionaryOffset + } + return 0 +} + +func (x *RangeChunk) GetDictionaryLength() uint32 { + if x != nil { + return x.DictionaryLength + } + return 0 +} + type HttpGetRange struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2233,418 +2249,424 @@ var file_dolt_services_remotesapi_v1alpha1_chunkstore_proto_rawDesc = []byte{ 0x74, 0x70, 0x47, 0x65, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x68, 0x61, - 0x73, 0x68, 0x65, 0x73, 0x22, 0x50, 0x0a, 0x0a, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x43, 0x68, 0x75, - 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0x67, 0x0a, 0x0c, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, - 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x45, 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x67, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, - 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x61, 0x6e, - 0x67, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, - 0xe9, 0x02, 0x0a, 0x0b, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x12, - 0x4c, 0x0a, 0x08, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, - 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x43, 0x68, 0x75, - 0x6e, 0x6b, 0x48, 0x00, 0x52, 0x07, 0x68, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x12, 0x57, 0x0a, - 0x0e, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x67, 0x65, 0x74, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, - 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x48, 0x00, 0x52, 0x0c, 0x68, 0x74, 0x74, 0x70, 0x47, 0x65, - 0x74, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, - 0x68, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, - 0x73, 0x68, 0x41, 0x66, 0x74, 0x65, 0x72, 0x12, 0x66, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65, - 0x73, 0x68, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x3d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x73, 0x68, 0x65, 0x73, 0x22, 0xaa, 0x01, 0x0a, 0x0a, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x43, 0x68, + 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, + 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x2b, 0x0a, 0x11, 0x64, 0x69, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x10, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x4f, 0x66, + 0x66, 0x73, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, + 0x72, 0x79, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x10, 0x64, 0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x72, 0x79, 0x4c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x22, 0x67, 0x0a, 0x0c, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x52, 0x61, 0x6e, 0x67, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x75, 0x72, 0x6c, 0x12, 0x45, 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x43, 0x68, 0x75, + 0x6e, 0x6b, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0xe9, 0x02, 0x0a, 0x0b, 0x44, + 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x12, 0x4c, 0x0a, 0x08, 0x68, 0x74, + 0x74, 0x70, 0x5f, 0x67, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x64, + 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2e, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x48, 0x00, 0x52, + 0x07, 0x68, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x12, 0x57, 0x0a, 0x0e, 0x68, 0x74, 0x74, 0x70, + 0x5f, 0x67, 0x65, 0x74, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, - 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, - 0x0e, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x42, - 0x0a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x25, 0x0a, 0x11, 0x48, - 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, - 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, - 0x72, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x09, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, - 0x12, 0x26, 0x0a, 0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x68, - 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x12, 0x53, 0x0a, 0x09, 0x68, 0x74, 0x74, 0x70, - 0x5f, 0x70, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x64, 0x6f, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x52, 0x61, 0x6e, 0x67, + 0x65, 0x48, 0x00, 0x52, 0x0c, 0x68, 0x74, 0x74, 0x70, 0x47, 0x65, 0x74, 0x52, 0x61, 0x6e, 0x67, + 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x61, 0x66, 0x74, + 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x41, 0x66, 0x74, + 0x65, 0x72, 0x12, 0x66, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x48, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, - 0x65, 0x48, 0x00, 0x52, 0x08, 0x68, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x42, 0x0a, 0x0a, - 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbb, 0x01, 0x0a, 0x16, 0x47, 0x65, - 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, - 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x68, 0x75, 0x6e, - 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0b, - 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, - 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, - 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, - 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x7c, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x44, 0x6f, - 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x42, 0x0a, 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x2e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, - 0x52, 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, - 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8b, 0x01, 0x0a, 0x10, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, - 0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x69, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4c, 0x65, 0x6e, 0x67, 0x74, - 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x68, 0x61, 0x73, - 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x48, 0x61, 0x73, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x68, 0x75, 0x6e, - 0x6b, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x43, 0x68, 0x75, - 0x6e, 0x6b, 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, - 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, - 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, - 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, - 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, - 0x12, 0x2e, 0x0a, 0x11, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x68, - 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, - 0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, - 0x12, 0x61, 0x0a, 0x12, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, - 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x52, 0x10, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61, - 0x69, 0x6c, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, - 0x78, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x04, 0x6c, 0x6f, 0x63, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, + 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, + 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x72, + 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x42, 0x0a, 0x0a, 0x08, 0x6c, 0x6f, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x25, 0x0a, 0x11, 0x48, 0x74, 0x74, 0x70, 0x50, 0x6f, + 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, + 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0x94, 0x01, + 0x0a, 0x09, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x12, 0x26, 0x0a, 0x0f, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, + 0x61, 0x73, 0x68, 0x12, 0x53, 0x0a, 0x09, 0x68, 0x74, 0x74, 0x70, 0x5f, 0x70, 0x6f, 0x73, 0x74, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, - 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, - 0x64, 0x4c, 0x6f, 0x63, 0x52, 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, + 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x50, + 0x6f, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x00, 0x52, 0x08, + 0x68, 0x74, 0x74, 0x70, 0x50, 0x6f, 0x73, 0x74, 0x42, 0x0a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbb, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, + 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, + 0x6f, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x68, 0x61, 0x73, + 0x68, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x68, 0x75, 0x6e, 0x6b, + 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, + 0x74, 0x68, 0x22, 0x7c, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, + 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, + 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x64, 0x6f, + 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, + 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x52, 0x04, 0x6c, 0x6f, 0x63, + 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x22, 0x8b, 0x01, 0x0a, 0x10, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, + 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x02, 0x69, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x21, 0x0a, 0x0c, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x48, 0x61, 0x73, 0x68, 0x12, + 0x1d, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x22, 0xa9, + 0x02, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, + 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, + 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x2e, 0x0a, 0x11, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0f, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x61, 0x0a, 0x12, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, + 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x10, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x1d, + 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, + 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x78, 0x0a, 0x15, 0x47, 0x65, + 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x2c, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x52, + 0x04, 0x6c, 0x6f, 0x63, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8f, 0x01, 0x0a, 0x0d, 0x52, 0x65, 0x62, 0x61, 0x73, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, + 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, + 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8f, 0x01, 0x0a, 0x0d, 0x52, 0x65, - 0x62, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, - 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, - 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, - 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, - 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x2f, 0x0a, 0x0e, 0x52, - 0x65, 0x62, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, - 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8d, 0x01, 0x0a, - 0x0b, 0x52, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, + 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, + 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, + 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x2f, 0x0a, 0x0e, 0x52, 0x65, 0x62, 0x61, 0x73, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, + 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, + 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x8d, 0x01, 0x0a, 0x0b, 0x52, 0x6f, 0x6f, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, + 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, + 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, + 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, + 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, + 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x4a, 0x0a, 0x0c, 0x52, 0x6f, 0x6f, 0x74, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x6f, 0x6f, 0x74, 0x5f, + 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x6f, 0x6f, 0x74, + 0x48, 0x61, 0x73, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x22, 0x45, 0x0a, 0x0e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, + 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xde, 0x02, 0x0a, 0x0d, 0x43, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, - 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, - 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x4a, 0x0a, 0x0c, - 0x52, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, - 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x08, 0x72, 0x6f, 0x6f, 0x74, 0x48, 0x61, 0x73, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, - 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, - 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x45, 0x0a, 0x0e, 0x43, 0x68, 0x75, 0x6e, - 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, - 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x1f, - 0x0a, 0x0b, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, - 0xde, 0x02, 0x0a, 0x0d, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x12, 0x18, 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, + 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6c, 0x61, 0x73, 0x74, 0x12, 0x5b, + 0x0a, 0x10, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x6e, + 0x66, 0x6f, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, + 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x68, 0x75, + 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0e, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x61, 0x0a, 0x12, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, + 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x10, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1b, + 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x2a, 0x0a, 0x0e, 0x43, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, + 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, + 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0xfb, 0x01, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x52, + 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, + 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x61, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x0e, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, - 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6c, - 0x61, 0x73, 0x74, 0x12, 0x5b, 0x0a, 0x10, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, - 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, - 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x0e, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x12, 0x61, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, - 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, - 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, - 0x22, 0x2a, 0x0a, 0x0e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0xfb, 0x01, 0x0a, - 0x16, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, + 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, + 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, + 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, + 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, + 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x92, 0x02, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, + 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x66, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x73, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x73, + 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x73, 0x0a, 0x18, 0x70, 0x75, 0x73, 0x68, 0x5f, 0x63, 0x6f, + 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, + 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x75, 0x73, 0x68, + 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x52, 0x16, 0x70, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, + 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x22, 0x54, 0x0a, 0x10, 0x43, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1f, + 0x0a, 0x0b, 0x6e, 0x62, 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x66, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x22, 0xc0, 0x01, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, + 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, + 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, + 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, + 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x27, + 0x0a, 0x0d, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f, 0x6f, 0x6e, 0x6c, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0c, 0x61, 0x70, 0x70, 0x65, 0x6e, + 0x64, 0x69, 0x78, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, + 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, + 0x61, 0x74, 0x68, 0x22, 0x82, 0x02, 0x0a, 0x0d, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, + 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d, + 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x10, 0x0a, + 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, + 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x41, 0x66, 0x74, 0x65, 0x72, + 0x12, 0x66, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, + 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, + 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb5, 0x01, 0x0a, 0x1a, 0x52, 0x65, 0x66, + 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, + 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, + 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, + 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, + 0x22, 0x8f, 0x01, 0x0a, 0x1b, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, + 0x72, 0x6c, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, 0x61, 0x66, + 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x41, 0x66, + 0x74, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x22, 0x99, 0x02, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, + 0x09, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x08, 0x72, 0x6f, 0x6f, 0x74, 0x48, 0x61, 0x73, 0x68, 0x12, 0x58, 0x0a, 0x0f, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, + 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, + 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x69, 0x0a, 0x18, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, + 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, + 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x15, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, + 0x69, 0x78, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0xba, + 0x03, 0x0a, 0x14, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x12, 0x61, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x10, 0x63, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1d, + 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x5b, + 0x0a, 0x10, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x69, 0x6e, + 0x66, 0x6f, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, + 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x68, 0x75, + 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0e, 0x63, 0x68, 0x75, + 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x62, 0x0a, 0x0f, 0x61, + 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, + 0x74, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x0e, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, + 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x50, 0x0a, 0x15, 0x41, + 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, - 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x92, 0x02, 0x0a, 0x17, 0x47, - 0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x66, 0x5f, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x66, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x73, 0x5f, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, - 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, - 0x61, 0x67, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, - 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x72, - 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x73, 0x0a, 0x18, 0x70, 0x75, - 0x73, 0x68, 0x5f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x64, - 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x50, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, - 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x52, 0x16, 0x70, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e, - 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x22, - 0x54, 0x0a, 0x10, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x66, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x62, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6e, 0x62, 0x73, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xc0, 0x01, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x42, 0x0a, 0x07, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x29, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, - 0x6f, 0x49, 0x64, 0x12, 0x27, 0x0a, 0x0d, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f, - 0x6f, 0x6e, 0x6c, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0c, - 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x1d, 0x0a, 0x0a, - 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, - 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x82, 0x02, 0x0a, 0x0d, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x69, - 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, - 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x68, 0x75, 0x6e, 0x6b, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x43, 0x68, 0x75, 0x6e, - 0x6b, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x75, 0x72, 0x6c, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x5f, - 0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, - 0x41, 0x66, 0x74, 0x65, 0x72, 0x12, 0x66, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, - 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, - 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, - 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, - 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0e, 0x72, - 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xb5, 0x01, - 0x0a, 0x1a, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, - 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, - 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, - 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, - 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, - 0x12, 0x17, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, - 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, - 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, - 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, - 0x6f, 0x50, 0x61, 0x74, 0x68, 0x22, 0x8f, 0x01, 0x0a, 0x1b, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, - 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x3f, 0x0a, 0x0d, 0x72, 0x65, 0x66, 0x72, 0x65, - 0x73, 0x68, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x72, 0x65, 0x66, 0x72, - 0x65, 0x73, 0x68, 0x41, 0x66, 0x74, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, - 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, - 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x99, 0x02, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x6f, 0x6f, 0x74, 0x48, 0x61, 0x73, 0x68, 0x12, - 0x58, 0x0a, 0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x69, 0x6e, - 0x66, 0x6f, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, - 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0d, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x69, 0x0a, 0x18, 0x61, 0x70, 0x70, - 0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, - 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x64, 0x6f, - 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x15, 0x61, - 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x22, 0xba, 0x03, 0x0a, 0x14, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, - 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x07, - 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, - 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, - 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x49, 0x64, 0x52, 0x06, 0x72, 0x65, 0x70, 0x6f, 0x49, 0x64, - 0x12, 0x61, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x5f, - 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x64, - 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x52, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x46, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x12, 0x5b, 0x0a, 0x10, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, - 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, - 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x0e, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x12, 0x62, 0x0a, 0x0f, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x5f, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4d, 0x61, - 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, 0x70, 0x6f, 0x50, 0x61, 0x74, 0x68, - 0x22, 0x50, 0x0a, 0x15, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x5f, 0x74, 0x6f, 0x6b, 0x65, - 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x2a, 0xa4, 0x01, 0x0a, 0x16, 0x50, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x63, 0x75, - 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x28, 0x0a, - 0x24, 0x50, 0x55, 0x53, 0x48, 0x5f, 0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, - 0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, - 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x2f, 0x0a, 0x2b, 0x50, 0x55, 0x53, 0x48, 0x5f, - 0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54, - 0x52, 0x4f, 0x4c, 0x5f, 0x49, 0x47, 0x4e, 0x4f, 0x52, 0x45, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x49, - 0x4e, 0x47, 0x5f, 0x53, 0x45, 0x54, 0x10, 0x01, 0x12, 0x2f, 0x0a, 0x2b, 0x50, 0x55, 0x53, 0x48, + 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x70, 0x6f, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x2a, 0xa4, 0x01, + 0x0a, 0x16, 0x50, 0x75, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, + 0x79, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x28, 0x0a, 0x24, 0x50, 0x55, 0x53, 0x48, 0x5f, 0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x43, 0x4f, 0x4e, - 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, 0x5f, 0x57, 0x4f, 0x52, 0x4b, - 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x45, 0x54, 0x10, 0x02, 0x2a, 0x89, 0x01, 0x0a, 0x16, 0x4d, 0x61, - 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x24, 0x4d, 0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54, - 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x58, 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, - 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x20, - 0x0a, 0x1c, 0x4d, 0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, - 0x44, 0x49, 0x58, 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x45, 0x54, 0x10, 0x01, - 0x12, 0x23, 0x0a, 0x1f, 0x4d, 0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50, - 0x45, 0x4e, 0x44, 0x49, 0x58, 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x50, 0x50, - 0x45, 0x4e, 0x44, 0x10, 0x02, 0x32, 0xb2, 0x0b, 0x0a, 0x11, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x53, - 0x74, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x88, 0x01, 0x0a, 0x0f, - 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, - 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x64, 0x6f, 0x6c, + 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x2f, 0x0a, 0x2b, 0x50, 0x55, 0x53, 0x48, 0x5f, 0x43, 0x4f, 0x4e, 0x43, 0x55, + 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f, 0x49, + 0x47, 0x4e, 0x4f, 0x52, 0x45, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x49, 0x4e, 0x47, 0x5f, 0x53, 0x45, + 0x54, 0x10, 0x01, 0x12, 0x2f, 0x0a, 0x2b, 0x50, 0x55, 0x53, 0x48, 0x5f, 0x43, 0x4f, 0x4e, 0x43, + 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x52, 0x4f, 0x4c, 0x5f, + 0x41, 0x53, 0x53, 0x45, 0x52, 0x54, 0x5f, 0x57, 0x4f, 0x52, 0x4b, 0x49, 0x4e, 0x47, 0x5f, 0x53, + 0x45, 0x54, 0x10, 0x02, 0x2a, 0x89, 0x01, 0x0a, 0x16, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, + 0x74, 0x41, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x78, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x28, 0x0a, 0x24, 0x4d, 0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50, 0x45, + 0x4e, 0x44, 0x49, 0x58, 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x20, 0x0a, 0x1c, 0x4d, 0x41, 0x4e, + 0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x58, 0x5f, 0x4f, + 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x45, 0x54, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x4d, + 0x41, 0x4e, 0x49, 0x46, 0x45, 0x53, 0x54, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x58, + 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x02, + 0x32, 0xb2, 0x0b, 0x0a, 0x11, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x88, 0x01, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x52, 0x65, + 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x76, 0x0a, 0x09, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75, - 0x6e, 0x6b, 0x73, 0x12, 0x33, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, - 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x61, 0x73, - 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8d, - 0x01, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, - 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, - 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x94, - 0x01, 0x0a, 0x17, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, - 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c, - 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, - 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, + 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x76, 0x0a, 0x09, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x33, + 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x48, 0x61, 0x73, 0x43, 0x68, 0x75, 0x6e, 0x6b, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8d, 0x01, 0x0a, 0x14, 0x47, 0x65, + 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, + 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3a, 0x2e, + 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x94, 0x01, 0x0a, 0x17, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, - 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x87, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, - 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x37, 0x2e, 0x64, - 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, - 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x6d, 0x0a, 0x06, 0x52, 0x65, 0x62, 0x61, 0x73, 0x65, 0x12, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, - 0x62, 0x61, 0x73, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x64, 0x6f, - 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x52, 0x65, 0x62, 0x61, 0x73, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x67, - 0x0a, 0x04, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x2e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, - 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x6f, 0x6f, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, - 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x6f, 0x6f, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6d, 0x0a, 0x06, 0x43, 0x6f, 0x6d, 0x6d, 0x69, - 0x74, 0x12, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, - 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x85, 0x01, 0x0a, 0x0e, 0x4c, 0x69, 0x73, 0x74, 0x54, - 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x38, 0x2e, 0x64, 0x6f, 0x6c, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x39, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, - 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x94, - 0x01, 0x0a, 0x13, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, - 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x3d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, - 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, - 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, - 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x82, 0x01, 0x0a, 0x0d, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x37, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, + 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x3a, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, + 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, + 0x12, 0x87, 0x01, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x37, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, - 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x54, - 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55, + 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x53, 0x5a, 0x51, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x6f, 0x6c, 0x74, 0x68, 0x75, 0x62, - 0x2f, 0x64, 0x6f, 0x6c, 0x74, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2f, 0x64, 0x6f, 0x6c, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x3b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, + 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6d, 0x0a, 0x06, 0x52, 0x65, + 0x62, 0x61, 0x73, 0x65, 0x12, 0x30, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x62, 0x61, 0x73, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, + 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x62, 0x61, 0x73, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x67, 0x0a, 0x04, 0x52, 0x6f, 0x6f, + 0x74, 0x12, 0x2e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x2f, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x6f, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x6d, 0x0a, 0x06, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x30, 0x2e, 0x64, + 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, + 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x85, 0x01, 0x0a, 0x0e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, + 0x69, 0x6c, 0x65, 0x73, 0x12, 0x38, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x39, + 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x94, 0x01, 0x0a, 0x13, 0x52, 0x65, + 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, + 0x6c, 0x12, 0x3d, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x3e, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x82, 0x01, 0x0a, 0x0d, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, + 0x65, 0x73, 0x12, 0x37, 0x2e, 0x64, 0x6f, 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, + 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x38, 0x2e, 0x64, 0x6f, + 0x6c, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2e, 0x72, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, + 0x41, 0x64, 0x64, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x53, 0x5a, 0x51, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x6f, 0x6c, 0x74, 0x68, 0x75, 0x62, 0x2f, 0x64, 0x6f, 0x6c, 0x74, + 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x64, 0x6f, + 0x6c, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x72, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3b, + 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x73, 0x61, 0x70, 0x69, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/go/go.mod b/go/go.mod index c9f89bfbc1..94a6dc0d10 100644 --- a/go/go.mod +++ b/go/go.mod @@ -56,7 +56,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/creasty/defaults v1.6.0 github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 - github.com/dolthub/go-mysql-server v0.19.1-0.20250207201905-b3a4c87c4fdc + github.com/dolthub/go-mysql-server v0.19.1-0.20250214204118-1e0e5e8f244a github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 github.com/dolthub/swiss v0.1.0 github.com/esote/minmaxheap v1.0.0 diff --git a/go/go.sum b/go/go.sum index 5038e3a6cf..8c3b08e2c1 100644 --- a/go/go.sum +++ b/go/go.sum @@ -179,8 +179,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U= github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0= github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90 h1:Sni8jrP0sy/w9ZYXoff4g/ixe+7bFCZlfCqXKJSU+zM= github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA= -github.com/dolthub/go-mysql-server v0.19.1-0.20250207201905-b3a4c87c4fdc h1:SdN7GRPtaqmLwfi6cVcyF4Oc8FbFUJ+mwsFRV++6iH4= -github.com/dolthub/go-mysql-server v0.19.1-0.20250207201905-b3a4c87c4fdc/go.mod h1:QQxZvPHOtycbC2bVmqmT6/Fov2g1/T1Rtm76wLd/Y1E= +github.com/dolthub/go-mysql-server v0.19.1-0.20250214204118-1e0e5e8f244a h1:3aeB6IsvQKOTxSRDeDuBjvOkHBolqYrhLUXa0UmgWjc= +github.com/dolthub/go-mysql-server v0.19.1-0.20250214204118-1e0e5e8f244a/go.mod h1:QQxZvPHOtycbC2bVmqmT6/Fov2g1/T1Rtm76wLd/Y1E= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI= github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q= github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE= diff --git a/go/libraries/doltcore/remotesrv/grpc.go b/go/libraries/doltcore/remotesrv/grpc.go index c675fa099b..b3090e587c 100644 --- a/go/libraries/doltcore/remotesrv/grpc.go +++ b/go/libraries/doltcore/remotesrv/grpc.go @@ -295,7 +295,12 @@ func (rs *RemoteChunkStore) StreamDownloadLocations(stream remotesapi.ChunkStore var ranges []*remotesapi.RangeChunk for h, r := range hashToRange { hCpy := h - ranges = append(ranges, &remotesapi.RangeChunk{Hash: hCpy[:], Offset: r.Offset, Length: r.Length}) + ranges = append(ranges, &remotesapi.RangeChunk{ + Hash: hCpy[:], + Offset: r.Offset, + Length: r.Length, + DictionaryOffset: r.DictOffset, + DictionaryLength: r.DictLength}) } url := rs.getDownloadUrl(md, prefix+"/"+loc) @@ -606,7 +611,7 @@ func getTableFileInfo( } appendixTableFileInfo := make([]*remotesapi.TableFileInfo, 0) for _, t := range tableList { - url := rs.getDownloadUrl(md, prefix+"/"+t.LocationPrefix()+t.FileID()) + url := rs.getDownloadUrl(md, prefix+"/"+t.LocationPrefix()+t.FileID()+t.LocationSuffix()) url, err = rs.sealer.Seal(url) if err != nil { return nil, status.Error(codes.Internal, "failed to get seal download url for "+t.FileID()) diff --git a/go/libraries/doltcore/remotesrv/http.go b/go/libraries/doltcore/remotesrv/http.go index eedbf92614..825f4ca0df 100644 --- a/go/libraries/doltcore/remotesrv/http.go +++ b/go/libraries/doltcore/remotesrv/http.go @@ -35,6 +35,7 @@ import ( "github.com/dolthub/dolt/go/libraries/utils/filesys" "github.com/dolthub/dolt/go/store/hash" + "github.com/dolthub/dolt/go/store/nbs" "github.com/dolthub/dolt/go/store/types" ) @@ -94,12 +95,18 @@ func (fh filehandler) ServeHTTP(respWr http.ResponseWriter, req *http.Request) { respWr.WriteHeader(http.StatusBadRequest) return } - _, ok := hash.MaybeParse(path[i+1:]) + + fileName := path[i+1:] + if strings.HasSuffix(fileName, nbs.ArchiveFileSuffix) { + fileName = fileName[:len(fileName)-len(nbs.ArchiveFileSuffix)] + } + _, ok := hash.MaybeParse(fileName) if !ok { - logger.WithField("last_path_component", path[i+1:]).Warn("bad request with unparseable last path component") + logger.WithField("last_path_component", fileName).Warn("bad request with unparseable last path component") respWr.WriteHeader(http.StatusBadRequest) return } + abs, err := fh.fs.Abs(path) if err != nil { logger.WithError(err).Error("could not get absolute path") diff --git a/go/libraries/doltcore/remotesrv/server.go b/go/libraries/doltcore/remotesrv/server.go index 46b7ec5182..6b451a6310 100644 --- a/go/libraries/doltcore/remotesrv/server.go +++ b/go/libraries/doltcore/remotesrv/server.go @@ -17,7 +17,6 @@ package remotesrv import ( "context" "crypto/tls" - "errors" "net" "net/http" "strings" @@ -29,7 +28,6 @@ import ( "google.golang.org/grpc" remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1" - "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/utils/filesys" ) @@ -80,14 +78,6 @@ func NewServer(args ServerArgs) (*Server, error) { args.Logger = logrus.NewEntry(logrus.StandardLogger()) } - storageMetadata, err := env.GetMultiEnvStorageMetadata(args.FS) - if err != nil { - return nil, err - } - if storageMetadata.ArchiveFilesPresent() { - return nil, errors.New("archive files present. Please run `dolt archive --revert` before running the server.") - } - s := new(Server) s.stopChan = make(chan struct{}) diff --git a/go/libraries/doltcore/remotestorage/chunk_cache.go b/go/libraries/doltcore/remotestorage/chunk_cache.go index 0412b20a0a..6345981b3e 100644 --- a/go/libraries/doltcore/remotestorage/chunk_cache.go +++ b/go/libraries/doltcore/remotestorage/chunk_cache.go @@ -21,21 +21,20 @@ import ( // ChunkCache is an interface used for caching chunks type ChunkCache interface { - // Put puts a slice of chunks into the cache. - Put(c []nbs.CompressedChunk) bool + // Put puts a slice of chunks into the cache. Error returned if the cache capacity has been exceeded. + Put(c []nbs.ToChunker) error // Get gets a map of hash to chunk for a set of hashes. In the event that a chunk is not in the cache, chunks.Empty. // is put in it's place - Get(h hash.HashSet) map[hash.Hash]nbs.CompressedChunk + Get(h hash.HashSet) map[hash.Hash]nbs.ToChunker // Has takes a set of hashes and returns the set of hashes that the cache currently does not have in it. Has(h hash.HashSet) (absent hash.HashSet) - // PutChunk puts a single chunk in the cache. true returns in the event that the chunk was cached successfully - // and false is returned if that chunk is already is the cache. - PutChunk(chunk nbs.CompressedChunk) bool + // PutChunk puts a single chunk in the cache. Returns an error if the cache capacity has been exceeded. + PutChunk(chunk nbs.ToChunker) error // GetAndClearChunksToFlush gets a map of hash to chunk which includes all the chunks that were put in the cache // between the last time GetAndClearChunksToFlush was called and now. - GetAndClearChunksToFlush() map[hash.Hash]nbs.CompressedChunk + GetAndClearChunksToFlush() map[hash.Hash]nbs.ToChunker } diff --git a/go/libraries/doltcore/remotestorage/chunk_fetcher.go b/go/libraries/doltcore/remotestorage/chunk_fetcher.go index 51bd4bac61..e9f7b3ad1d 100644 --- a/go/libraries/doltcore/remotestorage/chunk_fetcher.go +++ b/go/libraries/doltcore/remotestorage/chunk_fetcher.go @@ -18,9 +18,11 @@ import ( "context" "errors" "io" + "sync" "sync/atomic" "time" + "github.com/dolthub/gozstd" "golang.org/x/sync/errgroup" "google.golang.org/grpc" @@ -48,8 +50,14 @@ type ChunkFetcher struct { eg *errgroup.Group egCtx context.Context + // toGetCh is the channel used to request chunks. This will be initially given a root, + // and as refs are found, they will be added to the channel for workers to batch and request. toGetCh chan hash.HashSet - resCh chan nbs.CompressedChunk + + // resCh is the results channel for the fetcher. It is used both to return + // chunks themselves, and to indicate which chunks were requested but missing + // by having a Hash, but are empty. + resCh chan nbs.ToChunker abortCh chan struct{} stats StatsRecorder @@ -69,7 +77,7 @@ func NewChunkFetcher(ctx context.Context, dcs *DoltChunkStore) *ChunkFetcher { egCtx: ctx, toGetCh: make(chan hash.HashSet), - resCh: make(chan nbs.CompressedChunk), + resCh: make(chan nbs.ToChunker), abortCh: make(chan struct{}), stats: StatsFactory(), @@ -123,7 +131,7 @@ func (f *ChunkFetcher) CloseSend() error { // by |Get|. Returns |io.EOF| after |CloseSend| is called and all requested // chunks have been successfully received. Returns an error if this // |ChunkFetcher| is terminally failed or if the supplied |ctx| is |Done|. -func (f *ChunkFetcher) Recv(ctx context.Context) (nbs.CompressedChunk, error) { +func (f *ChunkFetcher) Recv(ctx context.Context) (nbs.ToChunker, error) { select { case <-ctx.Done(): return nbs.CompressedChunk{}, context.Cause(ctx) @@ -219,7 +227,7 @@ func fetcherHashSetToGetDlLocsReqsThread(ctx context.Context, reqCh chan hash.Ha // delivered in |reqCh|, and they will be delivered in order. // // This function handles backoff and retries for the underlying streaming RPC. -func fetcherRPCDownloadLocsThread(ctx context.Context, reqCh chan *remotesapi.GetDownloadLocsRequest, resCh chan []*remotesapi.DownloadLoc, client remotesapi.ChunkStoreServiceClient, storeRepoToken func(string), missingChunkCh chan nbs.CompressedChunk, host string) error { +func fetcherRPCDownloadLocsThread(ctx context.Context, reqCh chan *remotesapi.GetDownloadLocsRequest, resCh chan []*remotesapi.DownloadLoc, client remotesapi.ChunkStoreServiceClient, storeRepoToken func(string), missingChunkCh chan nbs.ToChunker, host string) error { stream, err := reliable.MakeCall[*remotesapi.GetDownloadLocsRequest, *remotesapi.GetDownloadLocsResponse]( ctx, reliable.CallOptions[*remotesapi.GetDownloadLocsRequest, *remotesapi.GetDownloadLocsResponse]{ @@ -327,33 +335,53 @@ func getMissingChunks(req *remotesapi.GetDownloadLocsRequest, resp *remotesapi.G return requested, nil } +// Delivered to a fetching thread by the download-coalescing thread, requests for the +// fetching thread to download all the entries in the |GetRange| and deliver them to +// the appropriate places. For |rangeType| Chunk, it delivers them to the fetched +// chunk channel. For |rangeType| Dictionary, it delivers them by calling |set| on the +// dictionaryCache with the fetched dictionary. type fetchResp struct { - get *GetRange - refresh func(ctx context.Context, err error, client remotesapi.ChunkStoreServiceClient) (string, error) + get *GetRange + refresh func(ctx context.Context, err error, client remotesapi.ChunkStoreServiceClient) (string, error) + rangeType rangeType + dictCache *dictionaryCache + path string } type fetchReq struct { - respCh chan fetchResp - cancelCh chan struct{} + respCh chan fetchResp } // A simple structure to keep track of *GetRange requests along with // |locationRefreshes| for the URL paths we have seen. type downloads struct { - ranges *ranges.Tree + // This Tree exclusively holds Chunk fetch ranges. + chunkRanges *ranges.Tree + // This Tree exclusively holds Dictionary fetch ranges. Every + // entry that we create in |dictionaryCache| which needs to be + // populated goes in here. These ranges must be fetched before + // (or concurrently with) any chunkRanges, since the chunk + // range fetches will block the fetching thread on the + // population of the dictionary cache entry. + dictRanges *ranges.Tree + // Holds all pending and fetched dictionaries for any chunk + // ranges that have gone into |chunkRanges|. + dictCache *dictionaryCache refreshes map[string]*locationRefresh } func newDownloads() downloads { return downloads{ - ranges: ranges.NewTree(chunkAggDistance), - refreshes: make(map[string]*locationRefresh), + chunkRanges: ranges.NewTree(chunkAggDistance), + dictRanges: ranges.NewTree(chunkAggDistance), + dictCache: &dictionaryCache{}, + refreshes: make(map[string]*locationRefresh), } } func (d downloads) Add(resp *remotesapi.DownloadLoc) { - gr := (*GetRange)(resp.Location.(*remotesapi.DownloadLoc_HttpGetRange).HttpGetRange) - path := gr.ResourcePath() + hgr := resp.Location.(*remotesapi.DownloadLoc_HttpGetRange).HttpGetRange + path := ResourcePath(hgr.Url) if v, ok := d.refreshes[path]; ok { v.Add(resp) } else { @@ -361,8 +389,14 @@ func (d downloads) Add(resp *remotesapi.DownloadLoc) { refresh.Add(resp) d.refreshes[path] = refresh } - for _, r := range gr.Ranges { - d.ranges.Insert(gr.Url, r.Hash, r.Offset, r.Length) + for _, r := range hgr.Ranges { + if r.DictionaryLength != 0 { + _, has := d.dictCache.getOrCreate(path, r.DictionaryOffset, r.DictionaryLength) + if !has { + d.dictRanges.Insert(hgr.Url, nil, r.DictionaryOffset, r.DictionaryLength, 0, 0) + } + } + d.chunkRanges.Insert(hgr.Url, r.Hash[:], r.Offset, r.Length, r.DictionaryOffset, r.DictionaryLength) } } @@ -370,84 +404,41 @@ func toGetRange(rs []*ranges.GetRange) *GetRange { ret := new(GetRange) for _, r := range rs { ret.Url = r.Url - ret.Ranges = append(ret.Ranges, &remotesapi.RangeChunk{ - Hash: r.Hash, - Offset: r.Offset, - Length: r.Length, + ret.Ranges = append(ret.Ranges, &Range{ + Hash: r.Hash, + Offset: r.Offset, + Length: r.Length, + DictOffset: r.DictOffset, + DictLength: r.DictLength, }) } return ret } +type rangeType int + +const ( + rangeType_Chunk rangeType = iota + rangeType_Dictionary +) + // Reads off |locCh| and assembles DownloadLocs into download ranges. func fetcherDownloadRangesThread(ctx context.Context, locCh chan []*remotesapi.DownloadLoc, fetchReqCh chan fetchReq, doneCh chan struct{}) error { downloads := newDownloads() - pending := make([]fetchReq, 0) - var toSend *GetRange for { - // pending is our slice of request threads that showed up - // asking for a download. We range through it and try to send - // them any work we have available. - for j := range pending { - // |toSend| could have come from a previous iteration - // of this loop or the outer loop. If it's |nil|, we - // can get the next range to download from - // |downloads.ranges|. - if toSend == nil { - max := downloads.ranges.DeleteMaxRegion() - if len(max) == 0 { - break - } - toSend = toGetRange(max) - } - path := toSend.ResourcePath() - refresh := downloads.refreshes[path] - - resp := fetchResp{ - get: toSend, - refresh: func(ctx context.Context, err error, client remotesapi.ChunkStoreServiceClient) (string, error) { - return refresh.GetURL(ctx, err, client) - }, - } - - select { - case pending[j].respCh <- resp: - toSend = nil - case <-pending[j].cancelCh: - // Because of dynamic thread pool sizing, a - // request thread could have been canceled and - // it has now gone away. If this happens, its - // respCh will be set to |nil| below and we - // will remove it from our |pending| set. But - // we need to hold onto |toSend| so that we do - // send it to a request thread eventually. - case <-ctx.Done(): - return context.Cause(ctx) - } - - pending[j].respCh = nil - } - - // Remove anything from |pending| that was actually delivered - // to. We use |respCh == nil| to indicate that the above loop - // delivered to the download thread. - newpending := make([]fetchReq, 0) - for i := range pending { - if pending[i].respCh != nil { - newpending = append(newpending, pending[i]) - } - } - pending = newpending - - // Once |locCh| closes, we set |locCh| to nil. If |locCh| is - // nil and our ranges Tree is empty, then we have delivered - // every download we will ever see to a download thread. We can - // close |doneCh| and return nil. - if locCh == nil && downloads.ranges.Len() == 0 { + hasWork := downloads.dictRanges.Len() > 0 || downloads.chunkRanges.Len() > 0 + if !hasWork && locCh == nil { + // Once |locCh| closes, we sit it to |nil|. If + // our range trees are empty then we have + // already delivered every download we will + // ever see to a download thread. close(doneCh) return nil } - + var reqCh chan fetchReq + if hasWork { + reqCh = fetchReqCh + } select { case req, ok := <-locCh: if !ok { @@ -457,8 +448,32 @@ func fetcherDownloadRangesThread(ctx context.Context, locCh chan []*remotesapi.D downloads.Add(loc) } } - case req := <-fetchReqCh: - pending = append(pending, req) + case req := <-reqCh: + var toSend *GetRange + var toSendType rangeType + if downloads.dictRanges.Len() > 0 { + max := downloads.dictRanges.DeleteMaxRegion() + toSend, toSendType = toGetRange(max), rangeType_Dictionary + } else { + // Necessarily non-empty, since |hasWork| is true... + max := downloads.chunkRanges.DeleteMaxRegion() + toSend, toSendType = toGetRange(max), rangeType_Chunk + } + path := toSend.ResourcePath() + refresh := downloads.refreshes[path] + resp := fetchResp{ + get: toSend, + refresh: func(ctx context.Context, err error, client remotesapi.ChunkStoreServiceClient) (string, error) { + return refresh.GetURL(ctx, err, client) + }, + rangeType: toSendType, + path: path, + dictCache: downloads.dictCache, + } + // Requester should deliver an exclusive, + // buffered channel where this deliver always + // succeeds. + req.respCh <- resp case <-ctx.Done(): return context.Cause(ctx) } @@ -527,7 +542,7 @@ func (cc *ConcurrencyControl) Run(ctx context.Context, done <-chan struct{}, ss } } -func fetcherDownloadURLThreads(ctx context.Context, fetchReqCh chan fetchReq, doneCh chan struct{}, chunkCh chan nbs.CompressedChunk, client remotesapi.ChunkStoreServiceClient, stats StatsRecorder, fetcher HTTPFetcher, params NetworkRequestParams) error { +func fetcherDownloadURLThreads(ctx context.Context, fetchReqCh chan fetchReq, doneCh chan struct{}, chunkCh chan nbs.ToChunker, client remotesapi.ChunkStoreServiceClient, stats StatsRecorder, fetcher HTTPFetcher, params NetworkRequestParams) error { eg, ctx := errgroup.WithContext(ctx) cc := &ConcurrencyControl{ MaxConcurrency: params.MaximumConcurrentDownloads, @@ -559,24 +574,68 @@ func fetcherDownloadURLThreads(ctx context.Context, fetchReqCh chan fetchReq, do return nil } -func fetcherDownloadURLThread(ctx context.Context, fetchReqCh chan fetchReq, doneCh <-chan struct{}, chunkCh chan nbs.CompressedChunk, client remotesapi.ChunkStoreServiceClient, stats StatsRecorder, health reliable.HealthRecorder, fetcher HTTPFetcher, params NetworkRequestParams) error { - respCh := make(chan fetchResp) - cancelCh := make(chan struct{}) +func deliverChunkCallback(chunkCh chan nbs.ToChunker, path string, dictCache *dictionaryCache) func(context.Context, []byte, *Range) error { + return func(ctx context.Context, bs []byte, rang *Range) error { + h := hash.New(rang.Hash[:]) + var cc nbs.ToChunker + if rang.DictLength != 0 { + payload, _ := dictCache.getOrCreate(path, rang.DictOffset, rang.DictLength) + dictRes, err := payload.Get() + if err != nil { + return err + } + cc = nbs.NewArchiveToChunker(h, dictRes.(*gozstd.DDict), bs) + } else { + var err error + cc, err = nbs.NewCompressedChunk(h, bs) + if err != nil { + return err + } + } + select { + case chunkCh <- cc: + case <-ctx.Done(): + return context.Cause(ctx) + } + return nil + } +} + +func setDictionaryCallback(dictCache *dictionaryCache, path string) func(context.Context, []byte, *Range) error { + return func(ctx context.Context, bs []byte, rang *Range) error { + var ddict *gozstd.DDict + decompressed, err := gozstd.Decompress(nil, bs) + if err == nil { + ddict, err = gozstd.NewDDict(decompressed) + } + payload, _ := dictCache.getOrCreate(path, rang.Offset, rang.Length) + payload.Set(ddict, err) + // XXX: For now, we fail here on any error, instead of when we try to use the dictionary... + // For now, the record in the cache will be terminally failed and is never removed. + return err + } +} + +func fetcherDownloadURLThread(ctx context.Context, fetchReqCh chan fetchReq, doneCh <-chan struct{}, chunkCh chan nbs.ToChunker, client remotesapi.ChunkStoreServiceClient, stats StatsRecorder, health reliable.HealthRecorder, fetcher HTTPFetcher, params NetworkRequestParams) error { + respCh := make(chan fetchResp, 1) for { select { case <-ctx.Done(): return context.Cause(ctx) case <-doneCh: return nil - case fetchReqCh <- fetchReq{respCh: respCh, cancelCh: cancelCh}: + case fetchReqCh <- fetchReq{respCh: respCh}: select { - case <-doneCh: - close(cancelCh) - return nil case <-ctx.Done(): return context.Cause(ctx) case fetchResp := <-respCh: - f := fetchResp.get.GetDownloadFunc(ctx, stats, health, fetcher, params, chunkCh, func(ctx context.Context, lastError error, resourcePath string) (string, error) { + var cb func(context.Context, []byte, *Range) error + if fetchResp.rangeType == rangeType_Chunk { + cb = deliverChunkCallback(chunkCh, fetchResp.path, fetchResp.dictCache) + } else { + cb = setDictionaryCallback(fetchResp.dictCache, fetchResp.path) + } + f := fetchResp.get.GetDownloadFunc(ctx, stats, health, fetcher, params, cb, func(ctx context.Context, lastError error, resourcePath string) (string, error) { return fetchResp.refresh(ctx, lastError, client) }) err := f() @@ -587,3 +646,48 @@ func fetcherDownloadURLThread(ctx context.Context, fetchReqCh chan fetchReq, don } } } + +// A dictionaryCache provides a rendezvous point for chunk fetches +// which have data dependencies on dictionary fetches. It stores a +// single record per |path|,|offset| tuple we see, and that record +// will be populated with the |*gozstd.DDict| that results from +// fetching those contents. Every |GetRange| that has a dictionary +// dependency gets the record out of the dictionary cache. The first +// time the cache entry is created, the download thread also schedules +// the dictionary itself to be fetched and populated through |Set|. +type dictionaryCache struct { + dictionaries sync.Map +} + +// DictionaryKey is the a globaly unique identifier for an archive dictionary. +type DictionaryKey struct { + // This is the short url to the resource, not including the query parameters - which are provided by the + // locationRefresher. + path string + off uint64 + len uint32 +} + +type DictionaryPayload struct { + done chan struct{} + res any + err error +} + +func (p *DictionaryPayload) Get() (any, error) { + <-p.done + return p.res, p.err +} + +func (p *DictionaryPayload) Set(res any, err error) { + p.res = res + p.err = err + close(p.done) +} + +func (dc *dictionaryCache) getOrCreate(path string, offset uint64, length uint32) (*DictionaryPayload, bool) { + key := DictionaryKey{path, offset, length} + entry, loaded := dc.dictionaries.LoadOrStore(key, &DictionaryPayload{done: make(chan struct{})}) + payload := entry.(*DictionaryPayload) + return payload, loaded +} diff --git a/go/libraries/doltcore/remotestorage/chunk_store.go b/go/libraries/doltcore/remotestorage/chunk_store.go index 70935dea1a..b2f8a0ee17 100644 --- a/go/libraries/doltcore/remotestorage/chunk_store.go +++ b/go/libraries/doltcore/remotestorage/chunk_store.go @@ -317,7 +317,7 @@ func (dcs *DoltChunkStore) Get(ctx context.Context, h hash.Hash) (chunks.Chunk, func (dcs *DoltChunkStore) GetMany(ctx context.Context, hashes hash.HashSet, found func(context.Context, *chunks.Chunk)) error { ae := atomicerr.New() decompressedSize := uint64(0) - err := dcs.GetManyCompressed(ctx, hashes, func(ctx context.Context, cc nbs.CompressedChunk) { + err := dcs.GetManyCompressed(ctx, hashes, func(ctx context.Context, cc nbs.ToChunker) { if ae.IsSet() { return } @@ -340,7 +340,7 @@ func (dcs *DoltChunkStore) GetMany(ctx context.Context, hashes hash.HashSet, fou // GetMany gets the Chunks with |hashes| from the store. On return, |foundChunks| will have been fully sent all chunks // which have been found. Any non-present chunks will silently be ignored. -func (dcs *DoltChunkStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error { +func (dcs *DoltChunkStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error { ctx, span := tracer.Start(ctx, "remotestorage.GetManyCompressed") defer span.End() @@ -353,7 +353,7 @@ func (dcs *DoltChunkStore) GetManyCompressed(ctx context.Context, hashes hash.Ha for h := range hashes { c := hashToChunk[h] - if c.IsEmpty() { + if c == nil || c.IsEmpty() { notCached = append(notCached, h) } else { found(ctx, c) @@ -371,11 +371,28 @@ func (dcs *DoltChunkStore) GetManyCompressed(ctx context.Context, hashes hash.Ha return nil } -type GetRange remotesapi.HttpGetRange +// GetRange is structurally similar to remotesapi.HttpGetRange, but +// with added functions. +type GetRange struct { + Url string + Ranges []*Range +} + +type Range struct { + Hash []byte + Offset uint64 + Length uint32 + DictOffset uint64 + DictLength uint32 +} + +func ResourcePath(urlS string) string { + u, _ := url.Parse(urlS) + return fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Path) +} func (gr *GetRange) ResourcePath() string { - u, _ := url.Parse(gr.Url) - return fmt.Sprintf("%s://%s%s", u.Scheme, u.Host, u.Path) + return ResourcePath(gr.Url) } func (gr *GetRange) Append(other *GetRange) { @@ -432,10 +449,11 @@ func sortRangesBySize(ranges []*GetRange) { type resourcePathToUrlFunc func(ctx context.Context, lastError error, resourcePath string) (url string, err error) -func (gr *GetRange) GetDownloadFunc(ctx context.Context, stats StatsRecorder, health reliable.HealthRecorder, fetcher HTTPFetcher, params NetworkRequestParams, chunkChan chan nbs.CompressedChunk, pathToUrl resourcePathToUrlFunc) func() error { +func (gr *GetRange) GetDownloadFunc(ctx context.Context, stats StatsRecorder, health reliable.HealthRecorder, fetcher HTTPFetcher, params NetworkRequestParams, resCb func(context.Context, []byte, *Range) error, pathToUrl resourcePathToUrlFunc) func() error { if len(gr.Ranges) == 0 { return func() error { return nil } } + return func() error { urlF := func(lastError error) (string, error) { url, err := pathToUrl(ctx, lastError, gr.ResourcePath()) @@ -466,55 +484,66 @@ func (gr *GetRange) GetDownloadFunc(ctx context.Context, stats StatsRecorder, he RespHeadersTimeout: params.RespHeadersTimeout, }) defer resp.Close() - reader := &RangeChunkReader{GetRange: gr, Reader: resp.Body} + reader := &RangeReader{GetRange: gr, Reader: resp.Body} for { - cc, err := reader.ReadChunk() + bs, rang, err := reader.ReadNextRange() if errors.Is(err, io.EOF) { return nil } if err != nil { return err } - select { - case chunkChan <- cc: - case <-ctx.Done(): - return context.Cause(ctx) + err = resCb(ctx, bs, rang) + if err != nil { + return err } } } } -type RangeChunkReader struct { +type RangeReader struct { GetRange *GetRange Reader io.Reader - i int - skip int + // The range we currently reading. + i int + // The |skip|, from the last range we read + // to the current range, which we need to + // exexcute before on the next call to + // |ReadNextRange| + skip int } -func (r *RangeChunkReader) ReadChunk() (nbs.CompressedChunk, error) { +func (r *RangeReader) ReadNextRange() ([]byte, *Range, error) { if r.skip > 0 { _, err := io.CopyN(io.Discard, r.Reader, int64(r.skip)) if err != nil { - return nbs.CompressedChunk{}, err + return nil, nil, err } r.skip = 0 } - if r.i >= len(r.GetRange.Ranges) { - return nbs.CompressedChunk{}, io.EOF - } - if r.i < len(r.GetRange.Ranges)-1 { - r.skip = int(r.GetRange.GapBetween(r.i, r.i+1)) - } - l := r.GetRange.Ranges[r.i].Length - h := hash.New(r.GetRange.Ranges[r.i].Hash) + + idx := r.i r.i += 1 + + if idx >= len(r.GetRange.Ranges) { + return nil, nil, io.EOF + } + if idx < len(r.GetRange.Ranges)-1 { + // If this isn't the last range, calculate and + // store the skip that will be necessary after + // we read this range. + r.skip = int(r.GetRange.GapBetween(idx, idx+1)) + } + + rang := r.GetRange.Ranges[idx] + l := rang.Length + buf := make([]byte, l) _, err := io.ReadFull(r.Reader, buf) if err != nil { - return nbs.CompressedChunk{}, err - } else { - return nbs.NewCompressedChunk(h, buf) + return nil, nil, err } + return buf, rang, nil } type locationRefresh struct { @@ -574,7 +603,7 @@ type RepoRequest interface { SetRepoPath(string) } -func (dcs *DoltChunkStore) readChunksAndCache(ctx context.Context, hashes []hash.Hash, found func(context.Context, nbs.CompressedChunk)) (err error) { +func (dcs *DoltChunkStore) readChunksAndCache(ctx context.Context, hashes []hash.Hash, found func(context.Context, nbs.ToChunker)) (err error) { toSend := hash.NewHashSet(hashes...) fetcher := dcs.ChunkFetcher(ctx) @@ -603,9 +632,9 @@ func (dcs *DoltChunkStore) readChunksAndCache(ctx context.Context, hashes []hash return err } // Don't forward on empty/not found chunks. - if len(cc.CompressedData) > 0 { - if dcs.cache.PutChunk(cc) { - return ErrCacheCapacityExceeded + if !cc.IsEmpty() { + if err := dcs.cache.PutChunk(cc); err != nil { + return err } found(egCtx, cc) } @@ -644,7 +673,7 @@ func (dcs *DoltChunkStore) HasMany(ctx context.Context, hashes hash.HashSet) (ha hashSl, byteSl := HashSetToSlices(notCached) absent := make(hash.HashSet) - var found []nbs.CompressedChunk + var found []nbs.ToChunker var err error batchItr(len(hashSl), maxHasManyBatchSize, func(st, end int) (stop bool) { @@ -702,8 +731,8 @@ func (dcs *DoltChunkStore) HasMany(ctx context.Context, hashes hash.HashSet) (ha } if len(found) > 0 { - if dcs.cache.Put(found) { - return hash.HashSet{}, ErrCacheCapacityExceeded + if err := dcs.cache.Put(found); err != nil { + return hash.HashSet{}, err } } @@ -738,8 +767,8 @@ func (dcs *DoltChunkStore) Put(ctx context.Context, c chunks.Chunk, getAddrs chu } cc := nbs.ChunkToCompressedChunk(c) - if dcs.cache.Put([]nbs.CompressedChunk{cc}) { - return ErrCacheCapacityExceeded + if err := dcs.cache.Put([]nbs.ToChunker{cc}); err != nil { + return err } return nil } @@ -1051,6 +1080,11 @@ func (dcs *DoltChunkStore) SupportedOperations() chunks.TableFileStoreOps { // WriteTableFile reads a table file from the provided reader and writes it to the chunk store. func (dcs *DoltChunkStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) error { + // Err if the suffix is an archive file + if strings.HasSuffix(fileId, nbs.ArchiveFileSuffix) { + return errors.New("cannot write archive file ids currently.") + } + fileIdBytes := hash.Parse(fileId) err := dcs.uploadTableFileWithRetries(ctx, fileIdBytes, uint64(numChunks), contentHash, getRd) if err != nil { @@ -1152,6 +1186,15 @@ func (drtf DoltRemoteTableFile) LocationPrefix() string { return "" } +func (drtf DoltRemoteTableFile) LocationSuffix() string { + u, _ := url.Parse(drtf.info.Url) + if strings.HasSuffix(u.Path, nbs.ArchiveFileSuffix) { + return nbs.ArchiveFileSuffix + } + + return "" +} + // FileID gets the id of the file func (drtf DoltRemoteTableFile) FileID() string { id := drtf.info.FileId diff --git a/go/libraries/doltcore/remotestorage/internal/ranges/ranges.go b/go/libraries/doltcore/remotestorage/internal/ranges/ranges.go index de1600bd0a..924756350e 100644 --- a/go/libraries/doltcore/remotestorage/internal/ranges/ranges.go +++ b/go/libraries/doltcore/remotestorage/internal/ranges/ranges.go @@ -21,18 +21,27 @@ import ( "github.com/google/btree" ) -// GetRange represents a way to get the contents for a Chunk from a given Url -// with an HTTP Range request. The chunk with hash |Hash| can be fetched using +// GetRange represents a range of remote data that has semantic meaning to the +// ChunkFetcher. These ranges are currently either Chunks, or Dictionaries. +// They can be fetched from the remote URL with an HTTP Range request. +// For a chunk range, the chunk with hash |Hash| can be fetched using // the |Url| with a Range request starting at |Offset| and reading |Length| -// bytes. +// bytes. A Dictionary does not have a meaningful Hash, but its identity is +// unique for a Url and Offset. // // A |GetRange| struct is a member of a |Region| in the |RegionHeap|. +// +// Chunk |GetRange|s which depend on Dictionaries can be constructed with +// some state which allows them to fetch those dictionaries from a shared +// chache when they need them. That is their GetDict callback. type GetRange struct { - Url string - Hash []byte - Offset uint64 - Length uint32 - Region *Region + Url string + Hash []byte + Offset uint64 + Length uint32 + DictOffset uint64 + DictLength uint32 + Region *Region } // A |Region| represents a continuous range of bytes within in a Url. @@ -145,12 +154,14 @@ func (t *Tree) Len() int { return t.t.Len() } -func (t *Tree) Insert(url string, hash []byte, offset uint64, length uint32) { +func (t *Tree) Insert(url string, hash []byte, offset uint64, length uint32, dictOffset uint64, dictLength uint32) { ins := &GetRange{ - Url: t.intern(url), - Hash: hash, - Offset: offset, - Length: length, + Url: t.intern(url), + Hash: hash, + Offset: offset, + Length: length, + DictOffset: dictOffset, + DictLength: dictLength, } t.t.ReplaceOrInsert(ins) diff --git a/go/libraries/doltcore/remotestorage/internal/ranges/ranges_test.go b/go/libraries/doltcore/remotestorage/internal/ranges/ranges_test.go index 9ec6b04e33..1f36ec296f 100644 --- a/go/libraries/doltcore/remotestorage/internal/ranges/ranges_test.go +++ b/go/libraries/doltcore/remotestorage/internal/ranges/ranges_test.go @@ -77,11 +77,11 @@ func TestTree(t *testing.T) { tree := NewTree(8 * 1024) // Insert 1KB ranges every 16 KB. for i, j := 0, 0; i < 16; i, j = i+1, j+16*1024 { - tree.Insert("A", []byte{}, uint64(j), 1024) + tree.Insert("A", []byte{}, uint64(j), 1024, 0, 0) } // Insert 1KB ranges every 16 KB, offset by 8KB. for i := 15*16*1024 + 8*1024; i >= 0; i -= 16 * 1024 { - tree.Insert("A", []byte{}, uint64(i), 1024) + tree.Insert("A", []byte{}, uint64(i), 1024, 0, 0) } assertTree(t, tree) }) @@ -89,11 +89,11 @@ func TestTree(t *testing.T) { tree := NewTree(8 * 1024) // Insert 1KB ranges every 16 KB, offset by 8KB. for i := 15*16*1024 + 8*1024; i >= 0; i -= 16 * 1024 { - tree.Insert("A", []byte{}, uint64(i), 1024) + tree.Insert("A", []byte{}, uint64(i), 1024, 0, 0) } // Insert 1KB ranges every 16 KB. for i, j := 0, 0; i < 16; i, j = i+1, j+16*1024 { - tree.Insert("A", []byte{}, uint64(j), 1024) + tree.Insert("A", []byte{}, uint64(j), 1024, 0, 0) } assertTree(t, tree) }) @@ -111,7 +111,7 @@ func TestTree(t *testing.T) { }) tree := NewTree(8 * 1024) for _, offset := range entries { - tree.Insert("A", []byte{}, offset, 1024) + tree.Insert("A", []byte{}, offset, 1024, 0, 0) } assertTree(t, tree) } @@ -126,7 +126,7 @@ func TestTree(t *testing.T) { "B", "A", "9", "8", } for i, j := 0, 0; i < 16; i, j = i+1, j+1024 { - tree.Insert(files[i], []byte{}, uint64(j), 1024) + tree.Insert(files[i], []byte{}, uint64(j), 1024, 0, 0) } assert.Equal(t, 16, tree.regions.Len()) assert.Equal(t, 16, tree.t.Len()) @@ -134,17 +134,17 @@ func TestTree(t *testing.T) { t.Run("MergeInMiddle", func(t *testing.T) { tree := NewTree(8 * 1024) // 1KB chunk at byte 0 - tree.Insert("A", []byte{}, 0, 1024) + tree.Insert("A", []byte{}, 0, 1024, 0, 0) // 1KB chunk at byte 16KB - tree.Insert("A", []byte{}, 16384, 1024) + tree.Insert("A", []byte{}, 16384, 1024, 0, 0) assert.Equal(t, 2, tree.regions.Len()) assert.Equal(t, 2, tree.t.Len()) // 1KB chunk at byte 8KB - tree.Insert("A", []byte{}, 8192, 1024) + tree.Insert("A", []byte{}, 8192, 1024, 0, 0) assert.Equal(t, 1, tree.regions.Len()) assert.Equal(t, 3, tree.t.Len()) - tree.Insert("A", []byte{}, 4096, 1024) - tree.Insert("A", []byte{}, 12228, 1024) + tree.Insert("A", []byte{}, 4096, 1024, 0, 0) + tree.Insert("A", []byte{}, 12228, 1024, 0, 0) assert.Equal(t, 1, tree.regions.Len()) assert.Equal(t, 5, tree.t.Len()) e, _ := tree.t.Min() @@ -184,7 +184,7 @@ func TestTree(t *testing.T) { t.Run("InsertAscending", func(t *testing.T) { tree := NewTree(4 * 1024) for _, e := range entries { - tree.Insert(e.url, []byte{e.id}, e.offset, e.length) + tree.Insert(e.url, []byte{e.id}, e.offset, e.length, 0, 0) } assertTree(t, tree) }) @@ -192,7 +192,7 @@ func TestTree(t *testing.T) { tree := NewTree(4 * 1024) for i := len(entries) - 1; i >= 0; i-- { e := entries[i] - tree.Insert(e.url, []byte{e.id}, e.offset, e.length) + tree.Insert(e.url, []byte{e.id}, e.offset, e.length, 0, 0) } assertTree(t, tree) }) @@ -205,7 +205,7 @@ func TestTree(t *testing.T) { }) tree := NewTree(4 * 1024) for _, e := range entries { - tree.Insert(e.url, []byte{e.id}, e.offset, e.length) + tree.Insert(e.url, []byte{e.id}, e.offset, e.length, 0, 0) } assertTree(t, tree) } diff --git a/go/libraries/doltcore/remotestorage/map_chunk_cache.go b/go/libraries/doltcore/remotestorage/map_chunk_cache.go index ea5dad323e..510ca54469 100644 --- a/go/libraries/doltcore/remotestorage/map_chunk_cache.go +++ b/go/libraries/doltcore/remotestorage/map_chunk_cache.go @@ -21,19 +21,24 @@ import ( "github.com/dolthub/dolt/go/store/nbs" ) +const ( + // averageChunkSize is used to estimate the size of chunk for purposes of avoiding excessive memory usage + averageChunkSize = 1 << 12 +) + // mapChunkCache is a ChunkCache implementation that stores everything in an in memory map. type mapChunkCache struct { mu *sync.Mutex - hashToChunk map[hash.Hash]nbs.CompressedChunk - toFlush map[hash.Hash]nbs.CompressedChunk + hashToChunk map[hash.Hash]nbs.ToChunker + toFlush map[hash.Hash]nbs.ToChunker cm CapacityMonitor } func newMapChunkCache() *mapChunkCache { return &mapChunkCache{ &sync.Mutex{}, - make(map[hash.Hash]nbs.CompressedChunk), - make(map[hash.Hash]nbs.CompressedChunk), + make(map[hash.Hash]nbs.ToChunker), + make(map[hash.Hash]nbs.ToChunker), NewUncappedCapacityMonitor(), } } @@ -42,14 +47,14 @@ func newMapChunkCache() *mapChunkCache { func NewMapChunkCacheWithMaxCapacity(maxCapacity int64) *mapChunkCache { return &mapChunkCache{ &sync.Mutex{}, - make(map[hash.Hash]nbs.CompressedChunk), - make(map[hash.Hash]nbs.CompressedChunk), + make(map[hash.Hash]nbs.ToChunker), + make(map[hash.Hash]nbs.ToChunker), NewFixedCapacityMonitor(maxCapacity), } } -// Put puts a slice of chunks into the cache. -func (mcc *mapChunkCache) Put(chnks []nbs.CompressedChunk) bool { +// Put puts a slice of chunks into the cache. Returns an error if the cache capacity has been exceeded. +func (mcc *mapChunkCache) Put(chnks []nbs.ToChunker) error { mcc.mu.Lock() defer mcc.mu.Unlock() @@ -63,8 +68,8 @@ func (mcc *mapChunkCache) Put(chnks []nbs.CompressedChunk) bool { } } - if mcc.cm.CapacityExceeded(len(c.FullCompressedChunk)) { - return true + if mcc.cm.CapacityExceeded(averageChunkSize) { + return ErrCacheCapacityExceeded } mcc.hashToChunk[h] = c @@ -74,13 +79,13 @@ func (mcc *mapChunkCache) Put(chnks []nbs.CompressedChunk) bool { } } - return false + return nil } // Get gets a map of hash to chunk for a set of hashes. In the event that a chunk is not in the cache, chunks.Empty. // is put in it's place -func (mcc *mapChunkCache) Get(hashes hash.HashSet) map[hash.Hash]nbs.CompressedChunk { - hashToChunk := make(map[hash.Hash]nbs.CompressedChunk) +func (mcc *mapChunkCache) Get(hashes hash.HashSet) map[hash.Hash]nbs.ToChunker { + hashToChunk := make(map[hash.Hash]nbs.ToChunker) mcc.mu.Lock() defer mcc.mu.Unlock() @@ -112,26 +117,26 @@ func (mcc *mapChunkCache) Has(hashes hash.HashSet) (absent hash.HashSet) { return absent } -func (mcc *mapChunkCache) PutChunk(ch nbs.CompressedChunk) bool { +func (mcc *mapChunkCache) PutChunk(ch nbs.ToChunker) error { mcc.mu.Lock() defer mcc.mu.Unlock() h := ch.Hash() if existing, ok := mcc.hashToChunk[h]; !ok || existing.IsEmpty() { - if mcc.cm.CapacityExceeded(len(ch.FullCompressedChunk)) { - return true + if mcc.cm.CapacityExceeded(averageChunkSize) { + return ErrCacheCapacityExceeded } mcc.hashToChunk[h] = ch mcc.toFlush[h] = ch } - return false + return nil } // GetAndClearChunksToFlush gets a map of hash to chunk which includes all the chunks that were put in the cache // between the last time GetAndClearChunksToFlush was called and now. -func (mcc *mapChunkCache) GetAndClearChunksToFlush() map[hash.Hash]nbs.CompressedChunk { - newToFlush := make(map[hash.Hash]nbs.CompressedChunk) +func (mcc *mapChunkCache) GetAndClearChunksToFlush() map[hash.Hash]nbs.ToChunker { + newToFlush := make(map[hash.Hash]nbs.ToChunker) mcc.mu.Lock() defer mcc.mu.Unlock() diff --git a/go/libraries/doltcore/remotestorage/map_chunk_cache_test.go b/go/libraries/doltcore/remotestorage/map_chunk_cache_test.go index 6944eb98c3..8cb5e118d1 100644 --- a/go/libraries/doltcore/remotestorage/map_chunk_cache_test.go +++ b/go/libraries/doltcore/remotestorage/map_chunk_cache_test.go @@ -27,8 +27,8 @@ import ( "github.com/dolthub/dolt/go/store/nbs" ) -func genRandomChunks(rng *rand.Rand, n int) (hash.HashSet, []nbs.CompressedChunk) { - chks := make([]nbs.CompressedChunk, n) +func genRandomChunks(rng *rand.Rand, n int) (hash.HashSet, []nbs.ToChunker) { + chks := make([]nbs.ToChunker, n) hashes := make(hash.HashSet) for i := 0; i < n; i++ { size := int(rng.Int31n(99) + 1) @@ -88,7 +88,7 @@ func TestMapChunkCache(t *testing.T) { toFlush = mapChunkCache.GetAndClearChunksToFlush() - expected := map[hash.Hash]nbs.CompressedChunk{moreChks[0].Hash(): moreChks[0]} + expected := map[hash.Hash]nbs.ToChunker{moreChks[0].Hash(): moreChks[0]} eq := reflect.DeepEqual(toFlush, expected) assert.True(t, eq, "Missing or unexpected chunks to flush (seed %d)", seed) } diff --git a/go/libraries/doltcore/remotestorage/noop_chunk_cache.go b/go/libraries/doltcore/remotestorage/noop_chunk_cache.go index 8edf589c31..b8848ef771 100644 --- a/go/libraries/doltcore/remotestorage/noop_chunk_cache.go +++ b/go/libraries/doltcore/remotestorage/noop_chunk_cache.go @@ -29,22 +29,22 @@ var noopChunkCache = &noopChunkCacheImpl{} type noopChunkCacheImpl struct { } -func (*noopChunkCacheImpl) Put(chnks []nbs.CompressedChunk) bool { - return false +func (*noopChunkCacheImpl) Put(chnks []nbs.ToChunker) error { + return nil } -func (*noopChunkCacheImpl) Get(hashes hash.HashSet) map[hash.Hash]nbs.CompressedChunk { - return make(map[hash.Hash]nbs.CompressedChunk) +func (*noopChunkCacheImpl) Get(hashes hash.HashSet) map[hash.Hash]nbs.ToChunker { + return make(map[hash.Hash]nbs.ToChunker) } func (*noopChunkCacheImpl) Has(hashes hash.HashSet) (absent hash.HashSet) { return hashes } -func (*noopChunkCacheImpl) PutChunk(ch nbs.CompressedChunk) bool { - return false +func (*noopChunkCacheImpl) PutChunk(ch nbs.ToChunker) error { + return nil } -func (*noopChunkCacheImpl) GetAndClearChunksToFlush() map[hash.Hash]nbs.CompressedChunk { +func (*noopChunkCacheImpl) GetAndClearChunksToFlush() map[hash.Hash]nbs.ToChunker { panic("noopChunkCache does not support GetAndClearChunksToFlush().") } diff --git a/go/store/chunks/tablefilestore.go b/go/store/chunks/tablefilestore.go index e6e38f4e36..773bc52195 100644 --- a/go/store/chunks/tablefilestore.go +++ b/go/store/chunks/tablefilestore.go @@ -31,6 +31,10 @@ type TableFile interface { // LocationPrefix LocationPrefix() string + // Used in conjuction with the FileID to create file paths to table files. Currently archive files are the only + // that take advantage of this, using .darc as the file suffix. + LocationSuffix() string + // NumChunks returns the number of chunks in a table file NumChunks() int diff --git a/go/store/datas/pull/clone.go b/go/store/datas/pull/clone.go index 4cd2943983..27a3ca9405 100644 --- a/go/store/datas/pull/clone.go +++ b/go/store/datas/pull/clone.go @@ -81,9 +81,11 @@ func mapTableFiles(tblFiles []chunks.TableFile) ([]string, map[string]chunks.Tab fileIDtoNumChunks := make(map[string]int) for i, tblFile := range tblFiles { - fileIDtoTblFile[tblFile.FileID()] = tblFile - fileIds[i] = tblFile.FileID() - fileIDtoNumChunks[tblFile.FileID()] = tblFile.NumChunks() + fileId := tblFile.FileID() + + fileIDtoTblFile[fileId] = tblFile + fileIds[i] = fileId + fileIDtoNumChunks[fileId] = tblFile.NumChunks() } return fileIds, fileIDtoTblFile, fileIDtoNumChunks @@ -136,7 +138,7 @@ func clone(ctx context.Context, srcTS, sinkTS chunks.TableFileStore, sinkCS chun } report(TableFileEvent{EventType: DownloadStart, TableFiles: []chunks.TableFile{tblFile}}) - err = sinkTS.WriteTableFile(ctx, tblFile.FileID(), tblFile.NumChunks(), nil, func() (io.ReadCloser, uint64, error) { + err = sinkTS.WriteTableFile(ctx, tblFile.FileID()+tblFile.LocationSuffix(), tblFile.NumChunks(), nil, func() (io.ReadCloser, uint64, error) { rd, contentLength, err := tblFile.Open(ctx) if err != nil { return nil, 0, err diff --git a/go/store/datas/pull/pull_chunk_fetcher.go b/go/store/datas/pull/pull_chunk_fetcher.go index 2e0d58a0f8..3d2c6d556a 100644 --- a/go/store/datas/pull/pull_chunk_fetcher.go +++ b/go/store/datas/pull/pull_chunk_fetcher.go @@ -26,7 +26,7 @@ import ( ) type GetManyer interface { - GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error + GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error } type ChunkFetcherable interface { @@ -52,7 +52,7 @@ type PullChunkFetcher struct { batchCh chan hash.HashSet doneCh chan struct{} - resCh chan nbs.CompressedChunk + resCh chan nbs.ToChunker } func NewPullChunkFetcher(ctx context.Context, getter GetManyer) *PullChunkFetcher { @@ -63,7 +63,7 @@ func NewPullChunkFetcher(ctx context.Context, getter GetManyer) *PullChunkFetche getter: getter, batchCh: make(chan hash.HashSet), doneCh: make(chan struct{}), - resCh: make(chan nbs.CompressedChunk), + resCh: make(chan nbs.ToChunker), } ret.eg.Go(func() error { return ret.fetcherThread(func() { @@ -86,9 +86,9 @@ func (f *PullChunkFetcher) fetcherThread(finalize func()) error { missing := batch.Copy() // Blocking get, no concurrency, only one fetcher. - err := f.getter.GetManyCompressed(f.ctx, batch, func(ctx context.Context, chk nbs.CompressedChunk) { + err := f.getter.GetManyCompressed(f.ctx, batch, func(ctx context.Context, chk nbs.ToChunker) { mu.Lock() - missing.Remove(chk.H) + missing.Remove(chk.Hash()) mu.Unlock() select { case <-ctx.Done(): @@ -139,7 +139,7 @@ func (f *PullChunkFetcher) Close() error { return f.eg.Wait() } -func (f *PullChunkFetcher) Recv(ctx context.Context) (nbs.CompressedChunk, error) { +func (f *PullChunkFetcher) Recv(ctx context.Context) (nbs.ToChunker, error) { select { case res, ok := <-f.resCh: if !ok { diff --git a/go/store/datas/pull/pull_chunk_fetcher_test.go b/go/store/datas/pull/pull_chunk_fetcher_test.go index 2fa3bd4e18..f2975efb67 100644 --- a/go/store/datas/pull/pull_chunk_fetcher_test.go +++ b/go/store/datas/pull/pull_chunk_fetcher_test.go @@ -70,8 +70,12 @@ func TestPullChunkFetcher(t *testing.T) { defer wg.Done() cmp, err := f.Recv(context.Background()) assert.NoError(t, err) - assert.Equal(t, cmp.H, gm.C.H) - assert.Equal(t, cmp.FullCompressedChunk, gm.C.FullCompressedChunk) + assert.Equal(t, cmp.Hash(), gm.C.H) + + cc, ok := cmp.(nbs.CompressedChunk) + assert.True(t, ok) + + assert.Equal(t, cc.FullCompressedChunk, gm.C.FullCompressedChunk) _, err = f.Recv(context.Background()) assert.ErrorIs(t, err, io.EOF) assert.NoError(t, f.Close()) @@ -92,8 +96,11 @@ func TestPullChunkFetcher(t *testing.T) { defer wg.Done() cmp, err := f.Recv(context.Background()) assert.NoError(t, err) - assert.Equal(t, cmp.H, h) - assert.Nil(t, cmp.FullCompressedChunk) + assert.Equal(t, cmp.Hash(), h) + + cc, ok := cmp.(nbs.CompressedChunk) + assert.True(t, ok) + assert.Nil(t, cc.FullCompressedChunk) _, err = f.Recv(context.Background()) assert.ErrorIs(t, err, io.EOF) assert.NoError(t, f.Close()) @@ -136,7 +143,7 @@ func TestPullChunkFetcher(t *testing.T) { type emptyGetManyer struct { } -func (emptyGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error { +func (emptyGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error { return nil } @@ -144,7 +151,7 @@ type deliveringGetManyer struct { C nbs.CompressedChunk } -func (d deliveringGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error { +func (d deliveringGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error { for _ = range hashes { found(ctx, d.C) } @@ -155,7 +162,7 @@ type blockingGetManyer struct { block chan struct{} } -func (b blockingGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error { +func (b blockingGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error { <-b.block return nil } @@ -165,6 +172,6 @@ type errorGetManyer struct { var getManyerErr = fmt.Errorf("always return an error") -func (errorGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.CompressedChunk)) error { +func (errorGetManyer) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, nbs.ToChunker)) error { return getManyerErr } diff --git a/go/store/datas/pull/puller.go b/go/store/datas/pull/puller.go index 56cfe4e7bb..a95e1e3559 100644 --- a/go/store/datas/pull/puller.go +++ b/go/store/datas/pull/puller.go @@ -359,17 +359,19 @@ func (p *Puller) Pull(ctx context.Context) error { if cChk.IsGhost() { return fmt.Errorf("attempted to push or pull ghost chunk: %w", nbs.ErrGhostChunkRequested) } - if len(cChk.FullCompressedChunk) == 0 { + if cChk.IsEmpty() { return errors.New("failed to get all chunks.") } - atomic.AddUint64(&p.stats.fetchedSourceBytes, uint64(len(cChk.FullCompressedChunk))) atomic.AddUint64(&p.stats.fetchedSourceChunks, uint64(1)) chnk, err := cChk.ToChunk() if err != nil { return err } + + atomic.AddUint64(&p.stats.fetchedSourceBytes, uint64(len(chnk.Data()))) + err = p.waf(chnk, func(h hash.Hash, _ bool) error { tracker.Seen(h) return nil @@ -379,9 +381,19 @@ func (p *Puller) Pull(ctx context.Context) error { } tracker.TickProcessed() - err = p.wr.AddCompressedChunk(ctx, cChk) - if err != nil { - return err + if compressedChunk, ok := cChk.(nbs.CompressedChunk); ok { + err = p.wr.AddCompressedChunk(ctx, compressedChunk) + if err != nil { + return err + } + } else if _, ok := cChk.(nbs.ArchiveToChunker); ok { + // NM4 - Until we can write quickly to archives..... + cc := nbs.ChunkToCompressedChunk(chnk) + + err = p.wr.AddCompressedChunk(ctx, cc) + if err != nil { + return err + } } } }) diff --git a/go/store/nbs/archive.go b/go/store/nbs/archive.go index 650b45a0c9..98692571fd 100644 --- a/go/store/nbs/archive.go +++ b/go/store/nbs/archive.go @@ -27,7 +27,7 @@ Chunks from the Archive. ByteSpans are arbitrary offset/lengths into the file which store (1) zstd dictionary data, and (2) compressed chunk data. Each Chunk is stored as a pair of ByteSpans (dict,data). Dictionary ByteSpans can (should) be used by multiple Chunks, so there are more ByteSpans than Chunks. The Index is used to map Chunks to ByteSpan pairs. These pairs are -called ChunkRefs, and were store them as [uint32,uint32] on disk. This allows us to quickly find the ByteSpans for a +called ChunkRefs, and we store them as [uint32,uint32] on disk. This allows us to quickly find the ByteSpans for a given Chunk with minimal processing at load time. A Dolt Archive file follows the following format: @@ -162,7 +162,7 @@ const ( archiveCheckSumSize + 1 + // version byte archiveFileSigSize - archiveFileSuffix = ".darc" + ArchiveFileSuffix = ".darc" ) /* diff --git a/go/store/nbs/archive_build.go b/go/store/nbs/archive_build.go index 349bf239d1..e30f61b7d7 100644 --- a/go/store/nbs/archive_build.go +++ b/go/store/nbs/archive_build.go @@ -57,7 +57,7 @@ func UnArchive(ctx context.Context, cs chunks.ChunkStore, smd StorageMetadata, p return err } if exists { - // We have a fast path to follow because oritinal table file is still on disk. + // We have a fast path to follow because original table file is still on disk. swapMap[arc.hash()] = orginTfId } else { // We don't have the original table file id, so we have to create a new one. diff --git a/go/store/nbs/archive_chunk_source.go b/go/store/nbs/archive_chunk_source.go index 3d3a59ec4d..23bb5d46d2 100644 --- a/go/store/nbs/archive_chunk_source.go +++ b/go/store/nbs/archive_chunk_source.go @@ -36,7 +36,7 @@ type archiveChunkSource struct { var _ chunkSource = &archiveChunkSource{} func newArchiveChunkSource(ctx context.Context, dir string, h hash.Hash, chunkCount uint32, q MemoryQuotaProvider) (archiveChunkSource, error) { - archiveFile := filepath.Join(dir, h.String()+archiveFileSuffix) + archiveFile := filepath.Join(dir, h.String()+ArchiveFileSuffix) file, size, err := openReader(archiveFile) if err != nil { @@ -141,6 +141,10 @@ func (acs archiveChunkSource) hash() hash.Hash { return acs.aRdr.footer.hash } +func (acs archiveChunkSource) name() string { + return acs.hash().String() + ArchiveFileSuffix +} + func (acs archiveChunkSource) currentSize() uint64 { return acs.aRdr.footer.fileSize } @@ -167,14 +171,54 @@ func (acs archiveChunkSource) clone() (chunkSource, error) { return archiveChunkSource{acs.file, rdr}, nil } -func (acs archiveChunkSource) getRecordRanges(_ context.Context, _ []getRecord, _ keeperF) (map[hash.Hash]Range, gcBehavior, error) { - return nil, gcBehavior_Continue, errors.New("Archive chunk source does not support getRecordRanges") +func (acs archiveChunkSource) getRecordRanges(_ context.Context, requests []getRecord, keeper keeperF) (map[hash.Hash]Range, gcBehavior, error) { + result := make(map[hash.Hash]Range, len(requests)) + for _, req := range requests { + hAddr := *req.a + idx := acs.aRdr.search(hAddr) + if idx < 0 { + // Chunk not found. + continue + } + if keeper != nil && keeper(hAddr) { + return nil, gcBehavior_Block, nil + } + + dictId, dataId := acs.aRdr.getChunkRef(idx) + dataSpan := acs.aRdr.getByteSpanByID(dataId) + dictSpan := acs.aRdr.getByteSpanByID(dictId) + + rng := Range{ + Offset: dataSpan.offset, + Length: uint32(dataSpan.length), + DictOffset: dictSpan.offset, + DictLength: uint32(dictSpan.length), + } + + result[hAddr] = rng + } + return result, gcBehavior_Continue, nil } -func (acs archiveChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { - return acs.getMany(ctx, eg, reqs, func(ctx context.Context, chk *chunks.Chunk) { - found(ctx, ChunkToCompressedChunk(*chk)) - }, keeper, stats) +func (acs archiveChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { + foundAll := true + for i, req := range reqs { + h := *req.a + toChk, err := acs.aRdr.getAsToChunker(h) + if err != nil { + return true, gcBehavior_Continue, err + } + if toChk == nil || toChk.IsEmpty() { + foundAll = false + } else { + if keeper != nil && keeper(h) { + return true, gcBehavior_Block, nil + } + found(ctx, toChk) + reqs[i].found = true + } + } + return !foundAll, gcBehavior_Continue, nil } func (acs archiveChunkSource) iterateAllChunks(ctx context.Context, cb func(chunks.Chunk), _ *Stats) error { diff --git a/go/store/nbs/archive_chunker.go b/go/store/nbs/archive_chunker.go new file mode 100644 index 0000000000..8d6bba96bd --- /dev/null +++ b/go/store/nbs/archive_chunker.go @@ -0,0 +1,65 @@ +// Copyright 2025 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nbs + +import ( + "github.com/dolthub/gozstd" + + "github.com/dolthub/dolt/go/store/chunks" + "github.com/dolthub/dolt/go/store/hash" +) + +type ArchiveToChunker struct { + h hash.Hash + dictionary *gozstd.DDict + chunkData []byte +} + +var _ ToChunker = (*ArchiveToChunker)(nil) + +func NewArchiveToChunker(h hash.Hash, dict *gozstd.DDict, chunkData []byte) ToChunker { + return ArchiveToChunker{h: h, dictionary: dict, chunkData: chunkData} +} + +func (a ArchiveToChunker) Hash() hash.Hash { + return a.h +} + +func (a ArchiveToChunker) ToChunk() (chunks.Chunk, error) { + dict := a.dictionary + data := a.chunkData + rawChunk, err := gozstd.DecompressDict(nil, data, dict) + if err != nil { + return chunks.EmptyChunk, err + } + + newChunk := chunks.NewChunk(rawChunk) + + // TODO: remove this once we have confidence in archives. + if newChunk.Hash() != a.h { + panic("Hash Mismatch!!") + } + + return newChunk, err +} + +func (a ArchiveToChunker) IsEmpty() bool { + return len(a.chunkData) == 0 +} + +func (a ArchiveToChunker) IsGhost() bool { + // archives are never ghosts. They are only instantiated when the chunk is found. + return false +} diff --git a/go/store/nbs/archive_reader.go b/go/store/nbs/archive_reader.go index c83d525bd4..a17df58a77 100644 --- a/go/store/nbs/archive_reader.go +++ b/go/store/nbs/archive_reader.go @@ -19,6 +19,7 @@ import ( "crypto/sha512" "encoding/binary" "encoding/json" + "errors" "fmt" "io" "math/bits" @@ -271,19 +272,33 @@ func (ar archiveReader) get(hash hash.Hash) ([]byte, error) { if err != nil || data == nil { return nil, err } + if dict == nil { + return nil, errors.New("runtime error: unable to get archived chunk. dictionary is nil") + } var result []byte - if dict == nil { - result, err = gozstd.Decompress(nil, data) - } else { - result, err = gozstd.DecompressDict(nil, data, dict) - } + result, err = gozstd.DecompressDict(nil, data, dict) if err != nil { return nil, err } return result, nil } +// getAsToChunker returns the chunk which is has not been decompressed. Similar to get, but with a different return type. +// If the hash is not found, a ToChunker instance with IsEmpty() == true is returned (no error) +func (ar archiveReader) getAsToChunker(h hash.Hash) (ToChunker, error) { + dict, data, err := ar.getRaw(h) + if err != nil { + return nil, err + } + + if data == nil { + return ArchiveToChunker{h, nil, []byte{}}, nil + } + + return ArchiveToChunker{h, dict, data}, nil +} + func (ar archiveReader) count() uint32 { return ar.footer.chunkCount } @@ -308,7 +323,7 @@ func (ar archiveReader) readByteSpan(bs byteSpan) ([]byte, error) { // getRaw returns the raw data for the given hash. If the hash is not found, nil is returned for both slices. Also, // no error is returned in this case. Errors will only be returned if there is an io error. // -// The data returned is still compressed, regardless of the dictionary being present or not. +// The data returned is still compressed, and the DDict is required to decompress it. func (ar archiveReader) getRaw(hash hash.Hash) (dict *gozstd.DDict, data []byte, err error) { idx := ar.search(hash) if idx < 0 { diff --git a/go/store/nbs/archive_test.go b/go/store/nbs/archive_test.go index 4de745e6a5..66164f91f3 100644 --- a/go/store/nbs/archive_test.go +++ b/go/store/nbs/archive_test.go @@ -115,41 +115,50 @@ func TestArchiveSingleChunkWithDictionary(t *testing.T) { func TestArchiverMultipleChunksMultipleDictionaries(t *testing.T) { writer := NewFixedBufferByteSink(make([]byte, 4096)) aw := newArchiveWriterWithSink(writer) - data1 := []byte{11, 11, 11, 11, 11, 11, 11, 11, 11, 11} // span 1 - dict1, _ := generateDictionary(1) // span 2 - data2 := []byte{22, 22, 22, 22, 22, 22, 22, 22, 22, 22} // span 3 - data3 := []byte{33, 33, 33, 33, 33, 33, 33, 33, 33, 33} // span 4 - data4 := []byte{44, 44, 44, 44, 44, 44, 44, 44, 44, 44} // span 5 - dict2, _ := generateDictionary(2) // span 6 + + defDict, _ := generateTerribleDefaultDictionary() // span 1 + data1 := []byte{11, 11, 11, 11, 11, 11, 11, 11, 11, 11} // span 2 + dict1, _ := generateDictionary(1) // span 3 + data2 := []byte{22, 22, 22, 22, 22, 22, 22, 22, 22, 22} // span 4 + data3 := []byte{33, 33, 33, 33, 33, 33, 33, 33, 33, 33} // span 5 + data4 := []byte{44, 44, 44, 44, 44, 44, 44, 44, 44, 44} // span 6 + dict2, _ := generateDictionary(2) // span 7 + + id, _ := aw.writeByteSpan(defDict) + assert.Equal(t, uint32(1), id) h1 := hashWithPrefix(t, 42) - id, _ := aw.writeByteSpan(data1) - assert.Equal(t, uint32(1), id) - _ = aw.stageChunk(h1, 0, 1) + id, _ = aw.writeByteSpan(data1) + assert.Equal(t, uint32(2), id) + _ = aw.stageChunk(h1, 1, 2) h2 := hashWithPrefix(t, 42) - _, _ = aw.writeByteSpan(dict1) - _, _ = aw.writeByteSpan(data2) - _ = aw.stageChunk(h2, 2, 3) + id, _ = aw.writeByteSpan(dict1) + assert.Equal(t, uint32(3), id) + id, _ = aw.writeByteSpan(data2) + assert.Equal(t, uint32(4), id) + _ = aw.stageChunk(h2, 3, 4) h3 := hashWithPrefix(t, 42) - _, _ = aw.writeByteSpan(data3) - _ = aw.stageChunk(h3, 2, 4) + id, _ = aw.writeByteSpan(data3) + assert.Equal(t, uint32(5), id) + _ = aw.stageChunk(h3, 3, 5) h4 := hashWithPrefix(t, 81) - _, _ = aw.writeByteSpan(data4) - _ = aw.stageChunk(h4, 0, 5) + id, _ = aw.writeByteSpan(data4) + assert.Equal(t, uint32(6), id) + _ = aw.stageChunk(h4, 0, 6) h5 := hashWithPrefix(t, 21) id, _ = aw.writeByteSpan(dict2) - assert.Equal(t, uint32(6), id) - _ = aw.stageChunk(h5, 6, 1) + assert.Equal(t, uint32(7), id) + _ = aw.stageChunk(h5, 7, 2) h6 := hashWithPrefix(t, 88) - _ = aw.stageChunk(h6, 6, 1) + _ = aw.stageChunk(h6, 7, 2) h7 := hashWithPrefix(t, 42) - _ = aw.stageChunk(h7, 2, 4) + _ = aw.stageChunk(h7, 3, 5) _ = aw.finalizeByteSpans() _ = aw.writeIndex() @@ -175,7 +184,7 @@ func TestArchiverMultipleChunksMultipleDictionaries(t *testing.T) { assert.False(t, aIdx.has(hashWithPrefix(t, 55))) dict, data, _ := aIdx.getRaw(h1) - assert.Nil(t, dict) + assert.NotNil(t, dict) assert.Equal(t, data1, data) dict, data, _ = aIdx.getRaw(h2) @@ -280,13 +289,17 @@ func TestMetadata(t *testing.T) { // zStd has a CRC check built into it, and it will get triggered when we // attempt to decompress a corrupted chunk. func TestArchiveChunkCorruption(t *testing.T) { - writer := NewFixedBufferByteSink(make([]byte, 1024)) + writer := NewFixedBufferByteSink(make([]byte, 4096)) aw := newArchiveWriterWithSink(writer) + + defDict, _ := generateTerribleDefaultDictionary() + _, _ = aw.writeByteSpan(defDict) + testBlob := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} _, _ = aw.writeByteSpan(testBlob) h := hashWithPrefix(t, 23) - _ = aw.stageChunk(h, 0, 1) + _ = aw.stageChunk(h, 1, 2) _ = aw.finalizeByteSpans() _ = aw.writeIndex() _ = aw.writeMetadata(nil) @@ -299,7 +312,7 @@ func TestArchiveChunkCorruption(t *testing.T) { assert.NoError(t, err) // Corrupt the data - writer.buff[3] = writer.buff[3] + 1 + writer.buff[len(defDict)+3] = writer.buff[len(defDict)+3] + 1 data, err := idx.get(h) assert.ErrorContains(t, err, "cannot decompress invalid src") @@ -592,7 +605,7 @@ func hashWithPrefix(t *testing.T, prefix uint64) hash.Hash { return hash.Hash(randomBytes) } -// For tests which need a dictionary, we generate a terrible one because we don't care about the actual compression. +// Most tests need a test dictionary. We generate a terrible one because we don't care about the actual compression. // We return both the raw form and the CDict form. func generateTerribleDefaultDictionary() ([]byte, *gozstd.CDict) { return generateDictionary(1977) @@ -676,7 +689,7 @@ func (tcs *testChunkSource) getMany(ctx context.Context, eg *errgroup.Group, req panic("never used") } -func (tcs *testChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { +func (tcs *testChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { panic("never used") } @@ -696,6 +709,10 @@ func (tcs *testChunkSource) hash() hash.Hash { panic("never used") } +func (tcs *testChunkSource) name() string { + panic("never used") +} + func (tcs *testChunkSource) reader(ctx context.Context) (io.ReadCloser, uint64, error) { panic("never used") } diff --git a/go/store/nbs/archive_writer.go b/go/store/nbs/archive_writer.go index 0223cf33c5..d555287af9 100644 --- a/go/store/nbs/archive_writer.go +++ b/go/store/nbs/archive_writer.go @@ -426,7 +426,7 @@ func (aw *archiveWriter) genFileName(path string) (string, error) { return "", err } - fileName := fmt.Sprintf("%s%s", h.String(), archiveFileSuffix) + fileName := fmt.Sprintf("%s%s", h.String(), ArchiveFileSuffix) fullPath := filepath.Join(path, fileName) return fullPath, nil } diff --git a/go/store/nbs/chunk_fetcher.go b/go/store/nbs/chunk_fetcher.go index 2be1a0aa78..8b4aae88a4 100644 --- a/go/store/nbs/chunk_fetcher.go +++ b/go/store/nbs/chunk_fetcher.go @@ -46,7 +46,7 @@ type ChunkFetcher interface { CloseSend() error - Recv(context.Context) (CompressedChunk, error) + Recv(context.Context) (ToChunker, error) Close() error } diff --git a/go/store/nbs/chunk_source_adapter.go b/go/store/nbs/chunk_source_adapter.go index f86d7527b4..013351ef6c 100644 --- a/go/store/nbs/chunk_source_adapter.go +++ b/go/store/nbs/chunk_source_adapter.go @@ -29,6 +29,10 @@ func (csa chunkSourceAdapter) hash() hash.Hash { return csa.h } +func (csa chunkSourceAdapter) name() string { + return csa.h.String() +} + func newReaderFromIndexData(ctx context.Context, q MemoryQuotaProvider, idxData []byte, name hash.Hash, tra tableReaderAt, blockSize uint64) (cs chunkSource, err error) { index, err := parseTableIndexByCopy(ctx, idxData, q) if err != nil { diff --git a/go/store/nbs/cmp_chunk_table_writer.go b/go/store/nbs/cmp_chunk_table_writer.go index 28cdde9563..e880afdcfe 100644 --- a/go/store/nbs/cmp_chunk_table_writer.go +++ b/go/store/nbs/cmp_chunk_table_writer.go @@ -75,17 +75,22 @@ func (tw *CmpChunkTableWriter) GetMD5() []byte { } // AddCmpChunk adds a compressed chunk -func (tw *CmpChunkTableWriter) AddCmpChunk(c CompressedChunk) error { - if c.IsGhost() { +func (tw *CmpChunkTableWriter) AddCmpChunk(tc ToChunker) error { + if tc.IsGhost() { // Ghost chunks cannot be written to a table file. They should // always be filtered by the write processes before landing // here. return ErrGhostChunkRequested } - if len(c.CompressedData) == 0 { + if tc.IsEmpty() { panic("NBS blocks cannot be zero length") } + c, ok := tc.(CompressedChunk) + if !ok { + panic("runtime error: Require a CompressedChunk instance") + } + uncmpLen, err := snappy.DecodedLen(c.CompressedData) if err != nil { diff --git a/go/store/nbs/cmp_chunk_table_writer_test.go b/go/store/nbs/cmp_chunk_table_writer_test.go index 6f0508b2e8..592ca683e4 100644 --- a/go/store/nbs/cmp_chunk_table_writer_test.go +++ b/go/store/nbs/cmp_chunk_table_writer_test.go @@ -48,10 +48,10 @@ func TestCmpChunkTableWriter(t *testing.T) { } reqs := toGetRecords(hashes) - found := make([]CompressedChunk, 0) + found := make([]ToChunker, 0) eg, egCtx := errgroup.WithContext(ctx) - _, _, err = tr.getManyCompressed(egCtx, eg, reqs, func(ctx context.Context, c CompressedChunk) { found = append(found, c) }, nil, &Stats{}) + _, _, err = tr.getManyCompressed(egCtx, eg, reqs, func(ctx context.Context, c ToChunker) { found = append(found, c) }, nil, &Stats{}) require.NoError(t, err) require.NoError(t, eg.Wait()) diff --git a/go/store/nbs/empty_chunk_source.go b/go/store/nbs/empty_chunk_source.go index 60b7f953d2..a341251ff7 100644 --- a/go/store/nbs/empty_chunk_source.go +++ b/go/store/nbs/empty_chunk_source.go @@ -50,7 +50,7 @@ func (ecs emptyChunkSource) getMany(ctx context.Context, eg *errgroup.Group, req return true, gcBehavior_Continue, nil } -func (ecs emptyChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { +func (ecs emptyChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { return true, gcBehavior_Continue, nil } @@ -66,6 +66,10 @@ func (ecs emptyChunkSource) hash() hash.Hash { return hash.Hash{} } +func (ecs emptyChunkSource) name() string { + return ecs.hash().String() +} + func (ecs emptyChunkSource) index() (tableIndex, error) { return onHeapTableIndex{}, nil } diff --git a/go/store/nbs/file_table_reader.go b/go/store/nbs/file_table_reader.go index a95c2109c3..d808b81563 100644 --- a/go/store/nbs/file_table_reader.go +++ b/go/store/nbs/file_table_reader.go @@ -54,7 +54,7 @@ func tableFileExists(ctx context.Context, dir string, h hash.Hash) (bool, error) } func archiveFileExists(ctx context.Context, dir string, h hash.Hash) (bool, error) { - darc := fmt.Sprintf("%s%s", h.String(), archiveFileSuffix) + darc := fmt.Sprintf("%s%s", h.String(), ArchiveFileSuffix) path := filepath.Join(dir, darc) _, err := os.Stat(path) @@ -169,6 +169,10 @@ func (ftr *fileTableReader) hash() hash.Hash { return ftr.h } +func (ftr *fileTableReader) name() string { + return ftr.h.String() +} + func (ftr *fileTableReader) Close() error { return ftr.tableReader.close() } diff --git a/go/store/nbs/gc_copier.go b/go/store/nbs/gc_copier.go index 77d9c46b12..4a49b6179d 100644 --- a/go/store/nbs/gc_copier.go +++ b/go/store/nbs/gc_copier.go @@ -56,7 +56,7 @@ func newGarbageCollectionCopier(tfp tableFilePersister) (*gcCopier, error) { return &gcCopier{writer, tfp}, nil } -func (gcc *gcCopier) addChunk(ctx context.Context, c CompressedChunk) error { +func (gcc *gcCopier) addChunk(ctx context.Context, c ToChunker) error { return gcc.writer.AddCmpChunk(c) } diff --git a/go/store/nbs/generational_chunk_store.go b/go/store/nbs/generational_chunk_store.go index de4257176d..9cc588fbe0 100644 --- a/go/store/nbs/generational_chunk_store.go +++ b/go/store/nbs/generational_chunk_store.go @@ -28,6 +28,7 @@ import ( ) var _ chunks.ChunkStore = (*GenerationalNBS)(nil) +var _ chunks.GenerationalCS = (*GenerationalNBS)(nil) var _ chunks.TableFileStore = (*GenerationalNBS)(nil) var _ chunks.GenerationalCS = (*GenerationalNBS)(nil) var _ chunks.ChunkStoreGarbageCollector = (*GenerationalNBS)(nil) @@ -144,14 +145,14 @@ func (gcs *GenerationalNBS) GetMany(ctx context.Context, hashes hash.HashSet, fo return gcs.ghostGen.GetMany(ctx, notFound, found) } -func (gcs *GenerationalNBS) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk)) error { +func (gcs *GenerationalNBS) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker)) error { return gcs.getManyCompressed(ctx, hashes, found, gcDependencyMode_TakeDependency) } -func (gcs *GenerationalNBS) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk), gcDepMode gcDependencyMode) error { +func (gcs *GenerationalNBS) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker), gcDepMode gcDependencyMode) error { var mu sync.Mutex notInOldGen := hashes.Copy() - err := gcs.oldGen.getManyCompressed(ctx, hashes, func(ctx context.Context, chunk CompressedChunk) { + err := gcs.oldGen.getManyCompressed(ctx, hashes, func(ctx context.Context, chunk ToChunker) { mu.Lock() delete(notInOldGen, chunk.Hash()) mu.Unlock() @@ -165,7 +166,7 @@ func (gcs *GenerationalNBS) getManyCompressed(ctx context.Context, hashes hash.H } notFound := notInOldGen.Copy() - err = gcs.newGen.getManyCompressed(ctx, notInOldGen, func(ctx context.Context, chunk CompressedChunk) { + err = gcs.newGen.getManyCompressed(ctx, notInOldGen, func(ctx context.Context, chunk ToChunker) { mu.Lock() delete(notFound, chunk.Hash()) mu.Unlock() diff --git a/go/store/nbs/ghost_store.go b/go/store/nbs/ghost_store.go index 38fdf5247c..b6d86ee04c 100644 --- a/go/store/nbs/ghost_store.go +++ b/go/store/nbs/ghost_store.go @@ -90,11 +90,11 @@ func (g GhostBlockStore) GetMany(ctx context.Context, hashes hash.HashSet, found return nil } -func (g GhostBlockStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk)) error { +func (g GhostBlockStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker)) error { return g.getManyCompressed(ctx, hashes, found, gcDependencyMode_TakeDependency) } -func (g GhostBlockStore) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk), gcDepMode gcDependencyMode) error { +func (g GhostBlockStore) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker), gcDepMode gcDependencyMode) error { for h := range hashes { if g.skippedRefs.Has(h) { found(ctx, NewGhostCompressedChunk(h)) diff --git a/go/store/nbs/ghost_store_test.go b/go/store/nbs/ghost_store_test.go index e6fc6146f7..aa347b1a6b 100644 --- a/go/store/nbs/ghost_store_test.go +++ b/go/store/nbs/ghost_store_test.go @@ -74,8 +74,8 @@ func TestGhostBlockStore(t *testing.T) { require.Equal(t, ghost, got[0].Hash()) }) t.Run("GetManyCompressed", func(t *testing.T) { - var got []CompressedChunk - err := bs.GetManyCompressed(ctx, hash.NewHashSet(absent, ghost), func(_ context.Context, c CompressedChunk) { + var got []ToChunker + err := bs.GetManyCompressed(ctx, hash.NewHashSet(absent, ghost), func(_ context.Context, c ToChunker) { got = append(got, c) }) require.NoError(t, err) diff --git a/go/store/nbs/journal_chunk_source.go b/go/store/nbs/journal_chunk_source.go index da977441ce..11126492b0 100644 --- a/go/store/nbs/journal_chunk_source.go +++ b/go/store/nbs/journal_chunk_source.go @@ -96,7 +96,7 @@ type journalRecord struct { } func (s journalChunkSource) getMany(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, *chunks.Chunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { - return s.getManyCompressed(ctx, eg, reqs, func(ctx context.Context, cc CompressedChunk) { + return s.getManyCompressed(ctx, eg, reqs, func(ctx context.Context, cc ToChunker) { ch, err := cc.ToChunk() if err != nil { eg.Go(func() error { @@ -115,7 +115,7 @@ func (s journalChunkSource) getMany(ctx context.Context, eg *errgroup.Group, req // and then (4) asynchronously perform reads. We release the journal read // lock after returning when all reads are completed, which can be after the // function returns. -func (s journalChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { +func (s journalChunkSource) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { defer trace.StartRegion(ctx, "journalChunkSource.getManyCompressed").End() var remaining bool @@ -185,6 +185,10 @@ func (s journalChunkSource) hash() hash.Hash { return journalAddr } +func (s journalChunkSource) name() string { + return s.hash().String() +} + // reader implements chunkSource. func (s journalChunkSource) reader(ctx context.Context) (io.ReadCloser, uint64, error) { rdr, sz, err := s.journal.snapshot(ctx) diff --git a/go/store/nbs/mem_table.go b/go/store/nbs/mem_table.go index 1fd8c0ffcd..ada8b66961 100644 --- a/go/store/nbs/mem_table.go +++ b/go/store/nbs/mem_table.go @@ -193,7 +193,7 @@ func (mt *memTable) getMany(ctx context.Context, eg *errgroup.Group, reqs []getR return remaining, gcBehavior_Continue, nil } -func (mt *memTable) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { +func (mt *memTable) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { var remaining bool for i, r := range reqs { data := mt.chunks[*r.a] diff --git a/go/store/nbs/mem_table_test.go b/go/store/nbs/mem_table_test.go index 42a1888a98..3e97be099c 100644 --- a/go/store/nbs/mem_table_test.go +++ b/go/store/nbs/mem_table_test.go @@ -306,7 +306,7 @@ func (crg chunkReaderGroup) getMany(ctx context.Context, eg *errgroup.Group, req return true, gcBehavior_Continue, nil } -func (crg chunkReaderGroup) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { +func (crg chunkReaderGroup) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { for _, haver := range crg { remaining, gcb, err := haver.getManyCompressed(ctx, eg, reqs, found, keeper, stats) if err != nil { diff --git a/go/store/nbs/metadata.go b/go/store/nbs/metadata.go index 6f443efb0b..d8cfb79efb 100644 --- a/go/store/nbs/metadata.go +++ b/go/store/nbs/metadata.go @@ -15,6 +15,7 @@ package nbs import ( + "errors" "os" "path/filepath" @@ -22,34 +23,38 @@ import ( "github.com/dolthub/dolt/go/store/hash" ) -type StorageType int - -const ( - Journal StorageType = iota - TableFileNewGen - TableFileOldGen - Archive -) - type ArchiveMetadata struct { originalTableFileId string } +type TableFileFormat int + +const ( + TypeNoms TableFileFormat = iota + TypeArchive +) + type StorageArtifact struct { - id hash.Hash - path string - storageType StorageType + // ID of the storage artifact. This is uses in the manifest to identify the artifact, but it is not the file name. + // as archives has a suffix. + id hash.Hash + // path to the storage artifact. + path string + // storageType is the type of the storage artifact. + storageType TableFileFormat + // arcMetadata is additional metadata for archive files. it is only set for storageType == TypeArchive. arcMetadata *ArchiveMetadata } type StorageMetadata struct { + // root is the path to storage. Specifically, it contains a .dolt directory. root string artifacts []StorageArtifact } func (sm *StorageMetadata) ArchiveFilesPresent() bool { for _, artifact := range sm.artifacts { - if artifact.storageType == Archive { + if artifact.storageType == TypeArchive { return true } } @@ -60,7 +65,7 @@ func (sm *StorageMetadata) ArchiveFilesPresent() bool { func (sm *StorageMetadata) RevertMap() map[hash.Hash]hash.Hash { revertMap := make(map[hash.Hash]hash.Hash) for _, artifact := range sm.artifacts { - if artifact.storageType == Archive { + if artifact.storageType == TypeArchive { md := artifact.arcMetadata revertMap[artifact.id] = hash.Parse(md.originalTableFileId) } @@ -68,6 +73,8 @@ func (sm *StorageMetadata) RevertMap() map[hash.Hash]hash.Hash { return revertMap } +// oldGenTableExists returns true if the table file exists in the oldgen directory. This is a file system check for +// a table file we have no record of, but may be useful in the process of reverting an archive operation. func (sm *StorageMetadata) oldGenTableExists(id hash.Hash) (bool, error) { path := filepath.Join(sm.root, ".dolt", "noms", "oldgen", id.String()) _, err := os.Stat(path) @@ -88,20 +95,9 @@ func GetStorageMetadata(path string) (StorageMetadata, error) { return StorageMetadata{}, err } - // TODO: new gen and journal information in storage metadata will be useful in the future. - // newGen := filepath.Join(path, ".dolt", "noms") - // newgenManifest := filepath.Join(newGen, "manifest") - - oldgen := filepath.Join(path, ".dolt", "noms", "oldgen") - oldgenManifest := filepath.Join(oldgen, "manifest") - - // If there is not oldgen manifest, then GC has never been run. Which is fine. We just don't have any oldgen. - if _, err := os.Stat(oldgenManifest); err != nil { - return StorageMetadata{}, nil - } - - // create a io.Reader for the manifest file - manifestReader, err := os.Open(oldgenManifest) + newGen := filepath.Join(path, ".dolt", "noms") + newgenManifest := filepath.Join(newGen, "manifest") + manifestReader, err := os.Open(newgenManifest) if err != nil { return StorageMetadata{}, err } @@ -116,53 +112,90 @@ func GetStorageMetadata(path string) (StorageMetadata, error) { // for each table in the manifest, get the table spec for i := 0; i < manifest.NumTableSpecs(); i++ { tableSpecInfo := manifest.GetTableSpecInfo(i) - - // If the oldgen/name exists, it's not an archive. If it exists with a .darc suffix, then it's an archive. - tfName := tableSpecInfo.GetName() - fullPath := filepath.Join(oldgen, tfName) - _, err := os.Stat(fullPath) - if err == nil { - // exists. Not an archive. - artifacts = append(artifacts, StorageArtifact{ - id: hash.Parse(tfName), - path: fullPath, - storageType: TableFileOldGen, - }) - } else if os.IsNotExist(err) { - arcName := tfName + ".darc" - arcPath := filepath.Join(oldgen, arcName) - _, err := os.Stat(arcPath) - if err == nil { - // reader for the path. State. call - reader, fileSize, err := openReader(arcPath) - if err != nil { - return StorageMetadata{}, err - } - - arcMetadata, err := newArchiveMetadata(reader, fileSize) - if err != nil { - return StorageMetadata{}, err - } - - artifacts = append(artifacts, StorageArtifact{ - id: hash.Parse(tfName), - path: arcPath, - storageType: Archive, - arcMetadata: arcMetadata, - }) - } else { - // any error is bad here. If the files don't exist, then the manifest is no good. - return StorageMetadata{}, err - } - } else { - // some other error. + artifact, err := buildArtifact(tableSpecInfo, newGen) + if err != nil { return StorageMetadata{}, err } + artifacts = append(artifacts, artifact) + } + + oldgen := filepath.Join(newGen, "oldgen") + oldgenManifest := filepath.Join(oldgen, "manifest") + + // If there is no oldgen manifest, then GC has never been run. Which is fine. We just don't have any oldgen. + if _, err := os.Stat(oldgenManifest); err != nil { + return StorageMetadata{path, artifacts}, nil + } + + manifestReader, err = os.Open(oldgenManifest) + if err != nil { + return StorageMetadata{}, err + } + manifest, err = ParseManifest(manifestReader) + if err != nil { + return StorageMetadata{}, err + } + + for i := 0; i < manifest.NumTableSpecs(); i++ { + tableSpecInfo := manifest.GetTableSpecInfo(i) + + artifact, err := buildArtifact(tableSpecInfo, oldgen) + if err != nil { + return StorageMetadata{}, err + } + artifacts = append(artifacts, artifact) } return StorageMetadata{path, artifacts}, nil } +func buildArtifact(info TableSpecInfo, genPath string) (StorageArtifact, error) { + tfName := info.GetName() + + // This code is going to be removed as soon as backup supports archives. + archive := false + fullPath := filepath.Join(genPath, tfName) + + _, err := os.Stat(fullPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + fullPath = filepath.Join(genPath, tfName+ArchiveFileSuffix) + } else { + return StorageArtifact{}, err + } + _, err = os.Stat(fullPath) + if err != nil { + return StorageArtifact{}, err + } + archive = true + } + + if !archive { + return StorageArtifact{ + id: hash.Parse(tfName), + path: fullPath, + storageType: TypeNoms, + }, nil + } else { + reader, fileSize, err := openReader(fullPath) + if err != nil { + return StorageArtifact{}, err + } + + arcMetadata, err := newArchiveMetadata(reader, fileSize) + if err != nil { + return StorageArtifact{}, err + } + + return StorageArtifact{ + id: hash.Parse(tfName), + path: fullPath, + storageType: TypeArchive, + arcMetadata: arcMetadata, + }, nil + } +} + func validateDir(path string) error { info, err := os.Stat(path) diff --git a/go/store/nbs/nbs_metrics_wrapper.go b/go/store/nbs/nbs_metrics_wrapper.go index d3a32993bf..769639985d 100644 --- a/go/store/nbs/nbs_metrics_wrapper.go +++ b/go/store/nbs/nbs_metrics_wrapper.go @@ -99,7 +99,7 @@ func (nbsMW *NBSMetricWrapper) PruneTableFiles(ctx context.Context) error { // GetManyCompressed gets the compressed Chunks with |hashes| from the store. On return, // |found| will have been fully sent all chunks which have been // found. Any non-present chunks will silently be ignored. -func (nbsMW *NBSMetricWrapper) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk)) error { +func (nbsMW *NBSMetricWrapper) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker)) error { atomic.AddInt32(&nbsMW.TotalChunkGets, int32(len(hashes))) return nbsMW.nbs.GetManyCompressed(ctx, hashes, found) } diff --git a/go/store/nbs/store.go b/go/store/nbs/store.go index 20e48cc76e..78ab22843b 100644 --- a/go/store/nbs/store.go +++ b/go/store/nbs/store.go @@ -67,7 +67,6 @@ const ( defaultMaxTables = 256 defaultManifestCacheSize = 1 << 23 // 8MB - preflushChunkCount = 8 ) var ( @@ -85,7 +84,7 @@ func makeGlobalCaches() { type NBSCompressedChunkStore interface { chunks.ChunkStore - GetManyCompressed(context.Context, hash.HashSet, func(context.Context, CompressedChunk)) error + GetManyCompressed(context.Context, hash.HashSet, func(context.Context, ToChunker)) error } type gcDependencyMode int @@ -96,7 +95,7 @@ const ( ) type CompressedChunkStoreForGC interface { - getManyCompressed(context.Context, hash.HashSet, func(context.Context, CompressedChunk), gcDependencyMode) error + getManyCompressed(context.Context, hash.HashSet, func(context.Context, ToChunker), gcDependencyMode) error } type NomsBlockStore struct { @@ -143,8 +142,10 @@ var _ chunks.ChunkStoreGarbageCollector = &NomsBlockStore{} const hasCacheSize = 100000 type Range struct { - Offset uint64 - Length uint32 + Offset uint64 + Length uint32 + DictOffset uint64 + DictLength uint32 } // ChunkJournal returns the ChunkJournal in use by this NomsBlockStore, or nil if no ChunkJournal is being used. @@ -156,19 +157,26 @@ func (nbs *NomsBlockStore) ChunkJournal() *ChunkJournal { } func (nbs *NomsBlockStore) GetChunkLocationsWithPaths(ctx context.Context, hashes hash.HashSet) (map[string]map[hash.Hash]Range, error) { - locs, err := nbs.GetChunkLocations(ctx, hashes) + sourcesToRanges, err := nbs.getChunkLocations(ctx, hashes) if err != nil { return nil, err } - toret := make(map[string]map[hash.Hash]Range, len(locs)) - for k, v := range locs { - toret[k.String()] = v + + res := make(map[string]map[hash.Hash]Range, len(sourcesToRanges)) + for csP, ranges := range sourcesToRanges { + cs := *csP + suffix := "" + if _, ok := cs.(archiveChunkSource); ok { + suffix = ArchiveFileSuffix + } + + res[cs.hash().String()+suffix] = ranges } - return toret, nil + return res, nil } -func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.HashSet) (map[hash.Hash]map[hash.Hash]Range, error) { - fn := func(css chunkSourceSet, gr []getRecord, ranges map[hash.Hash]map[hash.Hash]Range, keeper keeperF) (gcBehavior, error) { +func (nbs *NomsBlockStore) getChunkLocations(ctx context.Context, hashes hash.HashSet) (map[*chunkSource]map[hash.Hash]Range, error) { + fn := func(css chunkSourceSet, gr []getRecord, ranges map[*chunkSource]map[hash.Hash]Range, keeper keeperF) (gcBehavior, error) { for _, cs := range css { rng, gcb, err := cs.getRecordRanges(ctx, gr, keeper) if err != nil { @@ -177,14 +185,16 @@ func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.Ha if gcb != gcBehavior_Continue { return gcb, nil } + if len(rng) == 0 { + continue + } - h := hash.Hash(cs.hash()) - if m, ok := ranges[h]; ok { + if m, ok := ranges[&cs]; ok { for k, v := range rng { m[k] = v } } else { - ranges[h] = rng + ranges[&cs] = rng } } return gcBehavior_Continue, nil @@ -196,7 +206,7 @@ func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.Ha nbs.mu.Unlock() gr := toGetRecords(hashes) - ranges := make(map[hash.Hash]map[hash.Hash]Range) + ranges := make(map[*chunkSource]map[hash.Hash]Range) gcb, err := fn(tables.upstream, gr, ranges, keeper) if needsContinue, err := nbs.handleUnlockedRead(ctx, gcb, endRead, err); err != nil { @@ -214,6 +224,20 @@ func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.Ha return ranges, nil } + +} + +func (nbs *NomsBlockStore) GetChunkLocations(ctx context.Context, hashes hash.HashSet) (map[hash.Hash]map[hash.Hash]Range, error) { + sourcesToRanges, err := nbs.getChunkLocations(ctx, hashes) + if err != nil { + return nil, err + } + res := make(map[hash.Hash]map[hash.Hash]Range, len(hashes)) + for csP, ranges := range sourcesToRanges { + cs := *csP + res[cs.hash()] = ranges + } + return res, nil } func (nbs *NomsBlockStore) handleUnlockedRead(ctx context.Context, gcb gcBehavior, endRead func(), err error) (bool, error) { @@ -901,11 +925,11 @@ func (nbs *NomsBlockStore) GetMany(ctx context.Context, hashes hash.HashSet, fou ) } -func (nbs *NomsBlockStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk)) error { +func (nbs *NomsBlockStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker)) error { return nbs.getManyCompressed(ctx, hashes, found, gcDependencyMode_TakeDependency) } -func (nbs *NomsBlockStore) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, CompressedChunk), gcDepMode gcDependencyMode) error { +func (nbs *NomsBlockStore) getManyCompressed(ctx context.Context, hashes hash.HashSet, found func(context.Context, ToChunker), gcDepMode gcDependencyMode) error { ctx, span := tracer.Start(ctx, "nbs.GetManyCompressed", trace.WithAttributes(attribute.Int("num_hashes", len(hashes)))) defer span.End() return nbs.getManyWithFunc(ctx, hashes, gcDepMode, @@ -1482,8 +1506,9 @@ func (nbs *NomsBlockStore) StatsSummary() string { // tableFile is our implementation of TableFile. type tableFile struct { - info TableSpecInfo - open func(ctx context.Context) (io.ReadCloser, uint64, error) + info TableSpecInfo + open func(ctx context.Context) (io.ReadCloser, uint64, error) + suffix string } // LocationPrefix @@ -1491,6 +1516,10 @@ func (tf tableFile) LocationPrefix() string { return "" } +func (tf tableFile) LocationSuffix() string { + return tf.suffix +} + // FileID gets the id of the file func (tf tableFile) FileID() string { return tf.info.GetName() @@ -1555,12 +1584,18 @@ func getTableFiles(css map[hash.Hash]chunkSource, contents manifestContents, num if !ok { return nil, ErrSpecWithoutChunkSource } + tableFiles = append(tableFiles, newTableFile(cs, info)) } return tableFiles, nil } func newTableFile(cs chunkSource, info tableSpec) tableFile { + s := "" + if _, ok := cs.(archiveChunkSource); ok { + s = ArchiveFileSuffix + } + return tableFile{ info: info, open: func(ctx context.Context) (io.ReadCloser, uint64, error) { @@ -1570,6 +1605,7 @@ func newTableFile(cs chunkSource, info tableSpec) tableFile { } return r, s, nil }, + suffix: s, } } @@ -1625,7 +1661,7 @@ func (nbs *NomsBlockStore) Path() (string, bool) { } // WriteTableFile will read a table file from the provided reader and write it to the TableFileStore -func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) error { +func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileName string, numChunks int, contentHash []byte, getRd func() (io.ReadCloser, uint64, error)) error { tfp, ok := nbs.p.(tableFilePersister) if !ok { return errors.New("Not implemented") @@ -1636,7 +1672,7 @@ func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, nu return err } defer r.Close() - return tfp.CopyTableFile(ctx, r, fileId, sz, uint32(numChunks)) + return tfp.CopyTableFile(ctx, r, fileName, sz, uint32(numChunks)) } // AddTableFilesToManifest adds table files to the manifest @@ -1982,25 +2018,25 @@ func (i *markAndSweeper) SaveHashes(ctx context.Context, hashes []hash.Hash) err found := 0 var addErr error - err = i.src.getManyCompressed(ctx, toVisit, func(ctx context.Context, cc CompressedChunk) { + err = i.src.getManyCompressed(ctx, toVisit, func(ctx context.Context, tc ToChunker) { mu.Lock() defer mu.Unlock() if addErr != nil { return } found += 1 - if cc.IsGhost() { + if tc.IsGhost() { // Ghost chunks encountered on the walk can be left alone --- they // do not bring their dependencies, and because of how generational // store works, they will still be ghost chunks // in the store after the GC is finished. return } - addErr = i.gcc.addChunk(ctx, cc) + addErr = i.gcc.addChunk(ctx, tc) if addErr != nil { return } - c, err := cc.ToChunk() + c, err := tc.ToChunk() if err != nil { addErr = err return diff --git a/go/store/nbs/table.go b/go/store/nbs/table.go index c703f83997..1aa5d2e7be 100644 --- a/go/store/nbs/table.go +++ b/go/store/nbs/table.go @@ -187,7 +187,7 @@ type extractRecord struct { err error } -// Returned by read methods that take a |keeperFunc|, this lets a +// Returned by read methods that take a |keeperF|, this lets a // caller know whether the operation was successful or if it needs to // be retried. It may need to be retried if a GC is in progress but // the dependencies indicated by the operation cannot be added to the @@ -202,6 +202,10 @@ const ( gcBehavior_Block = true ) +// keeperF is a function that takes a hash.Hash and returns true if the hash is used by the GC system to indicate +// that the chunk requested may not be present in the future, and therefore |gcBehavior_Block| should be returned. This +// is used to allow read/write ops to the store by non-GC processes while GC is underway. The |keeperF| may be nil, +// in which case GC is not underway. If it's non-nil, and return false, it's ok to proceed with the operation (|gcBehavior_Continue|) type keeperF func(hash.Hash) bool type chunkReader interface { @@ -221,7 +225,7 @@ type chunkReader interface { // getManyCompressed sets getRecord.found to true, and calls |found| for each present getRecord query. // It returns true if any getRecord query was not found in this chunkReader. - getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) + getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) // count returns the chunk count for this chunkReader. count() (uint32, error) @@ -239,6 +243,10 @@ type chunkSource interface { // hash returns the hash address of this chunkSource. hash() hash.Hash + // name is the on disk short name for this chunkSource. Classically, this was a hash. Having files + // with suffixes (eg darc) was useful. + name() string + // opens a Reader to the first byte of the chunkData segment of this table. reader(context.Context) (io.ReadCloser, uint64, error) diff --git a/go/store/nbs/table_reader.go b/go/store/nbs/table_reader.go index de66ffd27b..c63d29d1f4 100644 --- a/go/store/nbs/table_reader.go +++ b/go/store/nbs/table_reader.go @@ -38,6 +38,13 @@ import ( // Do not read more than 128MB at a time. const maxReadSize = 128 * 1024 * 1024 +type ToChunker interface { + Hash() hash.Hash + ToChunk() (chunks.Chunk, error) + IsEmpty() bool + IsGhost() bool +} + // CompressedChunk represents a chunk of data in a table file which is still compressed via snappy. type CompressedChunk struct { // H is the hash of the chunk @@ -53,6 +60,8 @@ type CompressedChunk struct { ghost bool } +var _ ToChunker = CompressedChunk{} + // NewCompressedChunk creates a CompressedChunk func NewCompressedChunk(h hash.Hash, buff []byte) (CompressedChunk, error) { dataLen := uint64(len(buff)) - checksumSize @@ -323,10 +332,10 @@ var _ chunkReader = tableReader{} func (tr tableReader) readCompressedAtOffsets( ctx context.Context, rb readBatch, - found func(context.Context, CompressedChunk), + found func(context.Context, ToChunker), stats *Stats, ) error { - return tr.readAtOffsetsWithCB(ctx, rb, stats, func(ctx context.Context, cmp CompressedChunk) error { + return tr.readAtOffsetsWithCB(ctx, rb, stats, func(ctx context.Context, cmp ToChunker) error { found(ctx, cmp) return nil }) @@ -338,7 +347,7 @@ func (tr tableReader) readAtOffsets( found func(context.Context, *chunks.Chunk), stats *Stats, ) error { - return tr.readAtOffsetsWithCB(ctx, rb, stats, func(ctx context.Context, cmp CompressedChunk) error { + return tr.readAtOffsetsWithCB(ctx, rb, stats, func(ctx context.Context, cmp ToChunker) error { chk, err := cmp.ToChunk() if err != nil { @@ -354,7 +363,7 @@ func (tr tableReader) readAtOffsetsWithCB( ctx context.Context, rb readBatch, stats *Stats, - cb func(ctx context.Context, cmp CompressedChunk) error, + cb func(ctx context.Context, cmp ToChunker) error, ) error { readLength := rb.End() - rb.Start() buff := make([]byte, readLength) @@ -405,7 +414,7 @@ func (tr tableReader) getMany( err = tr.getManyAtOffsets(ctx, eg, offsetRecords, found, stats) return remaining, gcBehavior_Continue, err } -func (tr tableReader) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { +func (tr tableReader) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { // Pass #1: Iterate over |reqs| and |tr.prefixes| (both sorted by address) and build the set // of table locations which must be read in order to satisfy the getMany operation. offsetRecords, remaining, gcb, err := tr.findOffsets(reqs, keeper) @@ -419,7 +428,7 @@ func (tr tableReader) getManyCompressed(ctx context.Context, eg *errgroup.Group, return remaining, gcBehavior_Continue, err } -func (tr tableReader) getManyCompressedAtOffsets(ctx context.Context, eg *errgroup.Group, offsetRecords offsetRecSlice, found func(context.Context, CompressedChunk), stats *Stats) error { +func (tr tableReader) getManyCompressedAtOffsets(ctx context.Context, eg *errgroup.Group, offsetRecords offsetRecSlice, found func(context.Context, ToChunker), stats *Stats) error { return tr.getManyAtOffsetsWithReadFunc(ctx, eg, offsetRecords, stats, func( ctx context.Context, rb readBatch, diff --git a/go/store/nbs/table_set.go b/go/store/nbs/table_set.go index b897b3068d..939b8cbf58 100644 --- a/go/store/nbs/table_set.go +++ b/go/store/nbs/table_set.go @@ -228,7 +228,7 @@ func (ts tableSet) getMany(ctx context.Context, eg *errgroup.Group, reqs []getRe return f(ts.upstream) } -func (ts tableSet) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, CompressedChunk), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { +func (ts tableSet) getManyCompressed(ctx context.Context, eg *errgroup.Group, reqs []getRecord, found func(context.Context, ToChunker), keeper keeperF, stats *Stats) (bool, gcBehavior, error) { f := func(css chunkSourceSet) (bool, gcBehavior, error) { for _, haver := range css { remaining, gcb, err := haver.getManyCompressed(ctx, eg, reqs, found, keeper, stats) @@ -544,7 +544,7 @@ func (ts tableSet) rebase(ctx context.Context, specs []tableSpec, srcs chunkSour } else if existing, ok := srcs[spec.name]; ok { cs, err = existing.clone() } else { - cs, err = ts.p.Open(ctx, spec.name, spec.chunkCount, stats) // NM4 - spec.name is the tf/arch name. + cs, err = ts.p.Open(ctx, spec.name, spec.chunkCount, stats) } if err != nil { return err diff --git a/integration-tests/bats/archive-test-repo/noms/8p5e2m6skovfdjlh4jg3llr8sfvu384l b/integration-tests/bats/archive-test-repo/noms/8p5e2m6skovfdjlh4jg3llr8sfvu384l new file mode 100644 index 0000000000..aa14b7ce0e Binary files /dev/null and b/integration-tests/bats/archive-test-repo/noms/8p5e2m6skovfdjlh4jg3llr8sfvu384l differ diff --git a/integration-tests/bats/archive-test-repo/noms/LOCK b/integration-tests/bats/archive-test-repo/noms/LOCK new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integration-tests/bats/archive-test-repo/noms/manifest b/integration-tests/bats/archive-test-repo/noms/manifest new file mode 100644 index 0000000000..924d8f97c5 --- /dev/null +++ b/integration-tests/bats/archive-test-repo/noms/manifest @@ -0,0 +1 @@ +5:__DOLT__:6eseiohaelofp485e95ti1pgqe5qrseg:att0kn4lt0uqce5mdqf01ni9ediahoo4:6eseiohaelofp485e95ti1pgqe5qrseg:8p5e2m6skovfdjlh4jg3llr8sfvu384l:2 \ No newline at end of file diff --git a/integration-tests/bats/archive-test-repo/noms/oldgen/29o8a3uevcpr15tilcemb3s438edmoog.darc b/integration-tests/bats/archive-test-repo/noms/oldgen/29o8a3uevcpr15tilcemb3s438edmoog.darc new file mode 100644 index 0000000000..1b938008b8 Binary files /dev/null and b/integration-tests/bats/archive-test-repo/noms/oldgen/29o8a3uevcpr15tilcemb3s438edmoog.darc differ diff --git a/integration-tests/bats/archive-test-repo/noms/oldgen/LOCK b/integration-tests/bats/archive-test-repo/noms/oldgen/LOCK new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integration-tests/bats/archive-test-repo/noms/oldgen/dnu4lr5j8sstbj5usbld7alsnuj5nf23.darc b/integration-tests/bats/archive-test-repo/noms/oldgen/dnu4lr5j8sstbj5usbld7alsnuj5nf23.darc new file mode 100644 index 0000000000..c13a757ab4 Binary files /dev/null and b/integration-tests/bats/archive-test-repo/noms/oldgen/dnu4lr5j8sstbj5usbld7alsnuj5nf23.darc differ diff --git a/integration-tests/bats/archive-test-repo/noms/oldgen/manifest b/integration-tests/bats/archive-test-repo/noms/oldgen/manifest new file mode 100644 index 0000000000..95b6efd291 --- /dev/null +++ b/integration-tests/bats/archive-test-repo/noms/oldgen/manifest @@ -0,0 +1 @@ +5:__DOLT__:c3qnrebcud1nuibb18lmitlc442ql6me:00000000000000000000000000000000:c3qnrebcud1nuibb18lmitlc442ql6me:29o8a3uevcpr15tilcemb3s438edmoog:125:dnu4lr5j8sstbj5usbld7alsnuj5nf23:139 \ No newline at end of file diff --git a/integration-tests/bats/archive-test-repo/repo_state.json b/integration-tests/bats/archive-test-repo/repo_state.json new file mode 100755 index 0000000000..32f2d6197d --- /dev/null +++ b/integration-tests/bats/archive-test-repo/repo_state.json @@ -0,0 +1,6 @@ +{ + "head": "refs/heads/main", + "remotes": {}, + "backups": {}, + "branches": {} +} \ No newline at end of file diff --git a/integration-tests/bats/archive.bats b/integration-tests/bats/archive.bats index 0e31e7b0ae..d68b8695d4 100755 --- a/integration-tests/bats/archive.bats +++ b/integration-tests/bats/archive.bats @@ -11,6 +11,12 @@ setup() { } teardown() { + if [ -n "$remotesrv_pid" ]; then + kill "$remotesrv_pid" + wait "$remotesrv_pid" || : + remotesrv_pid="" + fi + assert_feature_version teardown_common } @@ -109,19 +115,6 @@ mutations_and_gc_statement() { [ "$files" -eq "2" ] } -@test "archive: archive with remotesrv no go" { - dolt sql -q "$(mutations_and_gc_statement)" - dolt archive - - run dolt sql-server --remotesapi-port=12321 - [ "$status" -eq 1 ] - [[ "$output" =~ "archive files present" ]] || false - - run remotesrv --repo-mode - [ "$status" -eq 1 ] - [[ "$output" =~ "archive files present" ]] || false -} - @test "archive: archive --revert (fast)" { dolt sql -q "$(mutations_and_gc_statement)" dolt archive @@ -143,19 +136,136 @@ mutations_and_gc_statement() { [ "$commits" -eq "66" ] } -@test "archive: archive backup no go" { - dolt sql -q "$(mutations_and_gc_statement)" - dolt archive +@test "archive: can clone archived repository" { + mkdir -p remote/.dolt + mkdir cloned - dolt backup add bac1 file://../bac1 - run dolt backup sync bac1 + # Copy the archive test repo to remote directory + cp -R $BATS_TEST_DIRNAME/archive-test-repo/* remote/.dolt + cd remote - [ "$status" -eq 1 ] - [[ "$output" =~ "archive files present" ]] || false + port=$( definePORT ) - # currently the cli and stored procedures are different code paths. - run dolt sql -q "call dolt_backup('sync', 'bac1')" - [ "$status" -eq 1 ] - # NM4 - TODO. This message is cryptic, but plumbing the error through is awkward. - [[ "$output" =~ "Archive chunk source" ]] || false + remotesrv --http-port $port --grpc-port $port --repo-mode & + remotesrv_pid=$! + [[ "$remotesrv_pid" -gt 0 ]] || false + + cd ../cloned + run dolt clone http://localhost:$port/test-org/test-repo repo1 + [ "$status" -eq 0 ] + cd repo1 + + # Verify we can read data + run dolt sql -q 'select sum(i) from tbl;' + [[ "$status" -eq 0 ]] || false + [[ "$output" =~ "138075" ]] || false # i = 1 - 525, sum is 138075 + + kill $remotesrv_pid + wait $remotesrv_pid || : + remotesrv_pid="" + + ## The above test is the setup for the next test - so we'll stick both in here. + ## This tests cloning from a clone. Archive files are generally in oldgen, but not the case with a fresh clone. + cd ../../ + mkdir clone2 + + cd cloned/repo1 # start the server using the clone from above. + port=$( definePORT ) + remotesrv --http-port $port --grpc-port $port --repo-mode & + remotesrv_pid=$! + [[ "$remotesrv_pid" -gt 0 ]] || false + + cd ../../clone2 + run dolt clone http://localhost:$port/test-org/test-repo repo2 + [ "$status" -eq 0 ] + cd repo2 + + run dolt sql -q 'select sum(i) from tbl;' + [[ "$status" -eq 0 ]] || false + [[ "$output" =~ "138075" ]] || false # i = 1 - 525, sum is 138075 + + + ## Temporary check. We want to ensure that backup will give an error, even when + ## there are archives in newgen. + mkdir ../backup + dolt backup add bac1 file://../backup + + run dolt backup sync bac1 + [ "$status" -eq 1 ] + [[ "$output" =~ "error: archive files present" ]] || false } + +@test "archive: can clone respiratory with mixed types" { + mkdir -p remote/.dolt + mkdir cloned + + # Copy the archive test repo to remote directory + cp -R $BATS_TEST_DIRNAME/archive-test-repo/* remote/.dolt + cd remote + + # Insert data (commits automatically), but don't gc/archive yet. Want to make sure we can still clone it. + dolt sql -q "$(insert_statement)" + + port=$( definePORT ) + + remotesrv --http-port $port --grpc-port $port --repo-mode & + remotesrv_pid=$! + [[ "$remotesrv_pid" -gt 0 ]] || false + + cd ../cloned + run dolt clone http://localhost:$port/test-org/test-repo repo1 + [ "$status" -eq 0 ] + cd repo1 + + # verify new data is there. + run dolt sql -q 'select sum(i) from tbl;' + [[ "$status" -eq 0 ]] || false + + [[ "$output" =~ "151525" ]] || false # i = 1 - 550, sum is 151525 +} + +@test "archive: can fetch chunks from an archived repo" { + mkdir -p remote/.dolt + mkdir cloned + + # Copy the archive test repo to remote directory + cp -R $BATS_TEST_DIRNAME/archive-test-repo/* remote/.dolt + cd remote + + port=$( definePORT ) + + remotesrv --http-port $port --grpc-port $port --repo-mode & + remotesrv_pid=$! + [[ "$remotesrv_pid" -gt 0 ]] || false + + cd ../cloned + dolt clone http://localhost:$port/test-org/test-repo repo1 + # Fetch when there are no changes. + cd repo1 + dolt fetch + + ## update the remote repo directly. Need to run the archive command when the server is stopped. + ## This will result in archived files on the remote, which we will need to read chunks from when we fetch. + cd ../../remote + kill $remotesrv_pid + wait $remotesrv_pid || : + remotesrv_pid="" + dolt sql -q "$(mutations_and_gc_statement)" + dolt archive + + remotesrv --http-port $port --grpc-port $port --repo-mode & + remotesrv_pid=$! + [[ "$remotesrv_pid" -gt 0 ]] || false + + cd ../cloned/repo1 + dolt fetch + + run dolt status + [ "$status" -eq 0 ] + + [[ "$output" =~ "Your branch is behind 'origin/main' by 20 commits, and can be fast-forwarded" ]] || false + + # Verify the repo has integrity. + dolt fsck +} + diff --git a/proto/dolt/services/remotesapi/v1alpha1/chunkstore.proto b/proto/dolt/services/remotesapi/v1alpha1/chunkstore.proto index 519bd616b0..7042ef1096 100644 --- a/proto/dolt/services/remotesapi/v1alpha1/chunkstore.proto +++ b/proto/dolt/services/remotesapi/v1alpha1/chunkstore.proto @@ -89,6 +89,13 @@ message RangeChunk { bytes hash = 1; uint64 offset = 2; uint32 length = 3; + + // Archive zStd dictionary spans are indicated with these fields. On the consumer side, the existence + // of a non-zero dictionary length indicates that the chunk is compressed using zStd. This implies that + // the chunkSource is an archive source. If the dictionary length is zero, the chunk is compressed using snappy + // compression (classic NOMS table file or journal chunk use snappy compression). + uint64 dictionary_offset = 4; + uint32 dictionary_length = 5; } message HttpGetRange {