mirror of
https://github.com/XTXMarkets/ternfs.git
synced 2026-01-05 18:40:16 -06:00
610 lines
15 KiB
C++
610 lines
15 KiB
C++
#pragma once
|
|
|
|
#include <cstdint>
|
|
#include <ostream>
|
|
#include <variant>
|
|
|
|
#include "Assert.hpp"
|
|
#include "Common.hpp"
|
|
#include "Bincode.hpp"
|
|
#include "Time.hpp"
|
|
|
|
enum class InodeType : uint8_t {
|
|
RESERVED = 0,
|
|
DIRECTORY = 1,
|
|
FILE = 2,
|
|
SYMLINK = 3,
|
|
};
|
|
|
|
struct ShardId {
|
|
uint8_t u8;
|
|
|
|
constexpr ShardId(): u8(0) {}
|
|
|
|
constexpr ShardId(uint8_t id): u8(id) {}
|
|
|
|
bool operator==(ShardId rhs) const {
|
|
return u8 == rhs.u8;
|
|
}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint8_t>(u8);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u8 = buf.unpackScalar<uint8_t>();
|
|
}
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& out, ShardId shard);
|
|
|
|
struct ReplicaId {
|
|
uint8_t u8;
|
|
|
|
constexpr ReplicaId(): u8(0) {}
|
|
|
|
constexpr ReplicaId(uint8_t id): u8(id) {
|
|
ALWAYS_ASSERT(valid());
|
|
}
|
|
|
|
bool operator==(ReplicaId rhs) const {
|
|
return u8 == rhs.u8;
|
|
}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint8_t>(u8);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u8 = buf.unpackScalar<uint8_t>();
|
|
}
|
|
|
|
constexpr bool valid() const {
|
|
return u8 < 5;
|
|
}
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& out, ReplicaId replica);
|
|
|
|
struct ShardReplicaId {
|
|
uint16_t u16;
|
|
|
|
constexpr ShardReplicaId() : u16(0) {}
|
|
|
|
constexpr ShardReplicaId(ShardId shid, ReplicaId rid): u16(((uint16_t)rid.u8 << 8) | shid.u8) {}
|
|
|
|
bool operator==(ShardReplicaId rhs) const {
|
|
return u16 == rhs.u16;
|
|
}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint16_t>(u16);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u16 = buf.unpackScalar<uint16_t>();
|
|
if (unlikely(!valid())) {
|
|
throw BINCODE_EXCEPTION("bad ShardReplicaId %s", u16);
|
|
}
|
|
}
|
|
|
|
constexpr ShardId shardId() const {
|
|
return ShardId(u16 & 0xFF);
|
|
}
|
|
|
|
constexpr ReplicaId replicaId() const {
|
|
return ReplicaId(u16 >> 8);
|
|
}
|
|
|
|
constexpr bool valid() const {
|
|
return replicaId().valid();
|
|
}
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& out, ShardReplicaId shrid);
|
|
|
|
// 63-bit:
|
|
// TTIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIISSSSSSSS
|
|
struct InodeId {
|
|
uint64_t u64;
|
|
|
|
constexpr InodeId(): u64(0) {}
|
|
|
|
static constexpr InodeId FromU64(uint64_t data) {
|
|
InodeId x;
|
|
x.u64 = data;
|
|
ALWAYS_ASSERT(x.valid());
|
|
return x;
|
|
}
|
|
|
|
static constexpr InodeId FromU64Unchecked(uint64_t data) {
|
|
InodeId x;
|
|
x.u64 = data;
|
|
return x;
|
|
}
|
|
|
|
constexpr InodeId(InodeType type, ShardId shard, uint64_t id): u64(((uint64_t)type << 61) | (id << 8) | (uint64_t)shard.u8) {
|
|
ALWAYS_ASSERT(valid());
|
|
}
|
|
|
|
constexpr bool valid() const {
|
|
return null() || ((id() < (1ull<<53)-1) && (type() != InodeType::RESERVED));
|
|
}
|
|
|
|
constexpr InodeType type() const {
|
|
return (InodeType)((u64 >> 61) & 0x03);
|
|
}
|
|
|
|
constexpr uint64_t id() const {
|
|
return (u64 & ~(7ull << 61)) >> 8;
|
|
}
|
|
|
|
constexpr ShardId shard() const {
|
|
return ShardId(u64 & 0xFF);
|
|
}
|
|
|
|
constexpr bool null() const {
|
|
return u64 == 0;
|
|
}
|
|
|
|
bool operator==(InodeId rhs) const {
|
|
return u64 == rhs.u64;
|
|
}
|
|
|
|
bool operator!=(InodeId rhs) const {
|
|
return u64 != rhs.u64;
|
|
}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint64_t>(u64);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u64 = buf.unpackScalar<uint64_t>();
|
|
if (unlikely(!valid())) {
|
|
throw BINCODE_EXCEPTION("bad InodeId %s", u64);
|
|
}
|
|
}
|
|
|
|
size_t packedSize() const {
|
|
return sizeof(*this);
|
|
}
|
|
};
|
|
|
|
template<>
|
|
struct std::hash<InodeId> {
|
|
std::size_t operator()(InodeId id) const noexcept {
|
|
return std::hash<uint64_t>{}(id.u64);
|
|
}
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& out, InodeId id);
|
|
|
|
constexpr InodeId NULL_INODE_ID = InodeId::FromU64(0);
|
|
constexpr InodeId ROOT_DIR_INODE_ID(InodeType::DIRECTORY, ShardId(0), 0);
|
|
|
|
// 63-bit:
|
|
// OTTIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIISSSSSSSS
|
|
struct InodeIdExtra {
|
|
uint64_t u64;
|
|
|
|
InodeIdExtra(): u64(0) {}
|
|
|
|
static InodeIdExtra FromU64(uint64_t data) {
|
|
InodeIdExtra x;
|
|
x.u64 = data;
|
|
ALWAYS_ASSERT(x.id().valid());
|
|
return x;
|
|
}
|
|
|
|
InodeIdExtra(InodeId id, bool extra) {
|
|
u64 = id.u64 | (extra ? (1ull<<63) : 0);
|
|
}
|
|
|
|
InodeId id() const {
|
|
return InodeId::FromU64(u64 & ~(1ull<<63));
|
|
}
|
|
|
|
bool extra() const {
|
|
return u64 >> 63;
|
|
}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint64_t>(u64);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u64 = buf.unpackScalar<uint64_t>();
|
|
if (unlikely(!id().valid())) {
|
|
throw BINCODE_EXCEPTION("bad OwnedInodeId %s", u64);
|
|
}
|
|
}
|
|
|
|
bool operator==(InodeIdExtra rhs) const {
|
|
return u64 == rhs.u64;
|
|
}
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& out, InodeIdExtra id);
|
|
|
|
struct Parity {
|
|
uint8_t u8;
|
|
|
|
constexpr Parity(): u8(0) {}
|
|
|
|
constexpr Parity(uint8_t data): u8(data) {}
|
|
|
|
constexpr Parity(uint8_t dataBlocks, uint8_t parityBlocks): u8(dataBlocks | (parityBlocks << 4)) {
|
|
ALWAYS_ASSERT(dataBlocks != 0 && dataBlocks < 16);
|
|
ALWAYS_ASSERT(parityBlocks < 16);
|
|
}
|
|
|
|
constexpr uint8_t dataBlocks() const {
|
|
return u8 & 0x0F;
|
|
}
|
|
|
|
constexpr uint8_t parityBlocks() const {
|
|
return u8 >> 4;
|
|
}
|
|
|
|
constexpr uint8_t blocks() const {
|
|
return dataBlocks()+parityBlocks();
|
|
}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint8_t>(u8);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u8 = buf.unpackScalar<uint8_t>();
|
|
}
|
|
|
|
bool operator==(Parity rhs) const {
|
|
return u8 == rhs.u8;
|
|
}
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& out, Parity parity);
|
|
|
|
constexpr Parity NO_PARITY(0);
|
|
|
|
constexpr uint8_t EMPTY_STORAGE = 0;
|
|
constexpr uint8_t INLINE_STORAGE = 1;
|
|
|
|
uint8_t storageClassByName(const char* name);
|
|
|
|
struct Crc {
|
|
uint32_t u32;
|
|
|
|
Crc() : u32(0) {}
|
|
Crc(uint32_t x) : u32(x) {}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar(u32);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u32 = buf.unpackScalar<uint32_t>();
|
|
}
|
|
|
|
size_t packedSize() const {
|
|
return sizeof(u32);
|
|
}
|
|
|
|
bool operator==(Crc other) const {
|
|
return u32 == other.u32;
|
|
}
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& out, Crc crc);
|
|
|
|
struct BlockServiceId {
|
|
uint64_t u64;
|
|
|
|
BlockServiceId(): u64(0) {}
|
|
BlockServiceId(uint64_t x): u64(x) {}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar(u64);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u64 = buf.unpackScalar<uint64_t>();
|
|
}
|
|
|
|
size_t packedSize() const {
|
|
return sizeof(u64);
|
|
}
|
|
|
|
bool operator==(BlockServiceId other) const {
|
|
return u64 == other.u64;
|
|
}
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& out, BlockServiceId crc);
|
|
|
|
// we reserve 3 bits so that we can fit ReplicaId in LeaderToken
|
|
struct LogIdx {
|
|
uint64_t u64;
|
|
|
|
|
|
constexpr LogIdx(): u64(0) {}
|
|
|
|
constexpr LogIdx(uint64_t idx): u64(idx) {
|
|
ALWAYS_ASSERT(valid());
|
|
}
|
|
|
|
LogIdx operator+(uint64_t offset) const {
|
|
return u64 + offset;
|
|
}
|
|
|
|
LogIdx& operator++() {
|
|
++u64;
|
|
return *this;
|
|
}
|
|
|
|
bool operator==(LogIdx rhs) const {
|
|
return u64 == rhs.u64;
|
|
}
|
|
|
|
bool operator<(LogIdx other) const {
|
|
return u64 < other.u64;
|
|
}
|
|
|
|
bool operator<=(LogIdx other) const {
|
|
return u64 <= other.u64;
|
|
}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint64_t>(u64);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u64 = buf.unpackScalar<uint64_t>();
|
|
}
|
|
|
|
constexpr bool valid() const {
|
|
return u64 < 0x2000000000000000ull;
|
|
}
|
|
};
|
|
|
|
constexpr LogIdx MAX_LOG_IDX = LogIdx(0xffffffffffffffffull >> 3);
|
|
|
|
std::ostream& operator<<(std::ostream& out, LogIdx idx);
|
|
|
|
struct LeaderToken {
|
|
uint64_t u64;
|
|
|
|
|
|
constexpr LeaderToken(): u64(0) {}
|
|
|
|
constexpr LeaderToken(ReplicaId replicaId, LogIdx idx): u64(idx.u64 << 3 | replicaId.u8) {
|
|
ALWAYS_ASSERT(replicaId.valid() && idx.valid());
|
|
}
|
|
|
|
bool operator==(LeaderToken rhs) const {
|
|
return u64 == rhs.u64;
|
|
}
|
|
|
|
constexpr bool operator<(LeaderToken rhs) const {
|
|
return u64 < rhs.u64;
|
|
}
|
|
|
|
constexpr ReplicaId replica() const {
|
|
return ReplicaId(u64 & 0x7);
|
|
}
|
|
|
|
constexpr LogIdx idx() const {
|
|
return LogIdx(u64 >> 3);
|
|
}
|
|
|
|
constexpr bool valid() const {
|
|
// we don't need to check LogIdx is valid as it any value is valid
|
|
return replica().valid();
|
|
}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint64_t>(u64);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
u64 = buf.unpackScalar<uint64_t>();
|
|
}
|
|
};
|
|
|
|
std::ostream& operator<<(std::ostream& out, LeaderToken token);
|
|
|
|
#include "MsgsGen.hpp"
|
|
|
|
// We often use this as a optional<EggsError>;
|
|
constexpr EggsError NO_ERROR = (EggsError)0;
|
|
|
|
constexpr bool isPrivilegedRequestKind(ShardMessageKind kind) {
|
|
return (uint8_t)kind & 0x80;
|
|
}
|
|
|
|
// >>> format(struct.unpack('<I', b'SHA\0')[0], 'x')
|
|
// '414853'
|
|
constexpr uint32_t SHARD_REQ_PROTOCOL_VERSION = 0x414853;
|
|
|
|
// >>> format(struct.unpack('<I', b'SHA\1')[0], 'x')
|
|
// '1414853'
|
|
constexpr uint32_t SHARD_RESP_PROTOCOL_VERSION = 0x1414853;
|
|
|
|
// >>> format(struct.unpack('<I', b'SHA\2')[0], 'x')
|
|
// '2414853'
|
|
constexpr uint32_t SHARD_LOG_PROTOCOL_VERSION = 0x2414853;
|
|
|
|
// >>> format(struct.unpack('<I', b'CDC\0')[0], 'x')
|
|
// '434443'
|
|
constexpr uint32_t CDC_REQ_PROTOCOL_VERSION = 0x434443;
|
|
|
|
// >>> format(struct.unpack('<I', b'CDC\1')[0], 'x')
|
|
// '1434443'
|
|
constexpr uint32_t CDC_RESP_PROTOCOL_VERSION = 0x1434443;
|
|
|
|
// >>> format(struct.unpack('<I', b'SHU\0')[0], 'x')
|
|
// '554853'
|
|
constexpr uint32_t SHUCKLE_REQ_PROTOCOL_VERSION = 0x554853;
|
|
|
|
// >>> format(struct.unpack('<I', b'SHU\1')[0], 'x')
|
|
// '1554853'
|
|
constexpr uint32_t SHUCKLE_RESP_PROTOCOL_VERSION = 0x1554853;
|
|
|
|
// >>> format(struct.unpack('<I', b'LOG\0')[0], 'x')
|
|
// '474f4c'
|
|
constexpr uint32_t LOG_REQ_PROTOCOL_VERSION = 0x474f4c;
|
|
|
|
// >>> format(struct.unpack('<I', b'LOG\1')[0], 'x')
|
|
// '1474f4c'
|
|
constexpr uint32_t LOG_RESP_PROTOCOL_VERSION = 0x1474f4c;
|
|
|
|
// If this doesn't parse, no point in continuing attempting to parse
|
|
// the request.
|
|
struct ShardRequestHeader {
|
|
uint64_t requestId;
|
|
// This is not guaranteed to be a valid shard request kind yet.
|
|
// The caller will have to validate.
|
|
ShardMessageKind kind;
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint32_t>(SHARD_REQ_PROTOCOL_VERSION);
|
|
buf.packScalar<uint64_t>(requestId);
|
|
buf.packScalar<uint8_t>((uint8_t)kind);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
uint32_t version = buf.unpackScalar<uint32_t>();
|
|
if (version != SHARD_REQ_PROTOCOL_VERSION) {
|
|
throw BINCODE_EXCEPTION("bad shard req protocol version %s, expected %s", version, SHARD_REQ_PROTOCOL_VERSION);
|
|
}
|
|
requestId = buf.unpackScalar<uint64_t>();
|
|
kind = (ShardMessageKind)buf.unpackScalar<uint8_t>();
|
|
}
|
|
};
|
|
|
|
struct ShardResponseHeader {
|
|
uint64_t requestId;
|
|
ShardMessageKind kind;
|
|
|
|
// protocol + requestId + kind
|
|
static constexpr uint16_t STATIC_SIZE = 4 + 8 + 1;
|
|
|
|
ShardResponseHeader() = default;
|
|
ShardResponseHeader(uint64_t requestId_, ShardMessageKind kind_): requestId(requestId_), kind(kind_) {}
|
|
|
|
void pack(BincodeBuf& buf) {
|
|
buf.packScalar<uint32_t>(SHARD_RESP_PROTOCOL_VERSION);
|
|
buf.packScalar<uint64_t>(requestId);
|
|
buf.packScalar<uint8_t>((uint8_t)kind);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
uint32_t version = buf.unpackScalar<uint32_t>();
|
|
if (version != SHARD_RESP_PROTOCOL_VERSION) {
|
|
throw BINCODE_EXCEPTION("bad shard resp protocol version %s, expected %s", version, SHARD_RESP_PROTOCOL_VERSION);
|
|
}
|
|
requestId = buf.unpackScalar<uint64_t>();
|
|
kind = (ShardMessageKind)buf.unpackScalar<uint8_t>();
|
|
}
|
|
};
|
|
|
|
// If this doesn't parse, no point in continuing attempting to parse
|
|
// the request.
|
|
struct CDCRequestHeader {
|
|
uint64_t requestId;
|
|
// This is not guaranteed to be a valid shard request kind yet.
|
|
// The caller will have to validate.
|
|
CDCMessageKind kind;
|
|
|
|
CDCRequestHeader() = default;
|
|
CDCRequestHeader(uint64_t requestId_, CDCMessageKind kind_): requestId(requestId_), kind(kind_) {}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
uint32_t version = buf.unpackScalar<uint32_t>();
|
|
if (version != CDC_REQ_PROTOCOL_VERSION) {
|
|
throw BINCODE_EXCEPTION("bad shard req protocol version %s, expected %s", version, CDC_REQ_PROTOCOL_VERSION);
|
|
}
|
|
requestId = buf.unpackScalar<uint64_t>();
|
|
kind = (CDCMessageKind)buf.unpackScalar<uint8_t>();
|
|
}
|
|
};
|
|
|
|
struct CDCResponseHeader {
|
|
uint64_t requestId;
|
|
CDCMessageKind kind;
|
|
|
|
// protocol + requestId + kind
|
|
static constexpr uint16_t STATIC_SIZE = 4 + 8 + 1;
|
|
|
|
CDCResponseHeader(uint64_t requestId_, CDCMessageKind kind_): requestId(requestId_), kind(kind_) {}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint32_t>(CDC_RESP_PROTOCOL_VERSION);
|
|
buf.packScalar<uint64_t>(requestId);
|
|
buf.packScalar<uint8_t>((uint8_t)kind);
|
|
}
|
|
};
|
|
|
|
|
|
// If this doesn't parse, no point in continuing attempting to parse
|
|
// the request.
|
|
struct LogRequestHeader {
|
|
uint64_t requestId;
|
|
// This is not guaranteed to be a valid log request kind yet.
|
|
// The caller will have to validate.
|
|
LogMessageKind kind;
|
|
|
|
static constexpr uint16_t STATIC_SIZE = 4 + 8 + 1;
|
|
|
|
LogRequestHeader() = default;
|
|
LogRequestHeader(uint64_t requestId_, LogMessageKind kind_): requestId(requestId_), kind(kind_) {}
|
|
|
|
void pack(BincodeBuf& buf) const {
|
|
buf.packScalar<uint32_t>(LOG_REQ_PROTOCOL_VERSION);
|
|
buf.packScalar<uint64_t>(requestId);
|
|
buf.packScalar<uint8_t>((uint8_t)kind);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
uint32_t version = buf.unpackScalar<uint32_t>();
|
|
if (version != LOG_REQ_PROTOCOL_VERSION) {
|
|
throw BINCODE_EXCEPTION("bad log req protocol version %s, expected %s", version, LOG_REQ_PROTOCOL_VERSION);
|
|
}
|
|
requestId = buf.unpackScalar<uint64_t>();
|
|
kind = (LogMessageKind)buf.unpackScalar<uint8_t>();
|
|
}
|
|
};
|
|
|
|
struct LogResponseHeader {
|
|
uint64_t requestId;
|
|
LogMessageKind kind;
|
|
|
|
// protocol + requestId + kind
|
|
static constexpr uint16_t STATIC_SIZE = 4 + 8 + 1;
|
|
|
|
LogResponseHeader() = default;
|
|
LogResponseHeader(uint64_t requestId_, LogMessageKind kind_): requestId(requestId_), kind(kind_) {}
|
|
|
|
void pack(BincodeBuf& buf) {
|
|
buf.packScalar<uint32_t>(LOG_RESP_PROTOCOL_VERSION);
|
|
buf.packScalar<uint64_t>(requestId);
|
|
buf.packScalar<uint8_t>((uint8_t)kind);
|
|
}
|
|
|
|
void unpack(BincodeBuf& buf) {
|
|
uint32_t version = buf.unpackScalar<uint32_t>();
|
|
if (version != LOG_RESP_PROTOCOL_VERSION) {
|
|
throw BINCODE_EXCEPTION("bad log resp protocol version %s, expected %s", version, LOG_RESP_PROTOCOL_VERSION);
|
|
}
|
|
requestId = buf.unpackScalar<uint64_t>();
|
|
kind = (LogMessageKind)buf.unpackScalar<uint8_t>();
|
|
}
|
|
};
|
|
|
|
static constexpr uint8_t SNAPSHOT_POLICY_TAG = 1;
|
|
static constexpr uint8_t SPAN_POLICY_TAG = 2;
|
|
static constexpr uint8_t BLOCK_POLICY_TAG = 3;
|
|
static constexpr uint8_t STRIPE_POLICY_TAG = 4;
|
|
static std::array<uint8_t, 4> REQUIRED_DIR_INFO_TAGS = {SNAPSHOT_POLICY_TAG, SPAN_POLICY_TAG, BLOCK_POLICY_TAG, STRIPE_POLICY_TAG};
|