mirror/ternfs-XTXMarkets
1
0
Fork 0
mirror of https://github.com/XTXMarkets/ternfs.git synced 2026-01-06 11:00:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add versions to some RocksDB values

Browse Source 
Only the ones where it is needed -- in some cases we can just
modify the keys (e.g. metadata stuff).

Also, come up with a sort of horrifying but more robust way
to specify the RocksDB values with the C preprocessor.
This commit is contained in:
Francesco Mazzoli
2023-05-22 08:03:01 +00:00
parent b292333baf
commit 1eab8ee6cf
5 changed files with 293 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
cpp/cdc/CDCDBData.hpp
View File

@@ -34,19 +34,12 @@ inline rocksdb::Slice cdcMetadataKey(const CDCMetadataKey* k) {
}
struct MakeDirectoryState {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(InodeId) + // dir id we've generated
sizeof(EggsTime) + // creation time for the edge
sizeof(EggsError); // exit error if we're rolling back
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
LE_VAL(InodeId, dirId, setDirId, 0)
LE_VAL(EggsTime, creationTime, setCreationTime, 8)
LE_VAL(EggsError, exitError, setExitError, 16)
FIELDS(
LE, InodeId, dirId, setDirId,
LE, EggsTime, creationTime, setCreationTime,
LE, EggsError, exitError, setExitError, // error if we're rolling back
END_STATIC
)
void start() {
setDirId(NULL_INODE_ID);
@@ -56,17 +49,11 @@ struct MakeDirectoryState {
};
struct RenameFileState {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(EggsTime) + // the time at which we created the current edge
sizeof(EggsError); // exit error if we're rolling back
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
LE_VAL(EggsTime, newCreationTime, setNewCreationTime, 0)
LE_VAL(EggsError, exitError, setExitError, 8)
FIELDS(
LE, EggsTime, newCreationTime, setNewCreationTime,
LE, EggsError, exitError, setExitError,
END_STATIC
)
void start() {
setExitError(NO_ERROR);
@@ -74,17 +61,11 @@ struct RenameFileState {
};
struct SoftUnlinkDirectoryState {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(InodeId) + // what we're currently stat'ing
sizeof(EggsError); // exit error if we're rolling back
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
LE_VAL(InodeId, statDirId, setStatDirId, 0)
LE_VAL(EggsError, exitError, setExitError, 8)
FIELDS(
LE, InodeId, statDirId, setStatDirId,
LE, EggsError, exitError, setExitError,
END_STATIC
)
void start() {
setExitError(NO_ERROR);
@@ -92,17 +73,11 @@ struct SoftUnlinkDirectoryState {
};
struct RenameDirectoryState {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(EggsTime) + // time at which we created the old edge
sizeof(EggsError); // exit error if we're rolling back
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
LE_VAL(EggsTime, newCreationTime, setNewCreationTime, 0)
LE_VAL(EggsError, exitError, setExitError, 8)
FIELDS(
LE, EggsTime, newCreationTime, setNewCreationTime,
LE, EggsError, exitError, setExitError,
END_STATIC
)
void start() {
setNewCreationTime({});
@@ -111,18 +86,12 @@ struct RenameDirectoryState {
};
struct HardUnlinkDirectoryState {
char* _data;
static constexpr size_t MAX_SIZE = 0;
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
FIELDS(END_STATIC)
void start() {}
};
struct CrossShardHardUnlinkFileState {
char* _data;
static constexpr size_t MAX_SIZE = 0;
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
FIELDS(END_STATIC)
void start() {}
};
@@ -136,18 +105,16 @@ constexpr size_t maxMaxSize() {
}
struct TxnState {
char* _data;
static constexpr size_t MIN_SIZE =
sizeof(CDCMessageKind) + // request type
sizeof(uint8_t); // step
FIELDS(
LE, CDCMessageKind, reqKind, setReqKind,
LE, uint8_t, step, setStep,
EMIT_OFFSET, MIN_SIZE,
END
)
static constexpr size_t MAX_SIZE =
MIN_SIZE +
maxMaxSize<MakeDirectoryState, RenameFileState, SoftUnlinkDirectoryState, RenameDirectoryState, HardUnlinkDirectoryState, CrossShardHardUnlinkFileState>();
U8_VAL(CDCMessageKind, reqKind, setReqKind, 0);
U8_VAL(uint8_t, step, setStep, 1);
size_t size() const {
size_t sz = MIN_SIZE;
switch (reqKind()) {

200
cpp/core/RocksDBUtils.hpp
View File

@@ -45,12 +45,6 @@ private:
std::string _msg;
};
// We use byteswap to go from LE to BE, and vice-versa.
static_assert(std::endian::native == std::endian::little);
inline uint64_t byteswapU64(uint64_t x) {
return __builtin_bswap64(x);
}
template<typename T>
struct StaticValue {
private:
@@ -95,6 +89,8 @@ public:
return ExternalValue((char*)slice.data(), slice.size());
}
// TODO make this have some dangerous name, since it's easy to make mistakes
// when manipulating external values
T operator()() {
return _val;
}
@@ -138,19 +134,10 @@ public:
}
};
#define LE_VAL(type, name, setName, offset) \
static_assert(sizeof(type) > 1); \
type name() const { \
type x; \
memcpy(&x, _data+offset, sizeof(x)); \
return x; \
} \
void setName(type x) { \
memcpy(_data+offset, &x, sizeof(x)); \
}
// We use byteswap to go from LE to BE, and vice-versa.
static_assert(std::endian::native == std::endian::little);
#define U8_VAL(type, name, setName, offset) \
static_assert(sizeof(type) == sizeof(uint8_t)); \
#define LE_VAL(type, name, setName, offset) \
type name() const { \
type x; \
memcpy(&x, _data+offset, sizeof(x)); \
@@ -169,42 +156,137 @@ public:
memcpy(_data+offset+1, bytes.data(), bytes.size()); \
}
// Actually needed to avoid confusing cpp with commas 🤪
template<size_t sz>
using FBytesArr = std::array<uint8_t, sz>;
#define FBYTES_VAL(sz, getName, setName, offset) \
std::array<uint8_t, sz> getName() const { \
std::array<uint8_t, sz> bs; \
FBytesArr<sz> getName() const { \
FBytesArr<sz> bs; \
memcpy(bs.data(), _data+offset, sz); \
return bs; \
} \
void setName(const std::array<uint8_t, sz>& bs) { \
void setName(const FBytesArr<sz>& bs) { \
memcpy(_data+offset, bs.data(), sz); \
}
#define BE64_VAL(type, name, setName, offset) \
static_assert(sizeof(type) == sizeof(uint64_t)); \
template<size_t sz>
struct FieldToBE;
template<>
struct FieldToBE<8> {
using Scalar = uint64_t;
static inline Scalar bswap(Scalar x) {
return __builtin_bswap64(x);
}
};
template<>
struct FieldToBE<4> {
using Scalar = uint32_t;
static inline Scalar bswap(Scalar x) {
return __builtin_bswap32(x);
}
};
template<>
struct FieldToBE<2> {
using Scalar = uint16_t;
static inline Scalar bswap(Scalar x) {
return __builtin_bswap16(x);
}
};
template<>
struct FieldToBE<1> {
using Scalar = uint8_t;
static inline Scalar bswap(Scalar x) { return x; }
};
#define BE_VAL(type, name, setName, offset) \
type name() const { \
uint64_t x; \
FieldToBE<sizeof(type)>::Scalar x; \
memcpy(&x, _data+offset, sizeof(x)); \
x = byteswapU64(x); /* BE -> LE */ \
x = FieldToBE<sizeof(type)>::bswap(x); /* BE -> LE */ \
type v; \
memcpy(&v, (char*)&x, sizeof(uint64_t)); \
memcpy(&v, (char*)&x, sizeof(x)); \
return v; \
} \
void setName(type v) { \
uint64_t x; \
memcpy(&x, (char*)&v, sizeof(uint64_t)); \
x = byteswapU64(x); /* LE -> BE */ \
FieldToBE<sizeof(type)>::Scalar x; \
memcpy(&x, (char*)&v, sizeof(type)); \
x = FieldToBE<sizeof(type)>::bswap(x); /* LE -> BE */ \
memcpy(_data+offset, &x, sizeof(x)); \
}
// See <https://www.scs.stanford.edu/~dm/blog/va-opt.html>
// for the reasoning behind this horror.
#define PARENS ()
#define COMMA ,
#define EXPAND(arg) EXPAND1(EXPAND1(EXPAND1(EXPAND1(arg))))
#define EXPAND1(arg) EXPAND2(EXPAND2(EXPAND2(EXPAND2(arg))))
#define EXPAND2(arg) EXPAND3(EXPAND3(EXPAND3(EXPAND3(arg))))
#define EXPAND3(arg) EXPAND4(EXPAND4(EXPAND4(EXPAND4(arg))))
#define EXPAND4(arg) arg
#define FIELDS_HELPER(offset, which, ...) \
which##_FIELDS_AGAIN PARENS (offset __VA_OPT__(,) __VA_ARGS__)
#define FIELDS_AGAIN() FIELDS_HELPER
#define FIELDS(...) \
char* _data; \
EXPAND(FIELDS_HELPER(0, __VA_ARGS__))
#define END_FIELDS(offset)
#define END_FIELDS_AGAIN() END_FIELDS
#define LE_FIELDS(offset, type, name, setName, ...) \
LE_VAL(type, name, setName, offset) \
FIELDS_AGAIN PARENS ((offset)+sizeof(type), __VA_ARGS__)
#define LE_FIELDS_AGAIN() LE_FIELDS
#define BYTES_FIELDS(offset, name, setName, ...) \
BYTES_VAL(name, setName, offset) \
FIELDS_AGAIN PARENS ((offset)+1+name().size(), __VA_ARGS__)
#define BYTES_FIELDS_AGAIN() BYTES_FIELDS
#define FBYTES_FIELDS(offset, sz, name, setName, ...) \
FBYTES_VAL(sz, name, setName, offset) \
FIELDS_AGAIN PARENS ((offset)+sz, __VA_ARGS__)
#define FBYTES_FIELDS_AGAIN() FBYTES_FIELDS
#define BE_FIELDS(offset, type, name, setName, ...) \
BE_VAL(type, name, setName, offset) \
FIELDS_AGAIN PARENS ((offset)+sizeof(type), __VA_ARGS__)
#define BE_FIELDS_AGAIN() BE_FIELDS
#define EMIT_OFFSET_FIELDS(offset, name, ...) \
static constexpr size_t name = offset; \
FIELDS_AGAIN PARENS ((offset), __VA_ARGS__)
#define EMIT_OFFSET_FIELDS_AGAIN() EMIT_OFFSET_FIELDS
#define EMIT_SIZE_FIELDS(offset, name, ...) \
inline size_t name() const { \
return (offset); \
}
#define EMIT_SIZE_FIELDS_AGAIN() EMIT_SIZE_FIELDS
// Useful for simple datatypes with a fixed, static size.
#define END_STATIC_FIELDS(offset) \
static constexpr size_t MAX_SIZE = offset; \
size_t size() const { return MAX_SIZE; } \
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
#define END_STATIC_FIELDS_AGAIN() END_STATIC_FIELDS
// When we need a simple u64 value (e.g. log index)
struct U64Value {
char* _data;
static constexpr size_t MAX_SIZE = sizeof(uint64_t);
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
LE_VAL(uint64_t, u64, setU64, 0)
FIELDS(
LE, uint64_t, u64, setU64,
END_STATIC
)
static StaticValue<U64Value> Static(uint64_t x) {
auto v = StaticValue<U64Value>();
@@ -214,13 +296,10 @@ struct U64Value {
};
struct I64Value {
char* _data;
static constexpr size_t MAX_SIZE = sizeof(int64_t);
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
LE_VAL(int64_t, i64, setI64, 0)
FIELDS(
LE, int64_t, i64, setI64,
END_STATIC,
)
static StaticValue<I64Value> Static(int64_t x) {
auto v = StaticValue<I64Value>();
@@ -231,13 +310,10 @@ struct I64Value {
// When we need a simple u64 key (e.g. log index)
struct U64Key {
char* _data;
static constexpr size_t MAX_SIZE = sizeof(uint64_t);
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
BE64_VAL(uint64_t, u64, setU64, 0)
FIELDS(
BE, uint64_t, u64, setU64,
END_STATIC
)
static StaticValue<U64Key> Static(uint64_t x) {
auto v = StaticValue<U64Key>();
@@ -250,13 +326,10 @@ struct U64Key {
// (and therefore BE), but it does make it a bit nicer to be able to traverse
// them like that.
struct InodeIdKey {
char* _data;
static constexpr size_t MAX_SIZE = sizeof(InodeId);
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
BE64_VAL(InodeId, id, setId, 0)
FIELDS(
BE, InodeId, id, setId,
END_STATIC
)
static StaticValue<InodeIdKey> Static(InodeId id) {
auto x = StaticValue<InodeIdKey>();
@@ -283,13 +356,10 @@ void bincodeFromRocksValue(const rocksdb::Slice& value, A& v) {
}
struct InodeIdValue {
char* _data;
static constexpr size_t MAX_SIZE = sizeof(InodeId);
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
LE_VAL(InodeId, id, setId, 0)
FIELDS(
LE, InodeId, id, setId,
END_STATIC
)
static StaticValue<InodeIdValue> Static(InodeId id) {
auto x = StaticValue<InodeIdValue>();
@@ -298,4 +368,4 @@ struct InodeIdValue {
}
};
std::shared_ptr<rocksdb::MergeOperator> CreateInt64AddOperator();
std::shared_ptr<rocksdb::MergeOperator> CreateInt64AddOperator();

20
cpp/shard/ShardDB.cpp
View File

@@ -351,6 +351,7 @@ struct ShardDBImpl {
LOG_INFO(_env, "creating root directory, since it does not exist");
DirectoryInfo info = defaultDirectoryInfo();
OwnedValue<DirectoryBody> dirBody(info);
dirBody().setVersion(0);
dirBody().setOwnerId(NULL_INODE_ID);
dirBody().setMtime({});
dirBody().setHashMode(HashMode::XXH3_63);
@@ -1643,11 +1644,12 @@ struct ShardDBImpl {
// write to rocks
StaticValue<TransientFileBody> transientFile;
transientFile().setVersion(0);
transientFile().setFileSize(0);
transientFile().setMtime(time);
transientFile().setDeadline(entry.deadlineTime);
transientFile().setLastSpanState(SpanState::CLEAN);
transientFile().setNote(entry.note.ref());
transientFile().setNoteDangerous(entry.note.ref());
auto k = InodeIdKey::Static(id);
ROCKS_DB_CHECKED(batch.Put(_transientCf, k.toSlice(), transientFile.toSlice()));
@@ -1704,6 +1706,7 @@ struct ShardDBImpl {
auto fileKey = InodeIdKey::Static(entry.fileId);
ROCKS_DB_CHECKED(batch.Delete(_transientCf, fileKey.toSlice()));
StaticValue<FileBody> file;
file().setVersion(0);
file().setMtime(time);
file().setFileSize(transientFile().fileSize());
ROCKS_DB_CHECKED(batch.Put(_filesCf, fileKey.toSlice(), file.toSlice()));
@@ -1839,6 +1842,7 @@ struct ShardDBImpl {
k().setName(name.ref());
k().setCreationTime(existingEdge().creationTime());
StaticValue<SnapshotEdgeBody> v;
v().setVersion(0);
// this was current, so it's now owned.
v().setTargetIdWithOwned(InodeIdExtra(existingEdge().targetIdWithLocked().id(), true));
ROCKS_DB_CHECKED(batch.Put(_edgesCf, k.toSlice(), v.toSlice()));
@@ -1848,6 +1852,7 @@ struct ShardDBImpl {
// OK, we're now ready to insert the current edge
StaticValue<CurrentEdgeBody> edgeBody;
edgeBody().setVersion(0);
edgeBody().setTargetIdWithLocked(InodeIdExtra(targetId, locked));
edgeBody().setCreationTime(creationTime);
ROCKS_DB_CHECKED(batch.Put(_edgesCf, edgeKey.toSlice(), edgeBody.toSlice()));
@@ -1922,6 +1927,7 @@ struct ShardDBImpl {
k().setName(name.ref());
k().setCreationTime(edgeBody().creationTime());
StaticValue<SnapshotEdgeBody> v;
v().setVersion(0);
v().setTargetIdWithOwned(InodeIdExtra(targetId, owned));
ROCKS_DB_CHECKED(batch.Put(_edgesCf, k.toSlice(), v.toSlice()));
k().setCreationTime(time);
@@ -1961,6 +1967,7 @@ struct ShardDBImpl {
{
auto dirKey = InodeIdKey::Static(entry.id);
OwnedValue<DirectoryBody> dir(entry.info);
dir().setVersion(0);
dir().setOwnerId(entry.ownerId);
dir().setMtime(time);
dir().setHashMode(HashMode::XXH3_63);
@@ -2025,6 +2032,7 @@ struct ShardDBImpl {
snapshotKey().setName(entry.name.ref());
snapshotKey().setCreationTime(edge().creationTime());
StaticValue<SnapshotEdgeBody> snapshotBody;
snapshotBody().setVersion(0);
snapshotBody().setTargetIdWithOwned(InodeIdExtra(entry.targetId, false));
ROCKS_DB_CHECKED(batch.Put(_edgesCf, snapshotKey.toSlice(), snapshotBody.toSlice()));
snapshotKey().setCreationTime(time);
@@ -2111,6 +2119,7 @@ struct ShardDBImpl {
// we need to create a new DirectoryBody rather than modify the old one, the info might have changed size
{
OwnedValue<DirectoryBody> newDir(entry.info);
newDir().setVersion(0);
newDir().setOwnerId(NULL_INODE_ID);
newDir().setMtime(time);
newDir().setHashMode(dir().hashMode());
@@ -2266,6 +2275,7 @@ struct ShardDBImpl {
}
OwnedValue<DirectoryBody> newDir(entry.info);
newDir().setVersion(0);
newDir().setOwnerId(dir().ownerId());
newDir().setMtime(dir().mtime());
newDir().setHashMode(dir().hashMode());
@@ -2374,11 +2384,12 @@ struct ShardDBImpl {
auto k = InodeIdKey::Static(entry.targetId);
ROCKS_DB_CHECKED(batch.Delete(_filesCf, k.toSlice()));
StaticValue<TransientFileBody> v;
v().setVersion(0);
v().setFileSize(file().fileSize());
v().setMtime(time);
v().setDeadline(0);
v().setLastSpanState(SpanState::CLEAN);
v().setNote(entry.name.ref());
v().setNoteDangerous(entry.name.ref());
ROCKS_DB_CHECKED(batch.Put(_transientCf, k.toSlice(), v.toSlice()));
}
@@ -2616,6 +2627,7 @@ struct ShardDBImpl {
// Now manufacture and add the span
OwnedValue<SpanBody> spanBody(entry.body.ref());
{
spanBody().setVersion(0);
spanBody().setSpanSize(entry.size);
spanBody().setCrc(entry.crc.u32);
spanBody().setStorageClass(entry.storageClass);
@@ -2695,6 +2707,7 @@ struct ShardDBImpl {
// in the block service -> files index.
OwnedValue<SpanBody> spanBody(entry.storageClass, entry.parity, entry.stripes);
{
spanBody().setVersion(0);
spanBody().setSpanSize(entry.size);
spanBody().setCrc(entry.crc.u32);
spanBody().setStorageClass(entry.storageClass);
@@ -2878,11 +2891,12 @@ struct ShardDBImpl {
// make a transient one
StaticValue<TransientFileBody> transientFile;
transientFile().setVersion(0);
transientFile().setFileSize(file().fileSize());
transientFile().setMtime(time);
transientFile().setDeadline(0);
transientFile().setLastSpanState(SpanState::CLEAN);
transientFile().setNote(entry.note.ref());
transientFile().setNoteDangerous(entry.note.ref());
ROCKS_DB_CHECKED(batch.Put(_transientCf, k.toSlice(), transientFile.toSlice()));
return NO_ERROR;

269
cpp/shard/ShardDBData.hpp
View File

@@ -34,61 +34,41 @@ inline rocksdb::Slice shardMetadataKey(const ShardMetadataKey* k) {
}
struct ShardInfoBody {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(ShardId) + // shardId
sizeof(std::array<uint8_t, 16>); // secretKey
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
U8_VAL (ShardId, shardId, setShardId, 0)
FBYTES_VAL(16, secretKey, setSecretKey, 1)
FIELDS(
LE, ShardId, shardId, setShardId,
FBYTES, 16, secretKey, setSecretKey,
END_STATIC
)
};
struct BlockServiceKey {
char* _data;
static constexpr size_t MAX_SIZE = sizeof(ShardMetadataKey) + sizeof(uint64_t);
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
U8_VAL(ShardMetadataKey, key, setKey, 0) // always BLOCK_SERVICE_KEY
BE64_VAL(uint64_t, blockServiceId, setBlockServiceId, 1)
FIELDS(
BE, ShardMetadataKey, key, setKey, // always BLOCK_SERVICE_KEY
BE, uint64_t, blockServiceId, setBlockServiceId,
END_STATIC
)
};
struct BlockServiceBody {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(uint64_t) + // id
sizeof(char[4]) + // ip1
sizeof(uint16_t) + // port1
sizeof(char[4]) + // ip2
sizeof(uint16_t) + // port2
sizeof(uint8_t) + // storage class
sizeof(char[16]) + // failure domain
sizeof(char[16]); // secret key
size_t size() const { return MAX_SIZE; }
void checkSize(size_t size) { ALWAYS_ASSERT(size == MAX_SIZE); }
LE_VAL(uint64_t, id, setId, 0)
FBYTES_VAL(4, ip1, setIp1, 8)
LE_VAL(uint16_t, port1, setPort1, 12)
FBYTES_VAL(4, ip2, setIp2, 14)
LE_VAL(uint16_t, port2, setPort2, 18)
U8_VAL(uint8_t, storageClass, setStorageClass, 20)
FBYTES_VAL(16, failureDomain, setFailureDomain, 21)
FBYTES_VAL(16, secretKey, setSecretKey, 37)
FIELDS(
LE, uint64_t, id, setId,
FBYTES, 4, ip1, setIp1,
LE, uint16_t, port1, setPort1,
FBYTES, 4, ip2, setIp2,
LE, uint16_t, port2, setPort2,
LE, uint8_t, storageClass, setStorageClass,
FBYTES, 16, failureDomain, setFailureDomain,
FBYTES, 16, secretKey, setSecretKey,
END_STATIC
)
};
struct CurrentBlockServicesBody {
char* _data;
static constexpr size_t MIN_SIZE =
sizeof(uint8_t); // number of current block services
FIELDS(
LE, uint8_t, length, setLength,
EMIT_OFFSET, MIN_SIZE,
END
)
void checkSize(size_t sz) {
ALWAYS_ASSERT(sz >= MIN_SIZE, "sz < MIN_SIZE (%s < %s)", sz, MIN_SIZE);
@@ -108,8 +88,6 @@ struct CurrentBlockServicesBody {
return MIN_SIZE + length()*sizeof(uint64_t);
}
U8_VAL(uint8_t, length, setLength, 0)
uint64_t at(uint64_t ix) const {
ALWAYS_ASSERT(ix < length());
uint64_t v;
@@ -132,85 +110,67 @@ enum class SpanState : uint8_t {
std::ostream& operator<<(std::ostream& out, SpanState state);
struct TransientFileBody {
char* _data;
FIELDS(
LE, uint8_t, version, setVersion,
LE, uint64_t, fileSize, setFileSize,
LE, EggsTime, mtime, setMtime,
LE, EggsTime, deadline, setDeadline,
LE, SpanState, lastSpanState, setLastSpanState,
EMIT_OFFSET, STATIC_SIZE,
BYTES, note, setNoteDangerous, // dangerous because we might not have enough space
EMIT_SIZE, size,
END
)
static constexpr size_t MIN_SIZE =
sizeof(uint64_t) + // size
sizeof(EggsTime) + // mtime
sizeof(EggsTime) + // deadline
sizeof(SpanState) + // lastSpanState
STATIC_SIZE +
sizeof(uint8_t); // noteLength
static constexpr size_t MAX_SIZE = MIN_SIZE + 255; // max note size
size_t size() const {
return MIN_SIZE + note().size();
}
void checkSize(size_t sz) {
ALWAYS_ASSERT(sz >= MIN_SIZE);
ALWAYS_ASSERT(sz == size());
}
LE_VAL(uint64_t, fileSize, setFileSize, 0)
LE_VAL(EggsTime, mtime, setMtime, 8)
LE_VAL(EggsTime, deadline, setDeadline, 16)
U8_VAL(SpanState, lastSpanState, setLastSpanState, 24)
BYTES_VAL(note, setNote, 25)
};
struct FileBody {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(uint64_t) + // size
sizeof(EggsTime); // mtime
size_t size() const { return MAX_SIZE; }
void checkSize(size_t sz) { ALWAYS_ASSERT(sz == MAX_SIZE); }
LE_VAL(uint64_t, fileSize, setFileSize, 0)
LE_VAL(EggsTime, mtime, setMtime, 8)
FIELDS(
LE, uint8_t, version, setVersion,
LE, uint64_t, fileSize, setFileSize,
LE, EggsTime, mtime, setMtime,
END_STATIC
)
};
struct SpanKey {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(InodeId) + // id
sizeof(uint64_t); // offset
size_t size() const { return MAX_SIZE; }
void checkSize(size_t sz) { ALWAYS_ASSERT(sz == MAX_SIZE); }
BE64_VAL(InodeId, fileId, setFileId, 0)
BE64_VAL(uint64_t, offset, setOffset, 8)
FIELDS(
BE, InodeId, fileId, setFileId,
BE, uint64_t, offset, setOffset,
END_STATIC
)
};
struct BlockBody {
char* _data;
static constexpr size_t SIZE =
sizeof(uint64_t) +
sizeof(uint64_t) +
sizeof(uint32_t);
LE_VAL(BlockServiceId, blockService, setBlockService, 0);
LE_VAL(uint64_t, blockId, setBlockId, 8);
LE_VAL(uint32_t, crc, setCrc, 16);
FIELDS(
LE, BlockServiceId, blockService, setBlockService,
LE, uint64_t, blockId, setBlockId,
LE, uint32_t, crc, setCrc,
END_STATIC
)
constexpr static size_t SIZE = MAX_SIZE;
};
struct SpanBlocksBody {
char* _data;
static constexpr size_t MIN_SIZE =
sizeof(uint8_t) + // parity
sizeof(uint8_t) + // stripes
sizeof(uint32_t); // cellSize
// after this:
// * []BlockBody blocks
// * []u32 stripesCrc
U8_VAL(Parity, parity, setParity, 0)
U8_VAL(uint8_t, stripes, setStripes, 1)
LE_VAL(uint32_t, cellSize, setCellSize, 2)
FIELDS(
LE, Parity, parity, setParity,
LE, uint8_t, stripes, setStripes,
LE, uint32_t, cellSize, setCellSize,
EMIT_OFFSET, MIN_SIZE,
END
)
// after this:
// * []BlockBody blocks
// * []u32 stripesCrc
SpanBlocksBody(char* data) : _data(data) {}
@@ -258,19 +218,17 @@ struct SpanBlocksBody {
};
struct SpanBody {
char* _data;
static constexpr size_t MIN_SIZE =
sizeof(uint32_t) + // size
sizeof(uint32_t) + // crc
sizeof(uint8_t); // storageClass
// after this:
// * Inline body for inline spans (bytes)
// * Blocks for normal spans
LE_VAL(uint32_t, spanSize, setSpanSize, 0)
LE_VAL(uint32_t, crc, setCrc, 4)
U8_VAL(uint8_t, storageClass, setStorageClass, 8)
FIELDS(
LE, uint8_t, version, setVersion,
LE, uint32_t, spanSize, setSpanSize,
LE, uint32_t, crc, setCrc,
LE, uint8_t, storageClass, setStorageClass,
EMIT_OFFSET, MIN_SIZE,
END
)
// after this:
// * Inline body for inline spans (bytes)
// * Blocks for normal spans
BincodeBytesRef inlineBody() const {
ALWAYS_ASSERT(storageClass() == INLINE_STORAGE);
@@ -337,22 +295,18 @@ enum class HashMode : uint8_t {
};
struct DirectoryBody {
char* _data;
FIELDS(
LE, uint8_t, version, setVersion,
LE, InodeId, ownerId, setOwnerId,
LE, EggsTime, mtime, setMtime,
LE, HashMode, hashMode, setHashMode,
LE, uint16_t, infoLength, setInfoLength,
EMIT_OFFSET, MIN_SIZE,
END
)
// After the static data we have a u16 length of the directory info, and then
// the directory info in bincode form, that is to say [tag: u8; len: u8; bytes; ...].
static constexpr size_t MIN_SIZE =
sizeof(InodeId) + // ownerId
sizeof(EggsTime) + // mtime
sizeof(HashMode) + // hashMode
sizeof(uint16_t); // infoLength
LE_VAL(InodeId, ownerId, setOwnerId, 0)
LE_VAL(EggsTime, mtime, setMtime, 8)
U8_VAL(HashMode, hashMode, setHashMode, 16)
LE_VAL(uint16_t, infoLength, setInfoLength, 17)
size_t size() const {
return MIN_SIZE + infoLength();
}
@@ -378,11 +332,18 @@ struct DirectoryBody {
};
struct EdgeKey {
char* _data;
FIELDS(
// first 63 bits: dir id, then whether the edge is current
BE, uint64_t, dirIdWithCurrentU64, setDirIdWithCurrentU64,
BE, uint64_t, nameHash, setNameHash,
EMIT_OFFSET, STATIC_SIZE,
BYTES, name, setName,
BE, EggsTime, creationTimeUnchecked, setCreationTimeUnchecked,
END
)
static constexpr size_t MIN_SIZE =
sizeof(uint64_t) + // first 63 bits: dirId, then whether the edge is current
sizeof(uint64_t) + // nameHash
STATIC_SIZE +
sizeof(uint8_t); // nameLength
// max name size, and an optional creation time if current=false
static constexpr size_t MAX_SIZE = MIN_SIZE + 255 + sizeof(EggsTime);
@@ -400,11 +361,6 @@ struct EdgeKey {
ALWAYS_ASSERT(sz == size());
}
BE64_VAL(uint64_t, dirIdWithCurrentU64, setDirIdWithCurrentU64, 0)
BE64_VAL(uint64_t, nameHash, setNameHash, 8)
BYTES_VAL(name, setName, 16)
BE64_VAL(EggsTime, creationTimeUnchecked, setCreationTimeUnchecked, (16+1+name().size()))
void setDirIdWithCurrent(InodeId id, bool current) {
setDirIdWithCurrentU64((id.u64 << 1) | current);
}
@@ -435,27 +391,20 @@ struct EdgeKey {
std::ostream& operator<<(std::ostream& out, const EdgeKey& edgeKey);
struct SnapshotEdgeBody {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(InodeIdExtra); // target, and if owned
size_t size() const { return MAX_SIZE; }
void checkSize(size_t sz) { ALWAYS_ASSERT(sz == MAX_SIZE); }
LE_VAL(InodeIdExtra, targetIdWithOwned, setTargetIdWithOwned, 0)
FIELDS(
LE, uint8_t, version, setVersion,
LE, InodeIdExtra, targetIdWithOwned, setTargetIdWithOwned,
END_STATIC
)
};
struct CurrentEdgeBody {
char* _data;
static constexpr size_t MAX_SIZE =
sizeof(InodeIdExtra) + // target, and if locked
sizeof(EggsTime); // creationTime
size_t size() const { return MAX_SIZE; }
void checkSize(size_t sz) { ALWAYS_ASSERT(sz == MAX_SIZE); }
LE_VAL(InodeIdExtra, targetIdWithLocked, setTargetIdWithLocked, 0)
LE_VAL(EggsTime, creationTime, setCreationTime, 8)
FIELDS(
LE, uint8_t, version, setVersion,
LE, InodeIdExtra, targetIdWithLocked, setTargetIdWithLocked,
LE, EggsTime, creationTime, setCreationTime,
END_STATIC
)
bool locked() const {
return targetIdWithLocked().extra();
@@ -475,6 +424,6 @@ struct BlockServiceToFileKey {
size_t size() const { return MAX_SIZE; }
void checkSize(size_t sz) { ALWAYS_ASSERT(sz == MAX_SIZE); }
BE64_VAL(BlockServiceId, blockServiceId, setBlockServiceId, 0)
BE64_VAL(InodeId, fileId, setFileId, 8)
BE_VAL(BlockServiceId, blockServiceId, setBlockServiceId, 0)
BE_VAL(InodeId, fileId, setFileId, 8)
};

4
cpp/tests/tests.cpp
View File

@@ -119,11 +119,12 @@ TEST_CASE("ShardDB data") {
auto transientFileId = InodeIdKey::Static({InodeType::FILE, ShardId(), 0});
BincodeBytes transientFileBodyNote("hello world");
StaticValue<TransientFileBody> transientFileBody;
transientFileBody().setVersion(0);
transientFileBody().setFileSize(123);
transientFileBody().setMtime({456});
transientFileBody().setDeadline({789});
transientFileBody().setLastSpanState(SpanState::CLEAN);
transientFileBody().setNote(transientFileBodyNote.ref());
transientFileBody().setNoteDangerous(transientFileBodyNote.ref());
ROCKS_DB_CHECKED(
db.db->Put({}, transientFileId.toSlice(), transientFileBody.toSlice())
@@ -145,6 +146,7 @@ TEST_CASE("ShardDB data") {
SUBCASE("File") {
auto fileId = InodeIdKey::Static({InodeType::FILE, ShardId(), 0});
StaticValue<FileBody> fileBody;
fileBody().setVersion(0);
fileBody().setMtime(123);
fileBody().setFileSize(456);