mirror/ternfs-XTXMarkets
1
0
Fork 0
mirror of https://github.com/XTXMarkets/ternfs.git synced 2026-01-06 11:00:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Linux 6.12

Browse Source
This commit is contained in:
Francesco Mazzoli
2025-10-02 12:04:54 +00:00
committed by Francesco Mazzoli
parent 2b0444e218
commit 03a2a08ec9
20 changed files with 5579 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/main.yml vendored
View File

@@ -119,9 +119,9 @@ jobs:
apt-get build-dep -y linux linux-image-unsigned-$(uname -r)
apt-get install -y fakeroot llvm libncurses-dev dwarves
- name: Download base qemu image
run: curl -fsSL -O https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img
run: curl -fsSL -O https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
- name: Run kmod tests
run: ./ci.py --short --prepare-image ${{ github.workspace }}/focal-server-cloudimg-amd64.img --kmod --leader-only
run: ./ci.py --short --prepare-image ${{ github.workspace }}/noble-server-cloudimg-amd64.img --kmod --leader-only
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v4

2
README.md
View File

@@ -171,7 +171,7 @@ Will run the integration tests as CI would (inside the Ubuntu docker image). You
To work with the qemu kmod tests you'll first need to download the base Ubuntu image we use for testing:
```
% wget -P kmod 'https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img'
% wget -P kmod 'https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img'
```
Then you can run the CI tests in kmod like so:

2
cpp/crc32c/crc32c_body.c
View File

@@ -223,7 +223,7 @@ static u64 crc32c_mul(u32 a, u32 b) {
#ifdef __clang__
__attribute__((no_sanitize("integer")))
#endif
u64 crc32c_mul(u32 a, u32 b32) {
static u64 crc32c_mul(u32 a, u32 b32) {
u64 b = (u64)b32 << 32;
u64 c = 0;
int i;

6
go/bincodegen/bincodegen.go
View File

@@ -713,6 +713,9 @@ func generateKmod(errors []string, shardReqResps []reqRespType, cdcReqResps []re
fmt.Fprintln(hOut, "// Automatically generated with go run bincodegen.")
fmt.Fprintln(hOut, "// Run `go generate ./...` from the go/ directory to regenerate it.")
fmt.Fprintln(hOut)
fmt.Fprintln(hOut, "#pragma GCC diagnostic push")
fmt.Fprintln(hOut, "#pragma GCC diagnostic ignored \"-Wunused-but-set-parameter\"")
fmt.Fprintln(hOut)
fmt.Fprintln(cOut, "// Copyright 2025 XTX Markets Technologies Limited")
fmt.Fprintln(cOut, "//")
@@ -773,6 +776,9 @@ func generateKmod(errors []string, shardReqResps []reqRespType, cdcReqResps []re
generateReqResps(registryReqResps)
generateReqResps(blocksReqResps)
fmt.Fprintln(hOut, "#pragma GCC diagnostic pop")
fmt.Fprintln(hOut)
return hOut.Bytes(), cOut.Bytes()
}

5
kmod/bincode.h
View File

@@ -9,7 +9,12 @@
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/bug.h>
#include <linux/version.h>
#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 12, 0))
#include <asm/unaligned.h>
#else
#include <linux/unaligned.h>
#endif
#endif
struct ternfs_bincode_put_ctx { char* start; char* cursor; char* end; };

5
kmod/bincodegen.h generated
View File

@@ -5,6 +5,9 @@
// Automatically generated with go run bincodegen.
// Run `go generate ./...` from the go/ directory to regenerate it.
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-but-set-parameter"
#define TERNFS_ERR_INTERNAL_ERROR 10
#define TERNFS_ERR_FATAL_ERROR 11
#define TERNFS_ERR_TIMEOUT 12
@@ -7006,3 +7009,5 @@ static inline void ternfs_fetch_block_with_crc_resp_get_finish(struct ternfs_bin
{ struct ternfs_fetch_block_with_crc_resp_start** __dummy __attribute__((unused)) = &(prev); }\
struct ternfs_fetch_block_with_crc_resp_end* next __attribute__((unused)) = NULL
#pragma GCC diagnostic pop

4
kmod/block.c
View File

@@ -1579,7 +1579,7 @@ int __init ternfs_block_init(void) {
"ternfs_fetch_block_request_cache",
sizeof(struct fetch_request),
0,
SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
SLAB_RECLAIM_ACCOUNT,
fetch_request_constructor
);
if (!fetch_request_cachep) { err = -ENOMEM; goto out_err; }
@@ -1588,7 +1588,7 @@ int __init ternfs_block_init(void) {
"ternfs_write_block_request_cache",
sizeof(struct write_request),
0,
SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
SLAB_RECLAIM_ACCOUNT,
write_request_constructor
);
if (!write_request_cachep) { err = -ENOMEM; goto out_fetch_request; }

0
kmod/config-kasan → kmod/config-kasan-5.4.237
View File

5479
kmod/config-kasan-6.12.49 Normal file
View File

File diff suppressed because it is too large Load Diff

4
kmod/dir.c
View File

@@ -562,7 +562,7 @@ int __init ternfs_dir_init(void) {
"ternfs_dirents_cache",
sizeof(struct ternfs_dirents),
0,
SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
SLAB_RECLAIM_ACCOUNT,
NULL
);
if (!ternfs_dirents_cachep) {
@@ -573,7 +573,7 @@ int __init ternfs_dir_init(void) {
"ternfs_readdir_ctx_cache",
sizeof(struct ternfs_readdir_ctx),
0,
SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
SLAB_RECLAIM_ACCOUNT,
NULL
);
if (!ternfs_readdir_ctx_cachep) {

14
kmod/fetchlinux.sh
View File

@@ -6,19 +6,21 @@
set -eu -o pipefail
version=linux-5.4.237
# version=5.4.237
version=6.12.49
# Download or resume
curl -C - -O "https://cdn.kernel.org/pub/linux/kernel/v5.x/${version}.tar.gz"
curl -C - -O "https://cdn.kernel.org/pub/linux/kernel/v${version%%.*}.x/linux-${version}.tar.gz"
# Check
sha512sum -c "${version}.tar.gz.sha512"
sha512sum -c "linux-${version}.tar.gz.sha512"
# Extract
tar xf "${version}.tar.gz"
tar xf "linux-${version}.tar.gz"
# Copy config
cp config-kasan ${version}/.config
cp "config-kasan-${version}" "linux-${version}/.config"
# Create symlink
ln -sf ${version} linux
rm -f linux
ln -sf linux-${version} linux

22
kmod/file.c
View File

@@ -8,6 +8,7 @@
#include <linux/sched/mm.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/version.h>
#include "bincode.h"
#include "inode.h"
@@ -175,6 +176,8 @@ static void hold_transient_span(struct ternfs_transient_span* span) {
atomic_inc(&span->refcount);
}
#define lru_to_page_impl(head) (list_entry((head)->prev, struct page, lru))
static bool put_transient_span(struct ternfs_transient_span* span) {
if (atomic_dec_return(&span->refcount) == 0) {
BUG_ON(spin_is_locked(&span->lock));
@@ -186,9 +189,9 @@ static bool put_transient_span(struct ternfs_transient_span* span) {
// It is however fine to adjust the OOM score here as technically
// the file system is now responsible for this memory and it's no
// longer tied to the process lifetime.
#define FREE_PAGES(__pages) \
while (!list_empty(__pages)) { \
struct page* victim = lru_to_page(__pages); \
#define FREE_PAGES(pages) \
while (!list_empty(pages)) { \
struct page* victim = lru_to_page_impl(pages); \
list_del(&victim->lru); \
put_page(victim); \
num_pages++; \
@@ -1110,7 +1113,11 @@ static loff_t file_lseek(struct file *file, loff_t offset, int whence) {
goto out;
}
file->f_pos = ppos;
file->f_version = 0; // what's this for?
// See 3352633ce6b221d64bf40644d412d9670e7d56e3 in linux repo, f_version
// was not well defined before anyway and they removed it.
#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 12, 0))
file->f_version = 0;
#endif
}
out:
@@ -1162,6 +1169,7 @@ static void process_file_pages(struct address_space *mapping, struct list_head *
}
}
#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0))
static int file_readpages(struct file *filp, struct address_space *mapping, struct list_head *pages, unsigned nr_pages) {
int err = 0;
@@ -1227,6 +1235,7 @@ out_err:
put_pages_list(pages);
return err;
}
#endif
static int file_readpage(struct file* filp, struct page* page) {
struct inode* inode = file_inode(filp);
@@ -1323,7 +1332,10 @@ retry:
out:
if (unlikely(err)) {
// SetPageError was removed because it was unused by that point
#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0))
SetPageError(page);
#endif
} else {
SetPageUptodate(page);
}
@@ -1494,7 +1506,7 @@ int __init ternfs_file_init(void) {
"ternfs_transient_span_cache",
sizeof(struct ternfs_transient_span),
0,
SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
SLAB_RECLAIM_ACCOUNT,
&init_transient_span
);
if (!ternfs_transient_span_cachep) { return -ENOMEM; }

2
kmod/gen_compile_commands.sh
View File

@@ -11,6 +11,6 @@ cd $SCRIPT_DIR
set -x
./linux/scripts/gen_compile_commands.py
./linux/scripts/clang-tools/gen_compile_commands.py
sed -i 's:-I.:-I./linux:g' ./compile_commands.json
sed -i 's:-include .:-include ./linux:g' ./compile_commands.json

2
kmod/inode.c
View File

@@ -83,7 +83,7 @@ static void ternfs_inode_init_once(void* ptr) {
int __init ternfs_inode_init(void) {
ternfs_inode_cachep = kmem_cache_create(
"ternfs_inode_cache", sizeof(struct ternfs_inode),
0, SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, ternfs_inode_init_once
0, SLAB_RECLAIM_ACCOUNT, ternfs_inode_init_once
);
if (!ternfs_inode_cachep) { return -ENOMEM; }

1
kmod/linux-6.12.49.tar.gz.sha512 Normal file
View File

@@ -0,0 +1 @@
0de0f50ca8ad1e90712d5dea3fcd362c1dbdc3f9ab488b8d5c4569857e756b4b7666d777d0d1079f3d5fc60930d731583cd0c3f61a97a5fbc6754289c92063a8 linux-6.12.49.tar.gz

6
kmod/registry.c
View File

@@ -2,8 +2,12 @@
//
// SPDX-License-Identifier: GPL-2.0-or-later
#include <asm/unaligned.h>
#include <linux/version.h>
#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 12, 0))
#include <asm/unaligned.h>
#else
#include <linux/unaligned.h>
#endif
#include <linux/module.h>
#include <linux/dns_resolver.h>
#include <linux/sunrpc/addr.h>

23
kmod/span.c
View File

@@ -506,6 +506,13 @@ retry:
put_fetch_span_pages(st);
}
#define lru_to_page_impl(head) (list_entry((head)->prev, struct page, lru))
static inline pgoff_t page_index_impl(struct page *page) {
return page->index;
}
int ternfs_span_get_pages(struct ternfs_block_span* block_span, struct address_space * mapping, struct list_head *pages, unsigned nr_pages, struct list_head *extra_pages) {
int err;
@@ -514,9 +521,9 @@ int ternfs_span_get_pages(struct ternfs_block_span* block_span, struct address_s
return -EIO;
}
loff_t off_start = page_offset(lru_to_page(pages));
loff_t off_start = page_offset(lru_to_page_impl(pages));
unsigned long page_ix;
unsigned long first_page_index = page_index(lru_to_page(pages));
unsigned long first_page_index = page_index_impl(lru_to_page_impl(pages));
// Work out start and end index for each block we need to fetch.
loff_t curr_off = off_start;
u64 span_offset = off_start - block_span->span.start;
@@ -587,7 +594,7 @@ int ternfs_span_get_pages(struct ternfs_block_span* block_span, struct address_s
for(i = 0; i < page_count; i++) {
page_ix = curr_off/PAGE_SIZE;
struct page *page = lru_to_page(pages);
struct page *page = lru_to_page_impl(pages);
if (page != NULL) {
if(page->index != page_ix) {
ternfs_warn("adding stripe_ix %d, block_ix=%d page_ix: have %ld, want %ld, curr_off=%lld, block_off=%d", stripe_ix, curr_block_ix, page->index, page_ix, curr_off, next_block_offset);
@@ -682,7 +689,7 @@ int ternfs_span_get_pages(struct ternfs_block_span* block_span, struct address_s
}
for (i = 0; i < num_zero_pages; i++) {
struct page* page = lru_to_page(pages);
struct page* page = lru_to_page_impl(pages);
BUG_ON(page == NULL);
list_del(&page->lru);
char* dst = kmap_atomic(page);
@@ -772,7 +779,7 @@ int ternfs_span_get_pages(struct ternfs_block_span* block_span, struct address_s
}
for (i = 0; i < num_zero_pages; i++) {
struct page *page = lru_to_page(&zero_pages);
struct page *page = lru_to_page_impl(&zero_pages);
BUG_ON(page == NULL);
list_del(&page->lru);
list_add_tail(&page->lru, pages);
@@ -1045,7 +1052,7 @@ int ternfs_span_init(void) {
"ternfs_block_span_cache",
sizeof(struct ternfs_block_span),
0,
SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
SLAB_RECLAIM_ACCOUNT,
NULL
);
if (!ternfs_block_span_cachep) {
@@ -1057,7 +1064,7 @@ int ternfs_span_init(void) {
"ternfs_inline_span_cache",
sizeof(struct ternfs_inline_span),
0,
SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
SLAB_RECLAIM_ACCOUNT,
offsetof(struct ternfs_inline_span, body),
sizeof(((struct ternfs_inline_span*)NULL)->body),
NULL
@@ -1070,7 +1077,7 @@ int ternfs_span_init(void) {
"ternfs_fetch_span_pages_cache",
sizeof(struct fetch_span_pages_state),
0,
SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD,
SLAB_RECLAIM_ACCOUNT,
&init_fetch_span_pages
);
if (!ternfs_fetch_span_pages_cachep) {

4
kmod/startvm.sh
View File

@@ -14,8 +14,8 @@ qemu-system-x86_64 \
-cpu host \
-kernel "${SCRIPT_DIR}/linux/arch/x86/boot/bzImage" \
-append "root=/dev/sda1 single console=ttyS0 systemd.unit=graphical.target" \
-drive file="${SCRIPT_DIR}/ubuntu.img",index=0,media=disk,cache=unsafe \
-drive file="${SCRIPT_DIR}/init.img",index=1,media=disk,cache=unsafe \
-drive file="${SCRIPT_DIR}/ubuntu.img,format=qcow2",index=0,media=disk,cache=unsafe \
-drive file="${SCRIPT_DIR}/init.img,format=raw",index=1,media=disk,cache=unsafe \
-m 128G \
-smp $(nproc),cores=$(nproc) \
-nographic \

10
kmod/sysctl.c
View File

@@ -35,12 +35,12 @@ int ternfs_debug_output = 0;
return 0;
static int drop_fetch_block_sockets;
static int ternfs_drop_fetch_block_sockets_sysctl(struct ctl_table* table, int write, void __sysctl_buffer* buffer, size_t* len, loff_t* ppos) {
static int ternfs_drop_fetch_block_sockets_sysctl(const struct ctl_table* table, int write, void __sysctl_buffer* buffer, size_t* len, loff_t* ppos) {
ternfs_do_sysctl(ternfs_drop_fetch_block_sockets);
}
static int drop_write_block_sockets;
static int ternfs_drop_write_block_sockets_sysctl(struct ctl_table* table, int write, void __sysctl_buffer* buffer, size_t* len, loff_t* ppos) {
static int ternfs_drop_write_block_sockets_sysctl(const struct ctl_table* table, int write, void __sysctl_buffer* buffer, size_t* len, loff_t* ppos) {
ternfs_do_sysctl(ternfs_drop_write_block_sockets);
}
@@ -154,13 +154,12 @@ static struct ctl_table ternfs_cb_sysctls[] = {
{}
};
#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0)
static struct ctl_table ternfs_cb_sysctl_dir[] = {
{
.procname = "eggsfs",
.mode = 0555,
#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0)
.child = ternfs_cb_sysctls,
#endif
},
{ }
};
@@ -169,12 +168,11 @@ static struct ctl_table ternfs_cb_sysctl_root[] = {
{
.procname = "fs",
.mode = 0555,
#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0)
.child = ternfs_cb_sysctl_dir,
#endif
},
{ }
};
#endif
static struct ctl_table_header* ternfs_callback_sysctl_table;

36
kmod/trace.h
View File

@@ -39,6 +39,7 @@ TRACE_DEFINE_ENUM(LOOKUP_JUMPED);
TRACE_DEFINE_ENUM(LOOKUP_ROOT);
#define show_lookup_flags(flags) \
__print_flags(flags, "|", \
{ LOOKUP_FOLLOW, "FOLLOW" }, \
{ LOOKUP_DIRECTORY, "DIRECTORY" }, \
@@ -54,6 +55,7 @@ TRACE_DEFINE_ENUM(LOOKUP_ROOT);
{ LOOKUP_ROOT, "ROOT" }, \
{ LOOKUP_EMPTY, "EMPTY" }, \
{ LOOKUP_DOWN, "DOWN" })
#else
#define show_lookup_flags(flags) \
@@ -72,6 +74,12 @@ TRACE_DEFINE_ENUM(LOOKUP_ROOT);
{ LOOKUP_DOWN, "DOWN" })
#endif
#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 10, 0))
#define __assign_str_impl(a, b) __assign_str(a, b)
#else
#define __assign_str_impl(a, b) __assign_str(a);
#endif
#define TERNFS_TRACE_EVENT_inode(_name) \
TRACE_EVENT(eggsfs_##_name, \
TP_PROTO(struct inode* inode), \
@@ -110,7 +118,7 @@ TRACE_DEFINE_ENUM(LOOKUP_ROOT);
), \
TP_fast_assign( \
__entry->dir = dir->i_ino; \
__assign_str(name, dentry->d_name.name); \
__assign_str_impl(name, dentry->d_name.name); \
), \
TP_printk("dir=%#llx name=%s", __entry->dir, __get_str(name)) \
)
@@ -127,7 +135,7 @@ TRACE_DEFINE_ENUM(LOOKUP_ROOT);
TP_fast_assign( \
__entry->dir = dir->i_ino; \
__entry->err = err; \
__assign_str(name, dentry->d_name.name); \
__assign_str_impl(name, dentry->d_name.name); \
), \
TP_printk("dir=%llx name=%s err=%d", __entry->dir, __get_str(name), __entry->err) \
)
@@ -144,7 +152,7 @@ TRACE_DEFINE_ENUM(LOOKUP_ROOT);
TP_fast_assign( \
__entry->dir = dir->i_ino; \
__entry->ino = inode ? inode->i_ino : 0; \
__assign_str(name, dentry->d_name.name); \
__assign_str_impl(name, dentry->d_name.name); \
), \
TP_printk("dir=%llx name=%s ino=%llx", __entry->dir, __get_str(name), __entry->ino) \
)
@@ -164,7 +172,7 @@ TRACE_DEFINE_ENUM(LOOKUP_ROOT);
__entry->dir = dir->i_ino; \
__entry->ino = inode ? inode->i_ino : 0; \
__entry->err = err; \
__assign_str(name, dentry->d_name.name); \
__assign_str_impl(name, dentry->d_name.name); \
), \
TP_printk("dir=%llx name=%s ino=%llx err=%d", __entry->dir, __get_str(name), __entry->ino, __entry->err) \
)
@@ -218,10 +226,10 @@ TRACE_EVENT(eggsfs_vfs_rename_enter,
__entry->old_dir_ino = old_dir->i_ino;
__entry->new_dir_ino = new_dir->i_ino;
__entry->ino = old_dentry->d_inode->i_ino;
__assign_str(old_parent, old_dentry->d_parent->d_name.name);
__assign_str(old_name, old_dentry->d_name.name)
__assign_str(new_parent, new_dentry->d_parent->d_name.name);
__assign_str(new_name, new_dentry->d_name.name)
__assign_str_impl(old_parent, old_dentry->d_parent->d_name.name);
__assign_str_impl(old_name, old_dentry->d_name.name)
__assign_str_impl(new_parent, new_dentry->d_parent->d_name.name);
__assign_str_impl(new_name, new_dentry->d_name.name)
),
TP_printk("ino=%lld old_dir=%lld old_name=%s/%s -> new_dir=%lld new_name=%s/%s", __entry->ino, __entry->old_dir_ino, __get_str(old_parent), __get_str(old_name), __entry->new_dir_ino, __get_str(new_parent), __get_str(new_name))
);
@@ -245,10 +253,10 @@ TRACE_EVENT(eggsfs_vfs_rename_exit,
__entry->new_dir_ino = new_dir->i_ino;
__entry->ino = old_dentry->d_inode->i_ino;
__entry->err = err;
__assign_str(old_parent, old_dentry->d_parent->d_name.name);
__assign_str(old_name, old_dentry->d_name.name)
__assign_str(new_parent, new_dentry->d_parent->d_name.name);
__assign_str(new_name, new_dentry->d_name.name)
__assign_str_impl(old_parent, old_dentry->d_parent->d_name.name);
__assign_str_impl(old_name, old_dentry->d_name.name)
__assign_str_impl(new_parent, new_dentry->d_parent->d_name.name);
__assign_str_impl(new_name, new_dentry->d_name.name)
),
TP_printk("ino=%lld old_dir=%lld old_name=%s/%s -> new_dir=%lld new_name=%s/%s err=%d", __entry->ino, __entry->old_dir_ino, __get_str(old_parent), __get_str(old_name), __entry->new_dir_ino, __get_str(new_parent), __get_str(new_name), __entry->err)
);
@@ -342,8 +350,8 @@ TRACE_EVENT(eggsfs_dentry_handle_enoent,
),
TP_fast_assign(
__entry->ino = dentry->d_inode ? dentry->d_inode->i_ino : 0;
__assign_str(name, dentry->d_parent->d_name.name);
__assign_str(name, dentry->d_name.name);
__assign_str_impl(name, dentry->d_parent->d_name.name);
__assign_str_impl(name, dentry->d_name.name);
),
TP_printk("ino=%lld name=%s/%s", __entry->ino, __get_str(parent), __get_str(name))
);