mirror/ternfs-XTXMarkets
1
0
Fork 0
mirror of https://github.com/XTXMarkets/ternfs.git synced 2026-01-01 08:21:50 -06:00
Code Issues Packages Projects Releases Wiki Activity
Files
cd86e632e2bda9d42c809ce0fa1adb4bf9f3ff39
ternfs-XTXMarkets/cpp/rs/rs.cpp
Francesco Mazzoli cd86e632e2 Implement RS recovery, although it won't really be used now... 
...since it only relies on block service flags, and we don't
set them right now.
2023-06-03 17:27:54 +00:00

489 lines
29 KiB
C++
Raw Blame History

#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <signal.h>
#include <immintrin.h>
#include <array>
#include "rs.h"
#define rs_warn(fmt, ...) fprintf(stderr, "rs: " fmt "\n" __VA_OPT__(,) __VA_ARGS__); raise(SIGABRT);
#define die(...) do { rs_warn(__VA_ARGS__); raise(SIGABRT); } while (0)
// See `valgrind.h`
static uint64_t rs_valgrind_client_request(uint64_t defaultResult, uint64_t reqID, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5) {
uint64_t args[6] = {reqID, arg1, arg2, arg3, arg4, arg5};
uint64_t result = defaultResult;
asm volatile ("rol $3, %%rdi\n\t"
"rol $13, %%rdi\n\t"
"rol $61, %%rdi\n\t"
"rol $51, %%rdi\n\t"
"xchg %%rbx, %%rbx\n\t"
: "+d" (result)
: "a" (args)
: "cc", "memory"
);
return result;
}
static bool rs_detect_valgrind() {
return rs_valgrind_client_request(0, 0x1001, 0, 0, 0, 0, 0);
}
// This will emit vbroadcastb
__attribute__((no_sanitize("integer")))
static inline __m256i broadcast_u8(uint8_t x) {
return _mm256_set_epi8(
x, x, x, x, x, x, x, x,
x, x, x, x, x, x, x, x,
x, x, x, x, x, x, x, x,
x, x, x, x, x, x, x, x
);
}
#include "rs_core.c"
uint8_t rs_parity(struct rs* r) {
return r->parity;
}
static struct rs* rs_cached[256] = {
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
};
bool rs_has_cpu_level(rs_cpu_level level) {
return rs_has_cpu_level_core(level);
}
static uint8_t rs_chosen_cpu_level = RS_CPU_SCALAR;
__attribute__((constructor))
void rs_detect_cpu_level() {
if (rs_has_cpu_level(RS_CPU_GFNI)) {
rs_chosen_cpu_level = RS_CPU_GFNI;
return;
}
if (rs_has_cpu_level(RS_CPU_AVX2)) {
rs_chosen_cpu_level = RS_CPU_AVX2;
return;
}
fprintf(stderr, "Picking scalar execution for RS library -- this will be slow.\n");
rs_chosen_cpu_level = RS_CPU_SCALAR;
return;
}
void rs_set_cpu_level(rs_cpu_level level) {
__atomic_store_n(&rs_chosen_cpu_level, level, __ATOMIC_RELAXED);
}
rs_cpu_level rs_get_cpu_level() {
return (rs_cpu_level)__atomic_load_n(&rs_chosen_cpu_level, __ATOMIC_RELAXED);
}
struct rs* rs_get(uint8_t parity) {
if (rs_data_blocks(parity) < 2 || rs_parity_blocks(parity) < 1) {
die("bad parity (%d,%d), expected at least 2 data blocks and 1 parity block.\n", rs_data_blocks(parity), rs_parity_blocks(parity));
}
struct rs* r = __atomic_load_n(&rs_cached[parity], __ATOMIC_RELAXED);
if (__builtin_expect(r == nullptr, 0)) {
r = (struct rs*)malloc(RS_SIZE(rs_data_blocks(parity), rs_parity_blocks(parity)));
rs_new_core(parity, r);
if (r == nullptr) {
die("could not allocate RS data");
}
struct rs* expected = nullptr;
if (!__atomic_compare_exchange_n(&rs_cached[parity], &expected, r, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
// somebody else got to it first
free(r);
r = __atomic_load_n(&rs_cached[parity], __ATOMIC_RELAXED);
}
}
return r;
}
template<int D, int P> __attribute__((noinline))
static void rs_compute_parity_scalar_tmpl(struct rs* r, uint64_t size, const uint8_t** data, uint8_t** parity) {
rs_compute_parity_scalar(D, P, r, size, data, parity);
}
template<int D, int P> __attribute__((noinline))
static void rs_compute_parity_avx2_tmpl(struct rs* r, uint64_t size, const uint8_t** data, uint8_t** parity) {
rs_compute_parity_avx2(D, P, r, size, data, parity);
}
template<int D, int P> __attribute__((noinline))
static void rs_compute_parity_gfni_tmpl(struct rs* r, uint64_t size, const uint8_t** data, uint8_t** parity) {
rs_compute_parity_gfni(D, P, r, size, data, parity);
}
template<int D, int P>
static void rs_compute_parity_tmpl(struct rs* r, uint64_t size, const uint8_t** data, uint8_t** parity) {
switch (rs_cpu_level l = rs_get_cpu_level()) {
case RS_CPU_SCALAR:
rs_compute_parity_scalar_tmpl<D, P>(r, size, data, parity);
break;
case RS_CPU_AVX2:
rs_compute_parity_avx2_tmpl<D, P>(r, size, data, parity);
break;
case RS_CPU_GFNI:
rs_compute_parity_gfni_tmpl<D, P>(r, size, data, parity);
break;
default:
die("bad cpu_level %d\n", l);
}
}
static void (*rs_compute_parity_funcs[256])(struct rs* r, uint64_t size, const uint8_t** data, uint8_t** parity);
void rs_compute_parity(struct rs* r, uint64_t size, const uint8_t** data, uint8_t** parity) {
rs_compute_parity_funcs[r->parity](r, size, data, parity);
}
template<int D> __attribute__((noinline))
static void rs_recover_matmul_scalar_tmpl(uint64_t size, const uint8_t** have, uint8_t* want, const uint8_t* mat) {
rs_recover_matmul_scalar(D, size, have, want, mat);
}
template<int D> __attribute__((noinline))
static void rs_recover_matmul_avx2_tmpl(uint64_t size, const uint8_t** have, uint8_t* want, const uint8_t* mat) {
rs_recover_matmul_avx2(D, size, have, want, mat);
}
template<int D> __attribute__((noinline))
static void rs_recover_matmul_gfni_tmpl(uint64_t size, const uint8_t** have, uint8_t* want, const uint8_t* mat) {
rs_recover_matmul_gfni(D, size, have, want, mat);
}
template<int D>
static void rs_recover_matmul_tmpl(uint64_t size, const uint8_t** have, uint8_t* want, const uint8_t* mat) {
switch (rs_cpu_level l = rs_get_cpu_level()) {
case RS_CPU_SCALAR:
rs_recover_matmul_scalar_tmpl<D>(size, have, want, mat);
break;
case RS_CPU_AVX2:
rs_recover_matmul_avx2_tmpl<D>(size, have, want, mat);
break;
case RS_CPU_GFNI:
rs_recover_matmul_gfni_tmpl<D>(size, have, want, mat);
break;
default:
die("bad cpu_level %d\n", l);
}
}
static void (*rs_recover_matmul_funcs[16])(uint64_t size, const uint8_t** have, uint8_t* want, const uint8_t* mat);
void rs_recover(
struct rs* r,
uint64_t size,
uint32_t have_blocks,
const uint8_t** have,
uint32_t want_block,
uint8_t* want
) {
int D = rs_data_blocks(r->parity);
u8* mat = (u8*)malloc(RS_RECOVER_MAT_SIZE(D));
if (mat == NULL) {
free(mat);
die("could not allocate mat");
}
if (!rs_recover_mat(r, have_blocks, want_block, mat)) {
free(mat);
die("could not get recover matrix");
}
rs_recover_matmul_funcs[D](size, have, want, mat);
free(mat);
}
__attribute__((constructor))
static void rs_initialize_compute_parity_funcs() {
rs_compute_parity_funcs[rs_mk_parity(0, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 1)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 2)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 3)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 4)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 5)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 6)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 7)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 8)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 9)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 10)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 11)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 12)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 13)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 14)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(0, 15)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 1)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 2)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 3)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 4)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 5)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 6)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 7)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 8)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 9)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 10)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 11)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 12)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 13)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 14)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(1, 15)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(2, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(2, 1)] = &rs_compute_parity_tmpl<2, 1>;
rs_compute_parity_funcs[rs_mk_parity(2, 2)] = &rs_compute_parity_tmpl<2, 2>;
rs_compute_parity_funcs[rs_mk_parity(2, 3)] = &rs_compute_parity_tmpl<2, 3>;
rs_compute_parity_funcs[rs_mk_parity(2, 4)] = &rs_compute_parity_tmpl<2, 4>;
rs_compute_parity_funcs[rs_mk_parity(2, 5)] = &rs_compute_parity_tmpl<2, 5>;
rs_compute_parity_funcs[rs_mk_parity(2, 6)] = &rs_compute_parity_tmpl<2, 6>;
rs_compute_parity_funcs[rs_mk_parity(2, 7)] = &rs_compute_parity_tmpl<2, 7>;
rs_compute_parity_funcs[rs_mk_parity(2, 8)] = &rs_compute_parity_tmpl<2, 8>;
rs_compute_parity_funcs[rs_mk_parity(2, 9)] = &rs_compute_parity_tmpl<2, 9>;
rs_compute_parity_funcs[rs_mk_parity(2, 10)] = &rs_compute_parity_tmpl<2, 10>;
rs_compute_parity_funcs[rs_mk_parity(2, 11)] = &rs_compute_parity_tmpl<2, 11>;
rs_compute_parity_funcs[rs_mk_parity(2, 12)] = &rs_compute_parity_tmpl<2, 12>;
rs_compute_parity_funcs[rs_mk_parity(2, 13)] = &rs_compute_parity_tmpl<2, 13>;
rs_compute_parity_funcs[rs_mk_parity(2, 14)] = &rs_compute_parity_tmpl<2, 14>;
rs_compute_parity_funcs[rs_mk_parity(2, 15)] = &rs_compute_parity_tmpl<2, 15>;
rs_compute_parity_funcs[rs_mk_parity(3, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(3, 1)] = &rs_compute_parity_tmpl<3, 1>;
rs_compute_parity_funcs[rs_mk_parity(3, 2)] = &rs_compute_parity_tmpl<3, 2>;
rs_compute_parity_funcs[rs_mk_parity(3, 3)] = &rs_compute_parity_tmpl<3, 3>;
rs_compute_parity_funcs[rs_mk_parity(3, 4)] = &rs_compute_parity_tmpl<3, 4>;
rs_compute_parity_funcs[rs_mk_parity(3, 5)] = &rs_compute_parity_tmpl<3, 5>;
rs_compute_parity_funcs[rs_mk_parity(3, 6)] = &rs_compute_parity_tmpl<3, 6>;
rs_compute_parity_funcs[rs_mk_parity(3, 7)] = &rs_compute_parity_tmpl<3, 7>;
rs_compute_parity_funcs[rs_mk_parity(3, 8)] = &rs_compute_parity_tmpl<3, 8>;
rs_compute_parity_funcs[rs_mk_parity(3, 9)] = &rs_compute_parity_tmpl<3, 9>;
rs_compute_parity_funcs[rs_mk_parity(3, 10)] = &rs_compute_parity_tmpl<3, 10>;
rs_compute_parity_funcs[rs_mk_parity(3, 11)] = &rs_compute_parity_tmpl<3, 11>;
rs_compute_parity_funcs[rs_mk_parity(3, 12)] = &rs_compute_parity_tmpl<3, 12>;
rs_compute_parity_funcs[rs_mk_parity(3, 13)] = &rs_compute_parity_tmpl<3, 13>;
rs_compute_parity_funcs[rs_mk_parity(3, 14)] = &rs_compute_parity_tmpl<3, 14>;
rs_compute_parity_funcs[rs_mk_parity(3, 15)] = &rs_compute_parity_tmpl<3, 15>;
rs_compute_parity_funcs[rs_mk_parity(4, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(4, 1)] = &rs_compute_parity_tmpl<4, 1>;
rs_compute_parity_funcs[rs_mk_parity(4, 2)] = &rs_compute_parity_tmpl<4, 2>;
rs_compute_parity_funcs[rs_mk_parity(4, 3)] = &rs_compute_parity_tmpl<4, 3>;
rs_compute_parity_funcs[rs_mk_parity(4, 4)] = &rs_compute_parity_tmpl<4, 4>;
rs_compute_parity_funcs[rs_mk_parity(4, 5)] = &rs_compute_parity_tmpl<4, 5>;
rs_compute_parity_funcs[rs_mk_parity(4, 6)] = &rs_compute_parity_tmpl<4, 6>;
rs_compute_parity_funcs[rs_mk_parity(4, 7)] = &rs_compute_parity_tmpl<4, 7>;
rs_compute_parity_funcs[rs_mk_parity(4, 8)] = &rs_compute_parity_tmpl<4, 8>;
rs_compute_parity_funcs[rs_mk_parity(4, 9)] = &rs_compute_parity_tmpl<4, 9>;
rs_compute_parity_funcs[rs_mk_parity(4, 10)] = &rs_compute_parity_tmpl<4, 10>;
rs_compute_parity_funcs[rs_mk_parity(4, 11)] = &rs_compute_parity_tmpl<4, 11>;
rs_compute_parity_funcs[rs_mk_parity(4, 12)] = &rs_compute_parity_tmpl<4, 12>;
rs_compute_parity_funcs[rs_mk_parity(4, 13)] = &rs_compute_parity_tmpl<4, 13>;
rs_compute_parity_funcs[rs_mk_parity(4, 14)] = &rs_compute_parity_tmpl<4, 14>;
rs_compute_parity_funcs[rs_mk_parity(4, 15)] = &rs_compute_parity_tmpl<4, 15>;
rs_compute_parity_funcs[rs_mk_parity(5, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(5, 1)] = &rs_compute_parity_tmpl<5, 1>;
rs_compute_parity_funcs[rs_mk_parity(5, 2)] = &rs_compute_parity_tmpl<5, 2>;
rs_compute_parity_funcs[rs_mk_parity(5, 3)] = &rs_compute_parity_tmpl<5, 3>;
rs_compute_parity_funcs[rs_mk_parity(5, 4)] = &rs_compute_parity_tmpl<5, 4>;
rs_compute_parity_funcs[rs_mk_parity(5, 5)] = &rs_compute_parity_tmpl<5, 5>;
rs_compute_parity_funcs[rs_mk_parity(5, 6)] = &rs_compute_parity_tmpl<5, 6>;
rs_compute_parity_funcs[rs_mk_parity(5, 7)] = &rs_compute_parity_tmpl<5, 7>;
rs_compute_parity_funcs[rs_mk_parity(5, 8)] = &rs_compute_parity_tmpl<5, 8>;
rs_compute_parity_funcs[rs_mk_parity(5, 9)] = &rs_compute_parity_tmpl<5, 9>;
rs_compute_parity_funcs[rs_mk_parity(5, 10)] = &rs_compute_parity_tmpl<5, 10>;
rs_compute_parity_funcs[rs_mk_parity(5, 11)] = &rs_compute_parity_tmpl<5, 11>;
rs_compute_parity_funcs[rs_mk_parity(5, 12)] = &rs_compute_parity_tmpl<5, 12>;
rs_compute_parity_funcs[rs_mk_parity(5, 13)] = &rs_compute_parity_tmpl<5, 13>;
rs_compute_parity_funcs[rs_mk_parity(5, 14)] = &rs_compute_parity_tmpl<5, 14>;
rs_compute_parity_funcs[rs_mk_parity(5, 15)] = &rs_compute_parity_tmpl<5, 15>;
rs_compute_parity_funcs[rs_mk_parity(6, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(6, 1)] = &rs_compute_parity_tmpl<6, 1>;
rs_compute_parity_funcs[rs_mk_parity(6, 2)] = &rs_compute_parity_tmpl<6, 2>;
rs_compute_parity_funcs[rs_mk_parity(6, 3)] = &rs_compute_parity_tmpl<6, 3>;
rs_compute_parity_funcs[rs_mk_parity(6, 4)] = &rs_compute_parity_tmpl<6, 4>;
rs_compute_parity_funcs[rs_mk_parity(6, 5)] = &rs_compute_parity_tmpl<6, 5>;
rs_compute_parity_funcs[rs_mk_parity(6, 6)] = &rs_compute_parity_tmpl<6, 6>;
rs_compute_parity_funcs[rs_mk_parity(6, 7)] = &rs_compute_parity_tmpl<6, 7>;
rs_compute_parity_funcs[rs_mk_parity(6, 8)] = &rs_compute_parity_tmpl<6, 8>;
rs_compute_parity_funcs[rs_mk_parity(6, 9)] = &rs_compute_parity_tmpl<6, 9>;
rs_compute_parity_funcs[rs_mk_parity(6, 10)] = &rs_compute_parity_tmpl<6, 10>;
rs_compute_parity_funcs[rs_mk_parity(6, 11)] = &rs_compute_parity_tmpl<6, 11>;
rs_compute_parity_funcs[rs_mk_parity(6, 12)] = &rs_compute_parity_tmpl<6, 12>;
rs_compute_parity_funcs[rs_mk_parity(6, 13)] = &rs_compute_parity_tmpl<6, 13>;
rs_compute_parity_funcs[rs_mk_parity(6, 14)] = &rs_compute_parity_tmpl<6, 14>;
rs_compute_parity_funcs[rs_mk_parity(6, 15)] = &rs_compute_parity_tmpl<6, 15>;
rs_compute_parity_funcs[rs_mk_parity(7, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(7, 1)] = &rs_compute_parity_tmpl<7, 1>;
rs_compute_parity_funcs[rs_mk_parity(7, 2)] = &rs_compute_parity_tmpl<7, 2>;
rs_compute_parity_funcs[rs_mk_parity(7, 3)] = &rs_compute_parity_tmpl<7, 3>;
rs_compute_parity_funcs[rs_mk_parity(7, 4)] = &rs_compute_parity_tmpl<7, 4>;
rs_compute_parity_funcs[rs_mk_parity(7, 5)] = &rs_compute_parity_tmpl<7, 5>;
rs_compute_parity_funcs[rs_mk_parity(7, 6)] = &rs_compute_parity_tmpl<7, 6>;
rs_compute_parity_funcs[rs_mk_parity(7, 7)] = &rs_compute_parity_tmpl<7, 7>;
rs_compute_parity_funcs[rs_mk_parity(7, 8)] = &rs_compute_parity_tmpl<7, 8>;
rs_compute_parity_funcs[rs_mk_parity(7, 9)] = &rs_compute_parity_tmpl<7, 9>;
rs_compute_parity_funcs[rs_mk_parity(7, 10)] = &rs_compute_parity_tmpl<7, 10>;
rs_compute_parity_funcs[rs_mk_parity(7, 11)] = &rs_compute_parity_tmpl<7, 11>;
rs_compute_parity_funcs[rs_mk_parity(7, 12)] = &rs_compute_parity_tmpl<7, 12>;
rs_compute_parity_funcs[rs_mk_parity(7, 13)] = &rs_compute_parity_tmpl<7, 13>;
rs_compute_parity_funcs[rs_mk_parity(7, 14)] = &rs_compute_parity_tmpl<7, 14>;
rs_compute_parity_funcs[rs_mk_parity(7, 15)] = &rs_compute_parity_tmpl<7, 15>;
rs_compute_parity_funcs[rs_mk_parity(8, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(8, 1)] = &rs_compute_parity_tmpl<8, 1>;
rs_compute_parity_funcs[rs_mk_parity(8, 2)] = &rs_compute_parity_tmpl<8, 2>;
rs_compute_parity_funcs[rs_mk_parity(8, 3)] = &rs_compute_parity_tmpl<8, 3>;
rs_compute_parity_funcs[rs_mk_parity(8, 4)] = &rs_compute_parity_tmpl<8, 4>;
rs_compute_parity_funcs[rs_mk_parity(8, 5)] = &rs_compute_parity_tmpl<8, 5>;
rs_compute_parity_funcs[rs_mk_parity(8, 6)] = &rs_compute_parity_tmpl<8, 6>;
rs_compute_parity_funcs[rs_mk_parity(8, 7)] = &rs_compute_parity_tmpl<8, 7>;
rs_compute_parity_funcs[rs_mk_parity(8, 8)] = &rs_compute_parity_tmpl<8, 8>;
rs_compute_parity_funcs[rs_mk_parity(8, 9)] = &rs_compute_parity_tmpl<8, 9>;
rs_compute_parity_funcs[rs_mk_parity(8, 10)] = &rs_compute_parity_tmpl<8, 10>;
rs_compute_parity_funcs[rs_mk_parity(8, 11)] = &rs_compute_parity_tmpl<8, 11>;
rs_compute_parity_funcs[rs_mk_parity(8, 12)] = &rs_compute_parity_tmpl<8, 12>;
rs_compute_parity_funcs[rs_mk_parity(8, 13)] = &rs_compute_parity_tmpl<8, 13>;
rs_compute_parity_funcs[rs_mk_parity(8, 14)] = &rs_compute_parity_tmpl<8, 14>;
rs_compute_parity_funcs[rs_mk_parity(8, 15)] = &rs_compute_parity_tmpl<8, 15>;
rs_compute_parity_funcs[rs_mk_parity(9, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(9, 1)] = &rs_compute_parity_tmpl<9, 1>;
rs_compute_parity_funcs[rs_mk_parity(9, 2)] = &rs_compute_parity_tmpl<9, 2>;
rs_compute_parity_funcs[rs_mk_parity(9, 3)] = &rs_compute_parity_tmpl<9, 3>;
rs_compute_parity_funcs[rs_mk_parity(9, 4)] = &rs_compute_parity_tmpl<9, 4>;
rs_compute_parity_funcs[rs_mk_parity(9, 5)] = &rs_compute_parity_tmpl<9, 5>;
rs_compute_parity_funcs[rs_mk_parity(9, 6)] = &rs_compute_parity_tmpl<9, 6>;
rs_compute_parity_funcs[rs_mk_parity(9, 7)] = &rs_compute_parity_tmpl<9, 7>;
rs_compute_parity_funcs[rs_mk_parity(9, 8)] = &rs_compute_parity_tmpl<9, 8>;
rs_compute_parity_funcs[rs_mk_parity(9, 9)] = &rs_compute_parity_tmpl<9, 9>;
rs_compute_parity_funcs[rs_mk_parity(9, 10)] = &rs_compute_parity_tmpl<9, 10>;
rs_compute_parity_funcs[rs_mk_parity(9, 11)] = &rs_compute_parity_tmpl<9, 11>;
rs_compute_parity_funcs[rs_mk_parity(9, 12)] = &rs_compute_parity_tmpl<9, 12>;
rs_compute_parity_funcs[rs_mk_parity(9, 13)] = &rs_compute_parity_tmpl<9, 13>;
rs_compute_parity_funcs[rs_mk_parity(9, 14)] = &rs_compute_parity_tmpl<9, 14>;
rs_compute_parity_funcs[rs_mk_parity(9, 15)] = &rs_compute_parity_tmpl<9, 15>;
rs_compute_parity_funcs[rs_mk_parity(10, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(10, 1)] = &rs_compute_parity_tmpl<10, 1>;
rs_compute_parity_funcs[rs_mk_parity(10, 2)] = &rs_compute_parity_tmpl<10, 2>;
rs_compute_parity_funcs[rs_mk_parity(10, 3)] = &rs_compute_parity_tmpl<10, 3>;
rs_compute_parity_funcs[rs_mk_parity(10, 4)] = &rs_compute_parity_tmpl<10, 4>;
rs_compute_parity_funcs[rs_mk_parity(10, 5)] = &rs_compute_parity_tmpl<10, 5>;
rs_compute_parity_funcs[rs_mk_parity(10, 6)] = &rs_compute_parity_tmpl<10, 6>;
rs_compute_parity_funcs[rs_mk_parity(10, 7)] = &rs_compute_parity_tmpl<10, 7>;
rs_compute_parity_funcs[rs_mk_parity(10, 8)] = &rs_compute_parity_tmpl<10, 8>;
rs_compute_parity_funcs[rs_mk_parity(10, 9)] = &rs_compute_parity_tmpl<10, 9>;
rs_compute_parity_funcs[rs_mk_parity(10, 10)] = &rs_compute_parity_tmpl<10, 10>;
rs_compute_parity_funcs[rs_mk_parity(10, 11)] = &rs_compute_parity_tmpl<10, 11>;
rs_compute_parity_funcs[rs_mk_parity(10, 12)] = &rs_compute_parity_tmpl<10, 12>;
rs_compute_parity_funcs[rs_mk_parity(10, 13)] = &rs_compute_parity_tmpl<10, 13>;
rs_compute_parity_funcs[rs_mk_parity(10, 14)] = &rs_compute_parity_tmpl<10, 14>;
rs_compute_parity_funcs[rs_mk_parity(10, 15)] = &rs_compute_parity_tmpl<10, 15>;
rs_compute_parity_funcs[rs_mk_parity(11, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(11, 1)] = &rs_compute_parity_tmpl<11, 1>;
rs_compute_parity_funcs[rs_mk_parity(11, 2)] = &rs_compute_parity_tmpl<11, 2>;
rs_compute_parity_funcs[rs_mk_parity(11, 3)] = &rs_compute_parity_tmpl<11, 3>;
rs_compute_parity_funcs[rs_mk_parity(11, 4)] = &rs_compute_parity_tmpl<11, 4>;
rs_compute_parity_funcs[rs_mk_parity(11, 5)] = &rs_compute_parity_tmpl<11, 5>;
rs_compute_parity_funcs[rs_mk_parity(11, 6)] = &rs_compute_parity_tmpl<11, 6>;
rs_compute_parity_funcs[rs_mk_parity(11, 7)] = &rs_compute_parity_tmpl<11, 7>;
rs_compute_parity_funcs[rs_mk_parity(11, 8)] = &rs_compute_parity_tmpl<11, 8>;
rs_compute_parity_funcs[rs_mk_parity(11, 9)] = &rs_compute_parity_tmpl<11, 9>;
rs_compute_parity_funcs[rs_mk_parity(11, 10)] = &rs_compute_parity_tmpl<11, 10>;
rs_compute_parity_funcs[rs_mk_parity(11, 11)] = &rs_compute_parity_tmpl<11, 11>;
rs_compute_parity_funcs[rs_mk_parity(11, 12)] = &rs_compute_parity_tmpl<11, 12>;
rs_compute_parity_funcs[rs_mk_parity(11, 13)] = &rs_compute_parity_tmpl<11, 13>;
rs_compute_parity_funcs[rs_mk_parity(11, 14)] = &rs_compute_parity_tmpl<11, 14>;
rs_compute_parity_funcs[rs_mk_parity(11, 15)] = &rs_compute_parity_tmpl<11, 15>;
rs_compute_parity_funcs[rs_mk_parity(12, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(12, 1)] = &rs_compute_parity_tmpl<12, 1>;
rs_compute_parity_funcs[rs_mk_parity(12, 2)] = &rs_compute_parity_tmpl<12, 2>;
rs_compute_parity_funcs[rs_mk_parity(12, 3)] = &rs_compute_parity_tmpl<12, 3>;
rs_compute_parity_funcs[rs_mk_parity(12, 4)] = &rs_compute_parity_tmpl<12, 4>;
rs_compute_parity_funcs[rs_mk_parity(12, 5)] = &rs_compute_parity_tmpl<12, 5>;
rs_compute_parity_funcs[rs_mk_parity(12, 6)] = &rs_compute_parity_tmpl<12, 6>;
rs_compute_parity_funcs[rs_mk_parity(12, 7)] = &rs_compute_parity_tmpl<12, 7>;
rs_compute_parity_funcs[rs_mk_parity(12, 8)] = &rs_compute_parity_tmpl<12, 8>;
rs_compute_parity_funcs[rs_mk_parity(12, 9)] = &rs_compute_parity_tmpl<12, 9>;
rs_compute_parity_funcs[rs_mk_parity(12, 10)] = &rs_compute_parity_tmpl<12, 10>;
rs_compute_parity_funcs[rs_mk_parity(12, 11)] = &rs_compute_parity_tmpl<12, 11>;
rs_compute_parity_funcs[rs_mk_parity(12, 12)] = &rs_compute_parity_tmpl<12, 12>;
rs_compute_parity_funcs[rs_mk_parity(12, 13)] = &rs_compute_parity_tmpl<12, 13>;
rs_compute_parity_funcs[rs_mk_parity(12, 14)] = &rs_compute_parity_tmpl<12, 14>;
rs_compute_parity_funcs[rs_mk_parity(12, 15)] = &rs_compute_parity_tmpl<12, 15>;
rs_compute_parity_funcs[rs_mk_parity(13, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(13, 1)] = &rs_compute_parity_tmpl<13, 1>;
rs_compute_parity_funcs[rs_mk_parity(13, 2)] = &rs_compute_parity_tmpl<13, 2>;
rs_compute_parity_funcs[rs_mk_parity(13, 3)] = &rs_compute_parity_tmpl<13, 3>;
rs_compute_parity_funcs[rs_mk_parity(13, 4)] = &rs_compute_parity_tmpl<13, 4>;
rs_compute_parity_funcs[rs_mk_parity(13, 5)] = &rs_compute_parity_tmpl<13, 5>;
rs_compute_parity_funcs[rs_mk_parity(13, 6)] = &rs_compute_parity_tmpl<13, 6>;
rs_compute_parity_funcs[rs_mk_parity(13, 7)] = &rs_compute_parity_tmpl<13, 7>;
rs_compute_parity_funcs[rs_mk_parity(13, 8)] = &rs_compute_parity_tmpl<13, 8>;
rs_compute_parity_funcs[rs_mk_parity(13, 9)] = &rs_compute_parity_tmpl<13, 9>;
rs_compute_parity_funcs[rs_mk_parity(13, 10)] = &rs_compute_parity_tmpl<13, 10>;
rs_compute_parity_funcs[rs_mk_parity(13, 11)] = &rs_compute_parity_tmpl<13, 11>;
rs_compute_parity_funcs[rs_mk_parity(13, 12)] = &rs_compute_parity_tmpl<13, 12>;
rs_compute_parity_funcs[rs_mk_parity(13, 13)] = &rs_compute_parity_tmpl<13, 13>;
rs_compute_parity_funcs[rs_mk_parity(13, 14)] = &rs_compute_parity_tmpl<13, 14>;
rs_compute_parity_funcs[rs_mk_parity(13, 15)] = &rs_compute_parity_tmpl<13, 15>;
rs_compute_parity_funcs[rs_mk_parity(14, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(14, 1)] = &rs_compute_parity_tmpl<14, 1>;
rs_compute_parity_funcs[rs_mk_parity(14, 2)] = &rs_compute_parity_tmpl<14, 2>;
rs_compute_parity_funcs[rs_mk_parity(14, 3)] = &rs_compute_parity_tmpl<14, 3>;
rs_compute_parity_funcs[rs_mk_parity(14, 4)] = &rs_compute_parity_tmpl<14, 4>;
rs_compute_parity_funcs[rs_mk_parity(14, 5)] = &rs_compute_parity_tmpl<14, 5>;
rs_compute_parity_funcs[rs_mk_parity(14, 6)] = &rs_compute_parity_tmpl<14, 6>;
rs_compute_parity_funcs[rs_mk_parity(14, 7)] = &rs_compute_parity_tmpl<14, 7>;
rs_compute_parity_funcs[rs_mk_parity(14, 8)] = &rs_compute_parity_tmpl<14, 8>;
rs_compute_parity_funcs[rs_mk_parity(14, 9)] = &rs_compute_parity_tmpl<14, 9>;
rs_compute_parity_funcs[rs_mk_parity(14, 10)] = &rs_compute_parity_tmpl<14, 10>;
rs_compute_parity_funcs[rs_mk_parity(14, 11)] = &rs_compute_parity_tmpl<14, 11>;
rs_compute_parity_funcs[rs_mk_parity(14, 12)] = &rs_compute_parity_tmpl<14, 12>;
rs_compute_parity_funcs[rs_mk_parity(14, 13)] = &rs_compute_parity_tmpl<14, 13>;
rs_compute_parity_funcs[rs_mk_parity(14, 14)] = &rs_compute_parity_tmpl<14, 14>;
rs_compute_parity_funcs[rs_mk_parity(14, 15)] = &rs_compute_parity_tmpl<14, 15>;
rs_compute_parity_funcs[rs_mk_parity(15, 0)] = nullptr;
rs_compute_parity_funcs[rs_mk_parity(15, 1)] = &rs_compute_parity_tmpl<15, 1>;
rs_compute_parity_funcs[rs_mk_parity(15, 2)] = &rs_compute_parity_tmpl<15, 2>;
rs_compute_parity_funcs[rs_mk_parity(15, 3)] = &rs_compute_parity_tmpl<15, 3>;
rs_compute_parity_funcs[rs_mk_parity(15, 4)] = &rs_compute_parity_tmpl<15, 4>;
rs_compute_parity_funcs[rs_mk_parity(15, 5)] = &rs_compute_parity_tmpl<15, 5>;
rs_compute_parity_funcs[rs_mk_parity(15, 6)] = &rs_compute_parity_tmpl<15, 6>;
rs_compute_parity_funcs[rs_mk_parity(15, 7)] = &rs_compute_parity_tmpl<15, 7>;
rs_compute_parity_funcs[rs_mk_parity(15, 8)] = &rs_compute_parity_tmpl<15, 8>;
rs_compute_parity_funcs[rs_mk_parity(15, 9)] = &rs_compute_parity_tmpl<15, 9>;
rs_compute_parity_funcs[rs_mk_parity(15, 10)] = &rs_compute_parity_tmpl<15, 10>;
rs_compute_parity_funcs[rs_mk_parity(15, 11)] = &rs_compute_parity_tmpl<15, 11>;
rs_compute_parity_funcs[rs_mk_parity(15, 12)] = &rs_compute_parity_tmpl<15, 12>;
rs_compute_parity_funcs[rs_mk_parity(15, 13)] = &rs_compute_parity_tmpl<15, 13>;
rs_compute_parity_funcs[rs_mk_parity(15, 14)] = &rs_compute_parity_tmpl<15, 14>;
rs_compute_parity_funcs[rs_mk_parity(15, 15)] = &rs_compute_parity_tmpl<15, 15>;
}
__attribute__((constructor))
static void rs_initialize_recover_matmul_funcs() {
rs_recover_matmul_funcs[0] = nullptr;
rs_recover_matmul_funcs[1] = nullptr;
rs_recover_matmul_funcs[2] = &rs_recover_matmul_tmpl<2>;
rs_recover_matmul_funcs[3] = &rs_recover_matmul_tmpl<3>;
rs_recover_matmul_funcs[4] = &rs_recover_matmul_tmpl<4>;
rs_recover_matmul_funcs[5] = &rs_recover_matmul_tmpl<5>;
rs_recover_matmul_funcs[6] = &rs_recover_matmul_tmpl<6>;
rs_recover_matmul_funcs[7] = &rs_recover_matmul_tmpl<7>;
rs_recover_matmul_funcs[8] = &rs_recover_matmul_tmpl<8>;
rs_recover_matmul_funcs[9] = &rs_recover_matmul_tmpl<9>;
rs_recover_matmul_funcs[10] = &rs_recover_matmul_tmpl<10>;
rs_recover_matmul_funcs[11] = &rs_recover_matmul_tmpl<11>;
rs_recover_matmul_funcs[12] = &rs_recover_matmul_tmpl<12>;
rs_recover_matmul_funcs[13] = &rs_recover_matmul_tmpl<13>;
rs_recover_matmul_funcs[14] = &rs_recover_matmul_tmpl<14>;
rs_recover_matmul_funcs[15] = &rs_recover_matmul_tmpl<15>;
}
Reference in New Issue View Git Blame Copy Permalink