diff options
Diffstat (limited to 'xlators/cluster/ec/src')
42 files changed, 42675 insertions, 0 deletions
diff --git a/xlators/cluster/ec/src/Makefile.am b/xlators/cluster/ec/src/Makefile.am new file mode 100644 index 00000000000..406a636bbc2 --- /dev/null +++ b/xlators/cluster/ec/src/Makefile.am @@ -0,0 +1,83 @@ +xlator_LTLIBRARIES = ec.la +xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster + +ec_sources := ec.c +ec_sources += ec-data.c +ec_sources += ec-helpers.c +ec_sources += ec-common.c +ec_sources += ec-generic.c +ec_sources += ec-locks.c +ec_sources += ec-dir-read.c +ec_sources += ec-dir-write.c +ec_sources += ec-inode-read.c +ec_sources += ec-inode-write.c +ec_sources += ec-combine.c +ec_sources += ec-method.c +ec_sources += ec-galois.c +ec_sources += ec-code.c +ec_sources += ec-code-c.c +ec_sources += ec-gf8.c +ec_sources += ec-heal.c +ec_sources += ec-heald.c + +ec_headers := ec.h +ec_headers += ec-mem-types.h +ec_headers += ec-helpers.h +ec_headers += ec-data.h +ec_headers += ec-fops.h +ec_headers += ec-common.h +ec_headers += ec-combine.h +ec_headers += ec-method.h +ec_headers += ec-galois.h +ec_headers += ec-code.h +ec_headers += ec-code-c.h +ec_headers += ec-gf8.h +ec_headers += ec-heald.h +ec_headers += ec-messages.h +ec_headers += ec-types.h + +if ENABLE_EC_DYNAMIC_INTEL + ec_sources += ec-code-intel.c + ec_headers += ec-code-intel.h +endif + +if ENABLE_EC_DYNAMIC_X64 + ec_sources += ec-code-x64.c + ec_headers += ec-code-x64.h +endif + +if ENABLE_EC_DYNAMIC_SSE + ec_sources += ec-code-sse.c + ec_headers += ec-code-sse.h +endif + +if ENABLE_EC_DYNAMIC_AVX + ec_sources += ec-code-avx.c + ec_headers += ec-code-avx.h +endif + +ec_ext_sources = $(top_builddir)/xlators/lib/src/libxlator.c + +ec_ext_headers = $(top_builddir)/xlators/lib/src/libxlator.h + +ec_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS) +ec_la_SOURCES = $(ec_sources) $(ec_headers) $(ec_ext_sources) $(ec_ext_headers) +ec_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la + +AM_CPPFLAGS = $(GF_CPPFLAGS) +AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src +AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src +AM_CPPFLAGS += -I$(top_srcdir)/rpc/rpc-lib/src +AM_CPPFLAGS += -I$(top_srcdir)/rpc/xdr/src +AM_CPPFLAGS += -I$(top_builddir)/rpc/xdr/src +AM_CPPFLAGS += -DGLUSTERFS_LIBEXECDIR=\"$(GLUSTERFS_LIBEXECDIR)\" + +AM_CFLAGS = -Wall $(GF_CFLAGS) + +CLEANFILES = + +install-data-hook: + ln -sf ec.so $(DESTDIR)$(xlatordir)/disperse.so + +uninstall-local: + rm -f $(DESTDIR)$(xlatordir)/disperse.so diff --git a/xlators/cluster/ec/src/ec-code-avx.c b/xlators/cluster/ec/src/ec-code-avx.c new file mode 100644 index 00000000000..70afaa00f54 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-avx.c @@ -0,0 +1,109 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <errno.h> + +#include "ec-code-intel.h" + +static void +ec_code_avx_prolog(ec_code_builder_t *builder) +{ + builder->loop = builder->address; +} + +static void +ec_code_avx_epilog(ec_code_builder_t *builder) +{ + ec_code_intel_op_add_i2r(builder, 32, REG_DX); + ec_code_intel_op_add_i2r(builder, 32, REG_DI); + ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX); + ec_code_intel_op_jne(builder, builder->loop); + + ec_code_intel_op_ret(builder, 0); +} + +static void +ec_code_avx_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + if (builder->linear) { + ec_code_intel_op_mov_m2avx( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_AX); + builder->base = idx; + } + ec_code_intel_op_mov_m2avx(builder, REG_AX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static void +ec_code_avx_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit) +{ + ec_code_intel_op_mov_avx2m(builder, src, REG_DI, REG_NULL, 0, + bit * builder->width); +} + +static void +ec_code_avx_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_intel_op_mov_avx2avx(builder, src, dst); +} + +static void +ec_code_avx_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_intel_op_xor_avx2avx(builder, src, dst); +} + +static void +ec_code_avx_xor3(ec_code_builder_t *builder, uint32_t dst, uint32_t src1, + uint32_t src2) +{ + ec_code_intel_op_mov_avx2avx(builder, src1, dst); + ec_code_intel_op_xor_avx2avx(builder, src2, dst); +} + +static void +ec_code_avx_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + if (builder->linear) { + ec_code_intel_op_xor_m2avx( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_AX); + builder->base = idx; + } + ec_code_intel_op_xor_m2avx(builder, REG_AX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static char *ec_code_avx_needed_flags[] = {"avx2", NULL}; + +ec_code_gen_t ec_code_gen_avx = {.name = "avx", + .flags = ec_code_avx_needed_flags, + .width = 32, + .prolog = ec_code_avx_prolog, + .epilog = ec_code_avx_epilog, + .load = ec_code_avx_load, + .store = ec_code_avx_store, + .copy = ec_code_avx_copy, + .xor2 = ec_code_avx_xor2, + .xor3 = ec_code_avx_xor3, + .xorm = ec_code_avx_xorm}; diff --git a/xlators/cluster/ec/src/ec-code-avx.h b/xlators/cluster/ec/src/ec-code-avx.h new file mode 100644 index 00000000000..fdca4ad2c8f --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-avx.h @@ -0,0 +1,18 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_AVX_H__ +#define __EC_CODE_AVX_H__ + +#include "ec-code.h" + +extern ec_code_gen_t ec_code_gen_avx; + +#endif /* __EC_CODE_AVX_H__ */ diff --git a/xlators/cluster/ec/src/ec-code-c.c b/xlators/cluster/ec/src/ec-code-c.c new file mode 100644 index 00000000000..acdc665c2cf --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-c.c @@ -0,0 +1,11679 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <inttypes.h> +#include <string.h> + +#include "ec-method.h" +#include "ec-code-c.h" + +#define WIDTH (EC_METHOD_WORD_SIZE / sizeof(uint64_t)) + +static void +gf8_muladd_00(void *out, void *in) +{ + memcpy(out, in, EC_METHOD_WORD_SIZE * 8); +} + +static void +gf8_muladd_01(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + out_ptr[0] ^= in_ptr[0]; + out_ptr[WIDTH] ^= in_ptr[WIDTH]; + out_ptr[WIDTH * 2] ^= in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] ^= in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] ^= in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] ^= in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] ^= in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] ^= in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_02(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in7; + out1 = in0; + out7 = in6; + out5 = in4; + out6 = in5; + out3 = in2 ^ in7; + out4 = in3 ^ in7; + out2 = in1 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_03(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in0 ^ in7; + tmp0 = in2 ^ in7; + out1 = in0 ^ in1; + out7 = in6 ^ in7; + out5 = in4 ^ in5; + out6 = in5 ^ in6; + out4 = in3 ^ in4 ^ in7; + out2 = tmp0 ^ in1; + out3 = tmp0 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_04(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in6; + out1 = in7; + out7 = in5; + out6 = in4; + tmp0 = in6 ^ in7; + out2 = in0 ^ in6; + out5 = in3 ^ in7; + out3 = tmp0 ^ in1; + out4 = tmp0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_05(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in0 ^ in6; + out1 = in1 ^ in7; + out7 = in5 ^ in7; + out6 = in4 ^ in6; + out2 = out0 ^ in2; + out3 = out1 ^ in3 ^ in6; + out5 = out7 ^ in3; + out4 = out6 ^ in2 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_06(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in6 ^ in7; + tmp0 = in1 ^ in6; + out1 = in0 ^ in7; + out7 = in5 ^ in6; + out6 = in4 ^ in5; + out4 = in2 ^ in3 ^ in6; + out5 = in3 ^ in4 ^ in7; + out3 = tmp0 ^ in2; + out2 = tmp0 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_07(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in6; + tmp1 = in5 ^ in6; + tmp2 = in0 ^ in7; + tmp3 = tmp0 ^ in3; + out6 = tmp1 ^ in4; + out7 = tmp1 ^ in7; + out0 = tmp2 ^ in6; + out1 = tmp2 ^ in1; + out3 = tmp3 ^ in1; + out4 = tmp3 ^ in4; + out5 = out4 ^ out7 ^ in2; + out2 = tmp0 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_08(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in5; + out1 = in6; + out7 = in4; + out6 = in3 ^ in7; + out3 = in0 ^ in5 ^ in6; + out5 = in2 ^ in6 ^ in7; + out2 = in5 ^ in7; + out4 = out2 ^ in1 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_09(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in0 ^ in5; + tmp0 = in3 ^ in6; + out1 = in1 ^ in6; + out7 = in4 ^ in7; + out2 = in2 ^ in5 ^ in7; + out3 = tmp0 ^ out0; + out6 = tmp0 ^ in7; + out4 = out1 ^ out7 ^ in5; + out5 = out2 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in5 ^ in7; + out1 = in0 ^ in6; + out7 = in4 ^ in6; + out2 = in1 ^ in5; + out6 = out0 ^ in3; + out3 = out0 ^ out1 ^ in2; + out5 = out7 ^ in2 ^ in7; + out4 = out2 ^ in3 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = in0 ^ in6; + tmp2 = in4 ^ in7; + out0 = in0 ^ in5 ^ in7; + out2 = tmp0 ^ in1; + out1 = tmp1 ^ in1; + out6 = tmp1 ^ out0 ^ in3; + out7 = tmp2 ^ in6; + out4 = tmp2 ^ out6 ^ in1; + out3 = out6 ^ in0 ^ in2; + out5 = tmp0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in5 ^ in6; + out1 = in6 ^ in7; + out7 = in4 ^ in5; + tmp0 = in1 ^ in5; + tmp1 = in0 ^ in7; + out5 = in2 ^ in3 ^ in6; + out6 = in3 ^ in4 ^ in7; + out2 = tmp1 ^ out0; + out4 = tmp0 ^ in2; + out3 = tmp0 ^ tmp1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in5; + tmp1 = in5 ^ in6; + out1 = in1 ^ in6 ^ in7; + out7 = tmp0 ^ in7; + out4 = tmp0 ^ in1 ^ in2; + out0 = tmp1 ^ in0; + tmp2 = tmp1 ^ in3; + out6 = tmp2 ^ out7; + out2 = out0 ^ in2 ^ in7; + out3 = out0 ^ out1 ^ in3; + out5 = tmp2 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in2 ^ in5; + tmp2 = in5 ^ in6; + out1 = in0 ^ in6 ^ in7; + out3 = tmp0 ^ tmp1; + out2 = tmp0 ^ tmp2; + tmp3 = tmp1 ^ in3; + out7 = tmp2 ^ in4; + out0 = tmp2 ^ in7; + out4 = tmp3 ^ in1 ^ in7; + out5 = tmp3 ^ out7; + out6 = out0 ^ out5 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_0F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in6 ^ in7; + tmp1 = tmp0 ^ in1; + tmp2 = tmp0 ^ in5; + out1 = tmp1 ^ in0; + out7 = tmp2 ^ in4; + out0 = tmp2 ^ in0; + out6 = out7 ^ in3; + out5 = out6 ^ in2 ^ in7; + tmp3 = tmp1 ^ out0 ^ in2; + out4 = tmp1 ^ out5; + out2 = tmp3 ^ in6; + out3 = tmp3 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_10(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in4; + out1 = in5; + out7 = in3 ^ in7; + tmp0 = in6 ^ in7; + out2 = in4 ^ in6; + tmp1 = out2 ^ in5; + out6 = tmp0 ^ in2; + out3 = tmp0 ^ tmp1; + out5 = out2 ^ out3 ^ in1; + out4 = tmp1 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_11(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in3; + out0 = in0 ^ in4; + out1 = in1 ^ in5; + out6 = in2 ^ in7; + out4 = in0 ^ in5 ^ in6; + out5 = in1 ^ in6 ^ in7; + out2 = in2 ^ in4 ^ in6; + out3 = in3 ^ in4 ^ in5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_12(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in4 ^ in7; + out1 = in0 ^ in5; + out3 = in2 ^ in4 ^ in5; + tmp0 = out0 ^ in6; + out2 = tmp0 ^ in1; + tmp1 = tmp0 ^ in3; + out6 = tmp0 ^ out3; + out5 = out2 ^ in5; + out7 = tmp1 ^ in4; + out4 = tmp1 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_13(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in3 ^ in6; + tmp0 = in0 ^ in5; + tmp1 = in4 ^ in7; + out6 = in2 ^ in5 ^ in7; + out4 = tmp0 ^ out7 ^ in7; + out1 = tmp0 ^ in1; + out0 = tmp1 ^ in0; + out5 = tmp1 ^ in1 ^ in6; + out3 = tmp1 ^ out6 ^ in3; + out2 = out5 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_14(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in4 ^ in6; + out1 = in5 ^ in7; + out2 = in0 ^ in4; + tmp0 = out0 ^ in5; + out7 = out1 ^ in3; + tmp1 = out1 ^ in2; + out3 = tmp0 ^ in1; + out6 = tmp0 ^ tmp1; + out4 = tmp1 ^ out2; + out5 = out3 ^ in3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_15(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in3 ^ in5; + tmp0 = in0 ^ in4; + out1 = in1 ^ in5 ^ in7; + out5 = in1 ^ in3 ^ in6; + out0 = tmp0 ^ in6; + out2 = tmp0 ^ in2; + out3 = out5 ^ in4 ^ in5; + out6 = out2 ^ in0 ^ in7; + out4 = tmp0 ^ out6 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_16(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in5; + tmp1 = in4 ^ in7; + tmp2 = in2 ^ in3 ^ in4; + out1 = tmp0 ^ in7; + out4 = tmp0 ^ tmp2; + out0 = tmp1 ^ in6; + tmp3 = tmp1 ^ in1; + out6 = out0 ^ in2 ^ in5; + out2 = tmp3 ^ in0; + out3 = out6 ^ in1; + out7 = tmp2 ^ out6; + out5 = tmp3 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_17(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = in3 ^ in6; + tmp2 = tmp0 ^ in4; + out4 = tmp0 ^ in0 ^ in3; + out7 = tmp1 ^ in5; + tmp3 = tmp1 ^ in1; + out6 = tmp2 ^ in7; + out5 = tmp3 ^ in4; + out3 = tmp3 ^ out6; + out0 = out3 ^ out4 ^ in1; + out2 = out3 ^ out7 ^ in0; + out1 = tmp2 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_18(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in4 ^ in5; + out1 = in5 ^ in6; + tmp0 = in4 ^ in7; + out5 = in1 ^ in2 ^ in5; + out6 = in2 ^ in3 ^ in6; + out2 = tmp0 ^ out1; + out7 = tmp0 ^ in3; + tmp1 = tmp0 ^ in0; + out3 = tmp1 ^ in6; + out4 = tmp1 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_19(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in1 ^ in2; + out7 = in3 ^ in4; + tmp0 = in0 ^ in7; + out6 = in2 ^ in3; + out1 = in1 ^ in5 ^ in6; + out0 = in0 ^ in4 ^ in5; + out4 = tmp0 ^ in1; + tmp1 = tmp0 ^ in6; + out2 = tmp1 ^ out0 ^ in2; + out3 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in5; + tmp1 = in5 ^ in6; + tmp2 = tmp0 ^ in1; + out0 = tmp0 ^ in7; + out1 = tmp1 ^ in0; + tmp3 = tmp1 ^ in3; + out5 = tmp2 ^ in2; + out2 = tmp2 ^ in6; + out7 = tmp3 ^ out0; + out6 = tmp3 ^ in2; + out4 = tmp3 ^ out2 ^ in0; + out3 = tmp0 ^ out1 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in4; + tmp1 = in2 ^ in5; + tmp2 = in3 ^ in6; + out5 = tmp0 ^ in1; + tmp3 = tmp0 ^ in0; + out6 = tmp1 ^ in3; + out0 = tmp1 ^ tmp3 ^ in7; + out7 = tmp2 ^ in4; + tmp4 = out5 ^ in6; + out3 = tmp2 ^ tmp3; + out2 = tmp4 ^ in5; + out4 = tmp4 ^ out3; + out1 = tmp3 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + tmp1 = in4 ^ in6; + tmp2 = in5 ^ in7; + out6 = tmp0 ^ tmp1; + out0 = tmp1 ^ in5; + out1 = tmp2 ^ in6; + tmp3 = tmp2 ^ in1; + tmp4 = tmp2 ^ in4; + out2 = tmp4 ^ in0; + out7 = tmp4 ^ in3; + out5 = tmp0 ^ tmp3; + out3 = tmp3 ^ out2; + out4 = out3 ^ in2 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in3; + tmp1 = in0 ^ in4; + tmp2 = in3 ^ in4; + tmp3 = in2 ^ in7; + out3 = tmp0 ^ tmp1; + out5 = tmp0 ^ tmp3; + tmp4 = tmp1 ^ in5; + out6 = tmp2 ^ in2; + out7 = tmp2 ^ in5; + out2 = tmp3 ^ tmp4; + out4 = out3 ^ out6 ^ in6; + out0 = tmp4 ^ in6; + out1 = out2 ^ out4 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in4; + tmp1 = in2 ^ in7; + tmp2 = tmp0 ^ in1; + out3 = tmp1 ^ tmp2; + out2 = tmp2 ^ in5; + out4 = out3 ^ in3 ^ in6; + tmp3 = out4 ^ in7; + out6 = tmp3 ^ out2 ^ in4; + out7 = tmp1 ^ out6; + out0 = out7 ^ in3; + out1 = tmp0 ^ out0; + out5 = tmp3 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_1F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in6; + tmp1 = tmp0 ^ in5; + out7 = tmp1 ^ in3; + out0 = tmp1 ^ in0 ^ in7; + out6 = out7 ^ in2 ^ in6; + out1 = out0 ^ in1 ^ in4; + out4 = out0 ^ out6 ^ in1; + out3 = tmp0 ^ out4; + out2 = out4 ^ out7 ^ in7; + out5 = out3 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_20(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in4; + out0 = in3 ^ in7; + tmp0 = in3 ^ in4; + tmp1 = in6 ^ in7; + out2 = out0 ^ in5; + out4 = tmp0 ^ in5; + out3 = tmp0 ^ tmp1; + out7 = tmp1 ^ in2; + out6 = tmp1 ^ in1 ^ in5; + out5 = out2 ^ out3 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_21(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in1 ^ in4; + tmp0 = in4 ^ in6; + out4 = in3 ^ in5; + out7 = in2 ^ in6; + out0 = in0 ^ in3 ^ in7; + out6 = in1 ^ in5 ^ in7; + out3 = tmp0 ^ in7; + out5 = tmp0 ^ in0; + out2 = out4 ^ in2 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_22(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in3; + out1 = in0 ^ in4; + out7 = in2 ^ in7; + out4 = in4 ^ in5 ^ in7; + out5 = in0 ^ in5 ^ in6; + out6 = in1 ^ in6 ^ in7; + out3 = in2 ^ in3 ^ in4 ^ in6; + out2 = in1 ^ in3 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_23(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in2; + out0 = in0 ^ in3; + out4 = in5 ^ in7; + out5 = in0 ^ in6; + out6 = in1 ^ in7; + out3 = in2 ^ in4 ^ in6; + out1 = in0 ^ in1 ^ in4; + out2 = out4 ^ out6 ^ in2 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_24(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in4 ^ in7; + tmp0 = in3 ^ in4; + out0 = in3 ^ in6 ^ in7; + out3 = tmp0 ^ in1; + tmp1 = out0 ^ in5; + out6 = tmp1 ^ out3; + out2 = tmp1 ^ in0; + out7 = tmp1 ^ in2 ^ in3; + out5 = out2 ^ in4; + out4 = tmp0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_25(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in4; + tmp0 = in2 ^ in5; + out1 = out3 ^ in7; + out7 = tmp0 ^ in6; + out6 = out1 ^ in5; + out4 = out7 ^ in3 ^ in7; + out2 = out4 ^ in0; + out0 = tmp0 ^ out2; + out5 = out0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_26(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in3 ^ in6; + tmp0 = in4 ^ in7; + out7 = in2 ^ in5 ^ in7; + tmp1 = out0 ^ in0 ^ in5; + out1 = tmp0 ^ in0; + tmp2 = tmp0 ^ in6; + out2 = tmp1 ^ in1; + out5 = tmp1 ^ in7; + out6 = tmp2 ^ in1; + out4 = tmp2 ^ out7; + out3 = out0 ^ out6 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_27(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out7 = in2 ^ in5; + out0 = in0 ^ in3 ^ in6; + out6 = in1 ^ in4 ^ in7; + out4 = out7 ^ in6; + out2 = out0 ^ out7 ^ in1; + out5 = out0 ^ in7; + out1 = out6 ^ in0; + out3 = out6 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_28(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3; + out1 = in4 ^ in6; + out0 = in3 ^ in5 ^ in7; + tmp0 = out1 ^ in7; + tmp1 = out0 ^ in4; + out7 = tmp0 ^ in2; + tmp2 = tmp0 ^ in1; + out3 = tmp1 ^ in0; + out6 = tmp1 ^ tmp2; + out4 = tmp2 ^ in3; + out5 = out3 ^ in2 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_29(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in3; + tmp0 = in1 ^ in3; + tmp1 = in4 ^ in6; + tmp2 = in0 ^ in4 ^ in7; + out6 = tmp0 ^ in5; + out4 = tmp0 ^ in6 ^ in7; + out1 = tmp1 ^ in1; + out7 = tmp1 ^ in2; + out3 = tmp2 ^ in5; + out5 = tmp2 ^ in2; + out0 = out3 ^ in3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in3 ^ in5; + tmp0 = in1 ^ in3; + tmp1 = in0 ^ in4; + out7 = in2 ^ in4 ^ in7; + out3 = tmp1 ^ out0 ^ in2; + out2 = tmp0 ^ in7; + out6 = tmp0 ^ in6; + out1 = tmp1 ^ in6; + out5 = tmp1 ^ out7 ^ in5; + out4 = out1 ^ in0 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in6; + out7 = in2 ^ in4; + tmp0 = in0 ^ in5; + tmp1 = in2 ^ in7; + out6 = in1 ^ in3; + out1 = out4 ^ in0 ^ in4; + out3 = tmp0 ^ out7; + out0 = tmp0 ^ in3; + out5 = tmp1 ^ in0; + out2 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = in2 ^ in3 ^ in4; + tmp2 = tmp0 ^ in6; + out4 = tmp1 ^ in1; + out5 = tmp1 ^ in0 ^ in5; + tmp3 = tmp2 ^ in4; + out6 = tmp2 ^ out4; + out7 = tmp3 ^ in7; + out2 = tmp3 ^ out5; + out3 = out6 ^ in0; + out0 = tmp1 ^ out7; + out1 = tmp0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + out4 = tmp0 ^ in1; + tmp1 = tmp0 ^ in0; + out2 = tmp1 ^ in6; + out5 = tmp1 ^ in4; + tmp2 = out2 ^ in2; + tmp3 = tmp2 ^ in5; + out0 = tmp3 ^ in7; + out7 = tmp3 ^ out5; + out6 = out4 ^ out7 ^ in6; + out3 = tmp2 ^ out6; + out1 = out0 ^ out6 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in7; + out0 = in3 ^ in5 ^ in6; + tmp1 = tmp0 ^ in0; + tmp2 = tmp0 ^ in2; + out1 = tmp1 ^ in6; + out4 = tmp2 ^ in1; + out7 = tmp2 ^ in5; + out3 = out0 ^ out4 ^ in0; + out2 = out3 ^ out7 ^ in7; + out6 = tmp1 ^ out2; + out5 = tmp1 ^ out7 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_2F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in2 ^ in5; + out4 = in1 ^ in2 ^ in7; + out6 = in1 ^ in3 ^ in4; + out5 = tmp0 ^ in2; + tmp2 = tmp0 ^ in6; + out7 = tmp1 ^ in4; + out0 = tmp2 ^ in5; + out2 = tmp2 ^ out4; + out1 = tmp2 ^ out6 ^ in7; + out3 = tmp1 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_30(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in4 ^ in5; + tmp0 = in3 ^ in6; + tmp1 = in4 ^ in7; + out6 = in1 ^ in2 ^ in5; + out3 = tmp0 ^ in5; + out4 = tmp0 ^ in0; + out7 = tmp0 ^ in2; + out0 = tmp1 ^ in3; + out2 = tmp1 ^ out3; + out5 = tmp1 ^ in0 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_31(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in5 ^ in6; + tmp0 = in4 ^ in5; + tmp1 = in0 ^ in3 ^ in4; + tmp2 = out3 ^ in2; + out1 = tmp0 ^ in1; + out0 = tmp1 ^ in7; + out4 = tmp1 ^ in6; + out6 = tmp2 ^ in1; + out2 = tmp2 ^ out0 ^ in0; + out5 = out1 ^ in0 ^ in7; + out7 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_32(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in3 ^ in4; + out7 = in2 ^ in3; + tmp0 = in5 ^ in6; + tmp1 = in0 ^ in7; + out6 = in1 ^ in2; + out1 = in0 ^ in4 ^ in5; + out2 = tmp0 ^ out0 ^ in1; + out3 = tmp0 ^ out7 ^ in7; + out4 = tmp1 ^ in6; + out5 = tmp1 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_33(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + tmp1 = in0 ^ in4; + tmp2 = in1 ^ in5; + out6 = in1 ^ in2 ^ in6; + out7 = tmp0 ^ in7; + out0 = tmp1 ^ in3; + out1 = tmp1 ^ tmp2; + tmp3 = tmp2 ^ in7; + tmp4 = tmp2 ^ in4 ^ in6; + out5 = tmp3 ^ in0; + out3 = tmp3 ^ out6; + out4 = tmp4 ^ out5; + out2 = tmp0 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_34(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in4 ^ in5; + tmp2 = tmp0 ^ in1; + tmp3 = tmp0 ^ in6; + out1 = tmp1 ^ in7; + tmp4 = tmp1 ^ in2; + out5 = tmp2 ^ in0; + out3 = tmp2 ^ out1; + out0 = tmp3 ^ in7; + out7 = tmp3 ^ tmp4; + out6 = tmp4 ^ in1; + out2 = out3 ^ out5 ^ in3; + out4 = tmp4 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_35(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in6; + tmp1 = in5 ^ in7; + out7 = tmp0 ^ tmp1 ^ in3; + out3 = tmp1 ^ in1; + out1 = out3 ^ in4; + tmp2 = out1 ^ in7; + out5 = tmp2 ^ in0 ^ in3; + out6 = tmp0 ^ tmp2; + out0 = out3 ^ out5 ^ in6; + out4 = tmp0 ^ out0; + out2 = out4 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_36(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in2; + tmp0 = in1 ^ in3; + out0 = in3 ^ in4 ^ in6; + out6 = in1 ^ in2 ^ in4; + out5 = tmp0 ^ in0; + tmp1 = out5 ^ in5; + out2 = tmp1 ^ in4; + out3 = tmp1 ^ out4; + out1 = tmp0 ^ out2 ^ in7; + out7 = out3 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_37(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in2 ^ in4; + tmp2 = tmp0 ^ in6; + out3 = tmp0 ^ in5; + out4 = tmp1 ^ in0; + out6 = tmp2 ^ in4; + out1 = out3 ^ out4 ^ in7; + tmp3 = out4 ^ in1 ^ in3; + out7 = tmp3 ^ out1; + out2 = tmp3 ^ in5; + out5 = tmp1 ^ out2; + out0 = tmp2 ^ tmp3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_38(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in3; + tmp0 = in3 ^ in4; + tmp1 = in5 ^ in7; + tmp2 = out3 ^ in1; + out2 = tmp0 ^ in6; + out0 = tmp0 ^ tmp1; + out4 = tmp1 ^ tmp2; + out7 = out2 ^ in2; + out1 = out2 ^ in3 ^ in5; + out6 = out4 ^ in0 ^ in2; + out5 = tmp2 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_39(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0; + tmp0 = in1 ^ in5; + tmp1 = tmp0 ^ in4; + out1 = tmp1 ^ in6; + out5 = out1 ^ in0 ^ in2; + tmp2 = tmp0 ^ out5; + out2 = tmp2 ^ in0 ^ in3; + out7 = out2 ^ in7; + out6 = tmp1 ^ out7; + out4 = tmp2 ^ out6; + out0 = out4 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in0 ^ in2; + tmp2 = in3 ^ in4; + tmp3 = in1 ^ in6; + tmp4 = in3 ^ in7; + out4 = tmp0 ^ in5; + out5 = tmp1 ^ tmp3; + out3 = tmp1 ^ tmp4; + out0 = tmp2 ^ in5; + out7 = tmp2 ^ in2; + tmp5 = tmp3 ^ in4; + out2 = tmp4 ^ tmp5; + out1 = tmp5 ^ out4; + out6 = tmp0 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in6; + tmp1 = in2 ^ in7; + tmp2 = tmp0 ^ in3; + out3 = tmp1 ^ in0; + out6 = tmp1 ^ tmp2; + out2 = out6 ^ in4; + out7 = tmp0 ^ out2; + out0 = out3 ^ out7 ^ in5; + out5 = out0 ^ out2 ^ in7; + out1 = tmp2 ^ out0; + out4 = out1 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in2 ^ in7; + tmp2 = in1 ^ in6 ^ in7; + out2 = tmp0 ^ in4; + out3 = tmp0 ^ tmp2; + out4 = tmp1 ^ out3 ^ in5; + out5 = tmp2 ^ out2 ^ in2; + out1 = out4 ^ out5 ^ in6; + out0 = out1 ^ in3; + out7 = tmp1 ^ out0; + out6 = tmp2 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = tmp0 ^ in3; + out2 = tmp1 ^ in4; + tmp2 = out2 ^ in5; + out4 = tmp2 ^ in1 ^ in6; + out5 = out4 ^ in7; + out6 = out5 ^ in0; + out7 = out6 ^ in1; + out0 = tmp0 ^ out7; + out1 = tmp1 ^ out5; + out3 = tmp2 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5; + tmp1 = tmp0 ^ in4; + out0 = tmp1 ^ in6; + out7 = tmp1 ^ in2; + out6 = out7 ^ in1 ^ in5 ^ in7; + out2 = out6 ^ in0 ^ in2; + out4 = out0 ^ out6 ^ in0; + out5 = tmp0 ^ out4; + out3 = out5 ^ in7; + out1 = out3 ^ out6 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_3F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + out3 = tmp0 ^ in2 ^ in6; + tmp1 = out3 ^ in5 ^ in7; + out4 = tmp1 ^ in4; + out5 = tmp1 ^ in3; + out1 = out4 ^ in2; + out7 = out1 ^ out3 ^ in3; + out2 = tmp0 ^ out7 ^ in5; + tmp2 = out2 ^ in0; + out6 = tmp2 ^ in6; + out0 = tmp1 ^ tmp2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_40(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in3 ^ in7; + tmp0 = in3 ^ in4; + tmp1 = in6 ^ in7; + out4 = tmp0 ^ in2; + out5 = tmp0 ^ in5; + out0 = tmp1 ^ in2; + out7 = tmp1 ^ in1 ^ in5; + out2 = out0 ^ in4; + out3 = out2 ^ out5 ^ in7; + out6 = out3 ^ out4 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_41(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2 ^ in3; + tmp0 = in5 ^ in6; + tmp1 = in6 ^ in7; + out5 = in3 ^ in4; + out1 = in1 ^ in3 ^ in7; + out6 = in0 ^ in4 ^ in5; + out3 = tmp0 ^ in2; + out7 = tmp0 ^ in1; + out2 = tmp1 ^ in4; + out0 = tmp1 ^ in0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_42(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2 ^ in6; + out5 = in3 ^ in5; + out1 = in0 ^ in3 ^ in7; + out7 = in1 ^ in5 ^ in7; + out4 = in2 ^ in4 ^ in7; + out6 = in0 ^ in4 ^ in6; + out2 = out0 ^ in1 ^ in4; + out3 = out5 ^ in6 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_43(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in3; + out7 = in1 ^ in5; + out4 = in2 ^ in7; + out6 = in0 ^ in4; + out0 = in0 ^ in2 ^ in6; + out3 = in5 ^ in6 ^ in7; + out2 = in1 ^ in4 ^ in6; + out1 = in0 ^ in1 ^ in3 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_44(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in3; + out0 = in2 ^ in7; + tmp0 = in4 ^ in7; + out7 = in1 ^ in6 ^ in7; + out6 = in0 ^ in5 ^ in6; + out4 = tmp0 ^ in3 ^ in6; + out3 = out0 ^ in1 ^ in3 ^ in5; + out2 = out0 ^ in0 ^ in4; + out5 = tmp0 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_45(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in1 ^ in3; + out7 = in1 ^ in6; + out5 = in4 ^ in7; + out6 = in0 ^ in5; + out0 = in0 ^ in2 ^ in7; + out4 = in3 ^ in6 ^ in7; + out2 = out5 ^ in0; + out3 = out0 ^ out6 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_46(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2; + out1 = in0 ^ in3; + out7 = in1 ^ in7; + out4 = in4 ^ in6; + out5 = in5 ^ in7; + out6 = in0 ^ in6; + out3 = in1 ^ in3 ^ in5; + out2 = out4 ^ out6 ^ in1 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_47(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in6; + out7 = in1; + out5 = in7; + out6 = in0; + tmp0 = in0 ^ in1; + out3 = in1 ^ in5; + out0 = in0 ^ in2; + out1 = tmp0 ^ in3; + out2 = tmp0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_48(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + out1 = in3 ^ in6 ^ in7; + out3 = tmp0 ^ in0; + out0 = tmp0 ^ out1 ^ in5; + tmp1 = out0 ^ in4; + out2 = tmp1 ^ in7; + out5 = tmp1 ^ in3; + out4 = out5 ^ in1; + out7 = tmp0 ^ out4; + out6 = tmp1 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_49(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in2; + tmp0 = in2 ^ in5; + out2 = in4 ^ in5 ^ in6; + tmp1 = tmp0 ^ out2 ^ in3; + out7 = out2 ^ in1; + out5 = tmp1 ^ in7; + out4 = out5 ^ out7 ^ in6; + out1 = tmp0 ^ out4; + out6 = out1 ^ out7 ^ in0; + out0 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in6; + tmp1 = in3 ^ in7; + out0 = tmp0 ^ in5; + out3 = tmp1 ^ in0; + out5 = tmp1 ^ out0; + out4 = out0 ^ in1 ^ in4; + out1 = out3 ^ in6; + out2 = out4 ^ in7; + out6 = out1 ^ in4; + out7 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in7; + tmp0 = in1 ^ in5; + tmp1 = in2 ^ in6; + tmp2 = out3 ^ in3; + out7 = tmp0 ^ in4; + out4 = tmp0 ^ tmp1; + tmp3 = tmp1 ^ in0; + out6 = tmp2 ^ in4; + out5 = tmp2 ^ tmp3; + out1 = tmp2 ^ in1 ^ in6; + out2 = out7 ^ in6 ^ in7; + out0 = tmp3 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in3 ^ in6; + tmp0 = in2 ^ in5; + tmp1 = out1 ^ in5 ^ in7; + out0 = tmp0 ^ in7; + tmp2 = tmp0 ^ in4; + out6 = tmp1 ^ in0; + out2 = tmp2 ^ in0; + out5 = tmp2 ^ in6; + out3 = tmp0 ^ out6 ^ in1; + out7 = out0 ^ out5 ^ in1; + out4 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in6; + out4 = in1 ^ in3 ^ in5; + tmp2 = tmp0 ^ in7; + out2 = tmp0 ^ in4; + out1 = tmp1 ^ in3; + out7 = tmp1 ^ in4; + out0 = tmp2 ^ in2; + out6 = tmp2 ^ in3; + out5 = out7 ^ in1 ^ in2; + out3 = tmp1 ^ out0 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2 ^ in5; + out7 = in1 ^ in4 ^ in7; + out1 = in0 ^ in3 ^ in6; + out5 = out0 ^ in6; + out4 = out7 ^ in5; + out3 = out1 ^ in1; + out6 = out1 ^ in7; + out2 = out4 ^ in0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_4F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in2 ^ in6; + out7 = in1 ^ in4; + out3 = in0 ^ in1 ^ in6; + out4 = in1 ^ in5 ^ in7; + out0 = in0 ^ in2 ^ in5; + out6 = in0 ^ in3 ^ in7; + out1 = out3 ^ in3; + out2 = out4 ^ in0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_50(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in7; + tmp0 = in3 ^ in5; + out0 = out2 ^ in4 ^ in6; + out1 = tmp0 ^ in7; + tmp1 = tmp0 ^ in6; + out3 = out0 ^ in3; + out7 = tmp1 ^ in1; + tmp2 = tmp1 ^ in0; + out5 = out3 ^ in1 ^ in2; + out4 = tmp2 ^ in2; + out6 = tmp2 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_51(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in7; + out3 = in2 ^ in4 ^ in6 ^ in7; + out0 = out3 ^ in0; + out6 = out0 ^ in5; + out4 = out6 ^ in3 ^ in7; + out1 = out0 ^ out4 ^ in1; + out7 = out1 ^ in6; + out5 = out7 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_52(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1 ^ in2; + tmp0 = in2 ^ in4; + tmp1 = in3 ^ in5; + tmp2 = in3 ^ in6; + tmp3 = in0 ^ in7; + out0 = tmp0 ^ in6; + out6 = tmp0 ^ tmp3; + out7 = tmp1 ^ in1; + out1 = tmp1 ^ tmp3; + out3 = tmp2 ^ in4; + out5 = tmp2 ^ in1 ^ in7; + out4 = tmp2 ^ out1 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_53(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1; + out3 = in4 ^ in6; + out0 = out3 ^ in0 ^ in2; + out6 = out0 ^ in7; + out4 = out6 ^ in5; + out7 = out0 ^ out4 ^ in1 ^ in3; + out1 = out7 ^ in0; + out5 = out7 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_54(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in3 ^ in5; + tmp0 = in1 ^ in3; + tmp1 = in2 ^ in4; + tmp2 = in0 ^ in7; + out5 = in1 ^ in4 ^ in6; + out4 = tmp2 ^ out1; + out7 = tmp0 ^ in6; + out3 = tmp0 ^ tmp1; + out0 = tmp1 ^ in7; + tmp3 = tmp2 ^ in2; + out2 = tmp3 ^ in6; + out6 = tmp3 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_55(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in3; + tmp1 = in1 ^ in4; + tmp2 = in6 ^ in7; + out7 = tmp0 ^ tmp2; + out1 = tmp0 ^ in5; + out3 = tmp1 ^ in2; + out5 = tmp1 ^ in5 ^ in6; + out2 = tmp2 ^ in0; + out4 = out5 ^ out7 ^ in0; + out6 = out2 ^ in2 ^ in5; + out0 = out5 ^ out6 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_56(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2 ^ in4; + tmp0 = in0 ^ in2; + out4 = in0 ^ in5; + out7 = in1 ^ in3; + out5 = in1 ^ in6; + out6 = tmp0 ^ in7; + out2 = tmp0 ^ out5; + out1 = out4 ^ in3; + out3 = out7 ^ in4 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_57(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in7; + out0 = in0 ^ in2 ^ in4; + out5 = in1 ^ in5 ^ in6; + out4 = tmp0 ^ in4; + out1 = tmp0 ^ in1 ^ in3; + out2 = tmp0 ^ out5; + out3 = tmp1 ^ in4; + out7 = tmp1 ^ in3; + out6 = tmp1 ^ out2 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_58(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in5; + tmp0 = in2 ^ in3 ^ in4; + out5 = tmp0 ^ in1; + out6 = tmp0 ^ in0 ^ in5; + out3 = out6 ^ in7; + tmp1 = out2 ^ out5; + out7 = tmp1 ^ in6; + out4 = tmp1 ^ out3 ^ in3; + out0 = out4 ^ out7 ^ in0; + out1 = tmp0 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_59(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in5; + tmp0 = in0 ^ in5 ^ in7; + out3 = tmp0 ^ in2 ^ in4; + out0 = out3 ^ in6; + tmp1 = out0 ^ in7; + out6 = tmp1 ^ in3; + out5 = out6 ^ in0 ^ in1 ^ in6; + out4 = tmp0 ^ out5; + out1 = tmp1 ^ out4; + out7 = out1 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in2 ^ in5; + out5 = tmp0 ^ in3; + out4 = tmp0 ^ in0; + tmp2 = tmp1 ^ in4; + out2 = tmp1 ^ in1 ^ in7; + out7 = tmp2 ^ out5; + out6 = out4 ^ out7 ^ in5; + out0 = tmp2 ^ in6; + out1 = out0 ^ out6 ^ in7; + out3 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + tmp1 = in0 ^ in4; + tmp2 = in1 ^ in5; + out5 = tmp0 ^ tmp2; + tmp3 = tmp1 ^ in6; + out3 = tmp1 ^ in5; + out2 = tmp2 ^ in7; + tmp4 = out3 ^ in2; + out7 = out2 ^ in3 ^ in4; + out0 = tmp4 ^ in6; + out6 = tmp0 ^ tmp3; + out4 = tmp2 ^ tmp4; + out1 = tmp3 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in6; + tmp1 = in0 ^ in2 ^ in5; + out1 = tmp0 ^ in5; + tmp2 = tmp0 ^ in1; + out2 = tmp1 ^ in6; + out6 = tmp1 ^ in3; + out4 = tmp2 ^ in0; + out7 = tmp2 ^ in4; + out3 = tmp1 ^ out7; + out0 = out3 ^ out4 ^ in7; + out5 = out0 ^ in1 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in0 ^ in6; + out2 = tmp1 ^ in5; + tmp2 = out2 ^ in3; + out6 = tmp2 ^ in2; + out1 = tmp0 ^ tmp2; + tmp3 = out1 ^ in4 ^ in5; + out4 = tmp3 ^ in0; + out7 = tmp3 ^ in7; + tmp4 = out4 ^ out6; + out5 = tmp4 ^ in7; + out0 = tmp0 ^ out5; + out3 = tmp1 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = in3 ^ in5; + tmp2 = in1 ^ in7; + out7 = in1 ^ in3 ^ in4; + out0 = tmp0 ^ in4; + tmp3 = tmp1 ^ in0; + out5 = tmp2 ^ in2; + out1 = tmp3 ^ in6; + out6 = tmp0 ^ tmp3; + tmp4 = tmp2 ^ out1; + out3 = tmp4 ^ in4; + out4 = tmp1 ^ tmp4; + out2 = tmp0 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_5F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in5; + tmp1 = in0 ^ in6; + tmp2 = tmp0 ^ in7; + tmp3 = tmp1 ^ in3; + out2 = tmp1 ^ tmp2; + out5 = tmp2 ^ in2; + out6 = tmp3 ^ in2; + out3 = out2 ^ in4; + out4 = out3 ^ in5; + out1 = tmp0 ^ tmp3; + out7 = tmp3 ^ out4; + out0 = out4 ^ out5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_60(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2 ^ in5; + tmp0 = in3 ^ in6; + out1 = in3 ^ in4 ^ in7; + out7 = out4 ^ in1; + tmp1 = out4 ^ in4; + out0 = tmp0 ^ in2; + out5 = tmp0 ^ in0; + out2 = tmp0 ^ tmp1; + out3 = tmp1 ^ in7; + out6 = out3 ^ out7 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_61(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + out4 = tmp0 ^ in4; + tmp1 = out4 ^ in3; + out3 = tmp1 ^ in7; + out2 = tmp1 ^ in2 ^ in6; + out1 = tmp0 ^ out3 ^ in1; + out0 = out2 ^ out4 ^ in0; + out7 = tmp1 ^ out1; + out6 = out0 ^ out1 ^ in2; + out5 = tmp0 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_62(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in4 ^ in5; + tmp0 = in0 ^ in3 ^ in4; + out1 = tmp0 ^ in7; + out5 = tmp0 ^ in6; + tmp1 = out1 ^ in0; + tmp2 = tmp1 ^ out3; + out4 = tmp2 ^ in2; + tmp3 = tmp2 ^ in1; + out0 = out4 ^ in5 ^ in6; + out7 = tmp3 ^ out0; + out6 = tmp0 ^ tmp3; + out2 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_63(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in1 ^ in7; + out3 = tmp0 ^ in5; + tmp2 = out3 ^ in6; + out4 = out3 ^ in2 ^ in7; + out5 = tmp2 ^ in0; + tmp3 = out5 ^ in3; + out0 = tmp3 ^ out4; + out2 = tmp1 ^ tmp2; + out6 = tmp1 ^ tmp3; + tmp4 = tmp0 ^ out2; + out1 = tmp4 ^ out5; + out7 = tmp4 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_64(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in2 ^ in3; + out1 = in3 ^ in4; + out7 = in1 ^ in2; + tmp0 = in4 ^ in5; + tmp1 = in0 ^ in7; + out4 = in5 ^ in6 ^ in7; + out2 = tmp0 ^ out0 ^ in0; + out3 = tmp0 ^ out7 ^ in6; + out5 = tmp1 ^ in6; + out6 = tmp1 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_65(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in4 ^ in5; + tmp2 = in6 ^ in7; + out7 = in1 ^ in2 ^ in7; + out1 = in1 ^ in3 ^ in4; + out0 = tmp0 ^ in2; + out2 = tmp0 ^ tmp1; + out4 = tmp1 ^ tmp2; + tmp3 = tmp2 ^ in0; + out3 = out4 ^ out7 ^ in3; + out5 = tmp3 ^ in5; + out6 = tmp3 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_66(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in2 ^ in3; + tmp2 = in0 ^ in4; + out7 = tmp0 ^ in6; + out0 = tmp1 ^ in7; + out1 = tmp2 ^ in3; + tmp3 = tmp2 ^ in6; + tmp4 = out1 ^ in5; + out5 = tmp3 ^ in7; + out4 = tmp3 ^ tmp4; + out2 = tmp0 ^ tmp4 ^ in7; + out6 = tmp1 ^ out2 ^ in4; + out3 = tmp3 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_67(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = tmp0 ^ in1; + tmp2 = tmp0 ^ in7; + out1 = tmp1 ^ in4; + out0 = tmp2 ^ in2; + tmp3 = out1 ^ in7; + out2 = tmp3 ^ in5; + out3 = out2 ^ in0 ^ in6; + out7 = tmp1 ^ out0 ^ in6; + out5 = tmp1 ^ out3; + out4 = tmp2 ^ out5; + out6 = tmp3 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_68(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in2 ^ in3 ^ in5; + tmp2 = tmp0 ^ in1; + tmp3 = tmp0 ^ in6; + out0 = tmp1 ^ in6; + out6 = tmp2 ^ in0; + out7 = tmp1 ^ tmp2; + out1 = tmp3 ^ in7; + out2 = out1 ^ in2; + out4 = tmp2 ^ out2; + out3 = out4 ^ out6 ^ in3; + out5 = tmp3 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_69(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in6 ^ in7; + out2 = tmp0 ^ in3 ^ in4; + out1 = out2 ^ in1; + out3 = out2 ^ in0 ^ in2; + out4 = out1 ^ in2 ^ in3; + out6 = out1 ^ in0 ^ in7; + out7 = out4 ^ in5 ^ in6; + out5 = out4 ^ out6 ^ in5; + out0 = tmp0 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in6; + out3 = in0 ^ in4 ^ in6; + tmp1 = tmp0 ^ in3; + out4 = tmp1 ^ in1; + tmp2 = tmp1 ^ in7; + out2 = out4 ^ in4; + out0 = tmp2 ^ in5; + out5 = tmp2 ^ out3; + out7 = out2 ^ in3 ^ in5; + out1 = tmp0 ^ out5; + out6 = tmp1 ^ out7 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in6; + out2 = tmp0 ^ in1 ^ in3; + out4 = out2 ^ in2; + tmp1 = out2 ^ in0; + out7 = out4 ^ in3 ^ in5 ^ in7; + out1 = tmp1 ^ in7; + out3 = tmp1 ^ in1; + out6 = tmp1 ^ in5; + out0 = tmp1 ^ out7 ^ in6; + out5 = tmp0 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1; + tmp0 = in2 ^ in3; + out5 = in0 ^ in2; + out1 = in3 ^ in4 ^ in6; + tmp1 = out5 ^ in1; + out0 = tmp0 ^ in5; + out6 = tmp0 ^ tmp1; + out3 = tmp1 ^ in4; + out7 = out3 ^ in0; + out2 = out6 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in4; + tmp0 = in0 ^ in2; + tmp1 = out4 ^ in3; + out7 = out4 ^ in2 ^ in7; + out5 = tmp0 ^ in5; + out3 = tmp0 ^ tmp1; + out1 = tmp1 ^ in6; + out0 = out5 ^ in3; + out2 = out3 ^ out7 ^ in4; + out6 = out1 ^ in0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in3; + tmp1 = in0 ^ in4; + out4 = tmp0 ^ in7; + out6 = tmp0 ^ in0 ^ in5; + out5 = tmp1 ^ in2; + tmp2 = tmp1 ^ in3; + out3 = tmp2 ^ out4; + out1 = tmp2 ^ in6; + out2 = tmp0 ^ out5; + out0 = out2 ^ out3 ^ in5; + out7 = out1 ^ out2 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_6F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in7; + tmp1 = tmp0 ^ in4; + tmp2 = tmp0 ^ in0 ^ in2; + out4 = tmp1 ^ in1; + out0 = tmp2 ^ in5; + out3 = out4 ^ in0; + out2 = out3 ^ in7; + out1 = out2 ^ in6; + out6 = out1 ^ in4 ^ in5; + out7 = tmp2 ^ out1; + out5 = tmp1 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_70(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2; + tmp0 = in2 ^ in4; + out2 = in2 ^ in3 ^ in5; + tmp1 = tmp0 ^ in6; + tmp2 = out2 ^ in7; + out0 = tmp1 ^ in3; + out4 = tmp1 ^ in0; + out7 = tmp2 ^ in1; + out6 = out4 ^ in1; + out5 = out7 ^ in0 ^ in2; + out1 = tmp0 ^ tmp2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_71(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3 ^ in5; + out3 = in2 ^ in3; + tmp0 = in0 ^ in2; + tmp1 = out2 ^ in1; + out4 = tmp0 ^ in6; + tmp2 = tmp0 ^ in1; + out7 = tmp1 ^ in2; + out1 = tmp1 ^ in4 ^ in7; + out0 = out4 ^ in3 ^ in4; + out6 = tmp2 ^ in4; + out5 = tmp2 ^ out3 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_72(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in7; + tmp0 = in0 ^ in4; + tmp1 = tmp0 ^ in3 ^ in7; + out1 = tmp1 ^ in5; + out5 = out1 ^ in1; + tmp2 = tmp0 ^ out5; + out2 = tmp2 ^ in2; + out7 = out2 ^ in6; + out6 = tmp1 ^ out7; + out4 = tmp2 ^ out6; + out0 = out4 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_73(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in3 ^ in7; + out2 = out3 ^ in1 ^ in5; + out1 = out2 ^ in0 ^ in4; + out5 = out1 ^ in5; + out6 = out1 ^ out3 ^ in2; + out0 = out2 ^ out6 ^ in6; + out7 = out0 ^ out1 ^ in3; + out4 = out0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_74(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in1 ^ in2 ^ in6; + out4 = in0 ^ in4 ^ in7; + out5 = in0 ^ in1 ^ in5; + out0 = tmp0 ^ in2; + out1 = tmp0 ^ in5; + out3 = tmp1 ^ in7; + out6 = tmp1 ^ in0; + out2 = tmp1 ^ out5 ^ in3; + out7 = out3 ^ in3 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_75(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in7; + tmp0 = in1 ^ in3; + out5 = in0 ^ in1; + out7 = tmp0 ^ in2; + tmp1 = tmp0 ^ in4; + out6 = out5 ^ in2; + tmp2 = out7 ^ in6; + out1 = tmp1 ^ in5; + out0 = tmp1 ^ out6; + out3 = tmp2 ^ in7; + out2 = tmp2 ^ out6 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_76(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in6; + tmp0 = in0 ^ in5; + tmp1 = in3 ^ in7; + tmp2 = tmp0 ^ in4; + tmp3 = tmp1 ^ in2; + out5 = tmp2 ^ in1; + out1 = tmp2 ^ in3; + out0 = tmp3 ^ in4; + out4 = out1 ^ in5; + out7 = tmp3 ^ out3; + out2 = tmp0 ^ out7; + out6 = tmp1 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_77(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in3; + tmp0 = in1 ^ in4; + tmp1 = in1 ^ in6; + tmp2 = out4 ^ in5; + out5 = tmp0 ^ in0; + out1 = tmp0 ^ tmp2; + out3 = tmp1 ^ in3; + out2 = tmp1 ^ tmp2 ^ in7; + out7 = out3 ^ in2; + tmp3 = out7 ^ in6; + out6 = tmp2 ^ tmp3; + out0 = tmp3 ^ out5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_78(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in2 ^ in7; + tmp2 = in0 ^ in5 ^ in6; + out2 = tmp1 ^ in3; + out3 = tmp2 ^ in2; + out5 = out3 ^ in1 ^ in3; + out0 = tmp0 ^ out3 ^ in4; + out1 = tmp1 ^ out0; + out4 = out1 ^ out5 ^ in5; + out7 = tmp0 ^ out4; + out6 = tmp2 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_79(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3 ^ in7; + tmp0 = in3 ^ in4; + tmp1 = in1 ^ in5; + tmp2 = tmp1 ^ in2; + out4 = tmp2 ^ in0 ^ in7; + tmp3 = out4 ^ in5; + out5 = tmp3 ^ out2 ^ in6; + out7 = tmp0 ^ tmp2; + out6 = tmp0 ^ tmp3; + out3 = tmp1 ^ out5; + out0 = out3 ^ in4; + out1 = tmp3 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + out2 = tmp0 ^ in3; + tmp1 = out2 ^ in4; + out4 = tmp1 ^ in0 ^ in5; + out5 = out4 ^ in6; + out6 = out5 ^ in7; + out7 = out6 ^ in0; + out0 = out7 ^ in1; + out1 = tmp0 ^ out6; + out3 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1 ^ in3; + tmp0 = in0 ^ in5; + out4 = tmp0 ^ out2 ^ in2; + tmp1 = out4 ^ in4; + out6 = tmp1 ^ in7; + out5 = tmp1 ^ in5 ^ in6; + out0 = out6 ^ in1 ^ in6; + tmp2 = out0 ^ in2; + out1 = tmp2 ^ in1; + out3 = tmp2 ^ in4; + out7 = tmp0 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5; + tmp1 = tmp0 ^ in4; + out0 = tmp1 ^ in2; + out1 = tmp1 ^ in6; + out7 = out0 ^ in1 ^ in5 ^ in7; + out5 = out1 ^ out7 ^ in0; + out3 = out5 ^ in6; + out6 = tmp0 ^ out5; + out2 = out6 ^ in1; + out4 = out2 ^ out7 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = tmp0 ^ in3; + tmp2 = tmp0 ^ in6; + out7 = tmp1 ^ in4; + tmp3 = tmp2 ^ in0; + out5 = tmp3 ^ in7; + out4 = tmp3 ^ in2 ^ in5; + out2 = tmp1 ^ out5; + out6 = tmp2 ^ out2; + out0 = out4 ^ out7 ^ in6; + out1 = tmp3 ^ out0; + out3 = out6 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in4; + tmp1 = in0 ^ in5; + out1 = tmp0 ^ tmp1 ^ in6; + out3 = tmp1 ^ in1; + out4 = out1 ^ in1 ^ in7; + tmp2 = out4 ^ in3; + out5 = tmp2 ^ in2; + out6 = tmp0 ^ out5; + out7 = tmp1 ^ out4 ^ in2; + out2 = out6 ^ in5 ^ in7; + out0 = tmp2 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_7F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in7; + tmp1 = tmp0 ^ in3 ^ in5; + tmp2 = tmp1 ^ in0; + out0 = tmp2 ^ in4; + out6 = tmp2 ^ in1; + out3 = tmp0 ^ out6; + tmp3 = out3 ^ in6; + out1 = tmp3 ^ in4; + out2 = tmp3 ^ in5; + out4 = tmp3 ^ in7; + out5 = tmp1 ^ out1; + out7 = out0 ^ out4 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_80(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + tmp1 = in4 ^ in5; + out1 = in2 ^ in6 ^ in7; + out5 = tmp0 ^ in4; + tmp2 = tmp0 ^ in1; + out6 = tmp1 ^ in3; + out7 = tmp1 ^ in0 ^ in6; + out4 = tmp2 ^ in7; + out3 = tmp2 ^ out6; + out2 = out3 ^ out5 ^ in6; + out0 = out2 ^ in3 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_81(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in6; + tmp1 = tmp0 ^ in3; + out6 = tmp1 ^ in5; + out5 = out6 ^ in2 ^ in6; + out3 = out5 ^ in1; + out2 = tmp0 ^ out3; + out1 = out3 ^ out6 ^ in7; + out4 = tmp1 ^ out1; + out7 = out2 ^ out4 ^ in0; + out0 = out7 ^ in1 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_82(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in2; + tmp0 = in6 ^ in7; + out5 = in2 ^ in3; + out6 = in3 ^ in4; + out7 = in0 ^ in4 ^ in5; + out0 = in1 ^ in5 ^ in6; + out1 = tmp0 ^ in0 ^ in2; + out2 = tmp0 ^ in3 ^ in5; + out3 = tmp0 ^ out0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_83(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in2 ^ in5; + tmp2 = in3 ^ in6; + out4 = in1 ^ in2 ^ in4; + out0 = tmp0 ^ in5 ^ in6; + out5 = tmp1 ^ in3; + tmp3 = tmp1 ^ in7; + out6 = tmp2 ^ in4; + out2 = tmp2 ^ tmp3; + tmp4 = tmp3 ^ out4; + out1 = tmp3 ^ out0; + out3 = tmp4 ^ in3; + out7 = tmp0 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_84(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in2 ^ in6; + out6 = in3 ^ in5; + out0 = in1 ^ in5 ^ in7; + out7 = in0 ^ in4 ^ in6; + out4 = in1 ^ in3 ^ in6; + out5 = in2 ^ in4 ^ in7; + out2 = out6 ^ in0 ^ in1; + out3 = out5 ^ in5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_85(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in6; + tmp1 = in3 ^ in6; + tmp2 = tmp0 ^ in4; + out1 = tmp0 ^ in2; + out6 = tmp1 ^ in5; + out4 = tmp2 ^ in3; + tmp3 = out1 ^ out6; + out2 = tmp3 ^ in0; + out3 = tmp2 ^ tmp3 ^ in7; + out7 = out2 ^ out3 ^ in1; + out5 = tmp1 ^ out3; + out0 = tmp2 ^ out7 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_86(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out6 = in3; + out7 = in0 ^ in4; + out0 = in1 ^ in5; + out5 = in2 ^ in7; + out3 = in4 ^ in5 ^ in6; + out1 = in0 ^ in2 ^ in6; + out4 = in1 ^ in6 ^ in7; + out2 = in0 ^ in3 ^ in5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_87(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out6 = in3 ^ in6; + tmp0 = in0 ^ in1; + out7 = in0 ^ in4 ^ in7; + out5 = in2 ^ in5 ^ in7; + out3 = out6 ^ in4 ^ in5; + out0 = tmp0 ^ in5; + tmp1 = tmp0 ^ in6; + out2 = out5 ^ in0 ^ in3; + out1 = tmp1 ^ in2; + out4 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_88(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in2 ^ in7; + tmp0 = in5 ^ in6; + out0 = in1 ^ in6 ^ in7; + out6 = in4 ^ in5 ^ in7; + out3 = out0 ^ out1 ^ in0 ^ in4; + out7 = tmp0 ^ in0; + tmp1 = tmp0 ^ in3; + out2 = out0 ^ in3; + out4 = tmp1 ^ in2; + out5 = tmp1 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_89(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in7; + tmp1 = in2 ^ in7; + tmp2 = tmp0 ^ in6; + out1 = tmp1 ^ in1; + out7 = tmp2 ^ in5; + out0 = tmp2 ^ in1; + out2 = out1 ^ in3 ^ in6; + out6 = out7 ^ in0 ^ in4; + out5 = out6 ^ in3; + out3 = tmp0 ^ out2 ^ in4; + out4 = tmp1 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1 ^ in6; + out7 = in0 ^ in5; + out2 = in3 ^ in6; + out6 = in4 ^ in7; + out1 = in0 ^ in2 ^ in7; + out3 = out0 ^ out6 ^ in0; + out4 = out1 ^ out7 ^ in6; + out5 = out2 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in3 ^ in6; + tmp2 = in5 ^ in7; + tmp3 = tmp0 ^ in7; + out0 = tmp0 ^ in6; + out2 = tmp1 ^ in2; + out5 = tmp1 ^ tmp2; + out7 = tmp2 ^ in0; + tmp4 = tmp3 ^ in4; + out1 = tmp3 ^ in2; + out6 = tmp4 ^ out0; + out4 = out6 ^ in2 ^ in5; + out3 = tmp1 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in2; + out0 = in1 ^ in7; + out7 = in0 ^ in6; + out5 = in4 ^ in6; + out6 = in5 ^ in7; + out2 = out0 ^ in0 ^ in3; + out3 = out5 ^ out7 ^ in2 ^ in7; + out4 = out6 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in1 ^ in2; + tmp0 = in6 ^ in7; + out0 = in0 ^ in1 ^ in7; + out5 = in4 ^ in5 ^ in6; + out6 = tmp0 ^ in5; + out7 = tmp0 ^ in0; + out4 = tmp0 ^ out5 ^ in3; + out2 = out0 ^ in2 ^ in3; + out3 = out2 ^ in1 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1; + out4 = in5; + out7 = in0; + out5 = in6; + out6 = in7; + out3 = in0 ^ in4; + out1 = in0 ^ in2; + out2 = in0 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_8F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in0 ^ in1; + tmp0 = in0 ^ in3; + out4 = in4 ^ in5; + out7 = in0 ^ in7; + out5 = in5 ^ in6; + out6 = in6 ^ in7; + out1 = out0 ^ in2; + out2 = tmp0 ^ in2; + out3 = tmp0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_90(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in2 ^ in6 ^ in7; + out3 = tmp0 ^ in7; + out1 = tmp1 ^ in5; + tmp2 = out1 ^ in4; + out6 = tmp2 ^ in3; + out5 = out6 ^ in1; + out4 = out5 ^ in0; + out0 = tmp0 ^ tmp2; + out7 = tmp0 ^ out4; + out2 = tmp1 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_91(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in4; + tmp1 = tmp0 ^ in3 ^ in5; + out2 = tmp1 ^ in1; + out6 = tmp1 ^ in7; + tmp2 = out2 ^ in5 ^ in7; + out3 = tmp2 ^ in4; + out5 = tmp2 ^ in6; + out1 = tmp1 ^ out5 ^ in2; + tmp3 = out1 ^ in0; + out4 = tmp3 ^ in3; + out0 = tmp0 ^ tmp3; + out7 = tmp2 ^ tmp3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_92(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1; + tmp0 = in4 ^ in5; + tmp1 = tmp0 ^ in1; + out2 = tmp0 ^ in3 ^ in7; + out0 = tmp1 ^ in6; + out7 = out2 ^ in0; + out4 = out0 ^ in0 ^ in2; + out5 = out4 ^ out7 ^ in5; + out6 = tmp1 ^ out5; + out1 = out6 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_93(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in3; + tmp0 = in2 ^ in7; + tmp1 = out3 ^ in6; + tmp2 = tmp0 ^ in4; + out5 = tmp0 ^ tmp1; + out6 = tmp2 ^ in3; + out2 = out6 ^ in5; + out0 = out2 ^ out5 ^ in0; + out7 = tmp1 ^ out0; + out1 = tmp2 ^ out0; + out4 = out1 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_94(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2 ^ in6; + tmp0 = in1 ^ in4 ^ in5; + out1 = out3 ^ in5; + out5 = tmp0 ^ out3; + out0 = tmp0 ^ in7; + out4 = tmp0 ^ in0 ^ in3; + out6 = out1 ^ in3 ^ in7; + out2 = out4 ^ in6; + out7 = out0 ^ out2 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_95(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + out3 = tmp0 ^ in6; + tmp1 = tmp0 ^ in7; + tmp2 = out3 ^ in0; + out6 = tmp1 ^ in5; + tmp3 = tmp2 ^ in4; + out7 = tmp3 ^ in2; + tmp4 = tmp3 ^ in5; + out2 = tmp4 ^ in1; + tmp5 = out2 ^ in6; + out0 = tmp1 ^ tmp5; + out1 = tmp5 ^ out7; + out4 = tmp2 ^ out1; + out5 = tmp4 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_96(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in6 ^ in7; + tmp0 = in1 ^ in5; + tmp1 = in5 ^ in6; + out6 = out3 ^ in2 ^ in3; + out0 = tmp0 ^ in4; + tmp2 = tmp1 ^ in2; + out4 = out0 ^ in0 ^ in7; + out1 = tmp2 ^ in0; + out5 = tmp2 ^ in1; + out7 = tmp0 ^ out4 ^ in3; + out2 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_97(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in4; + tmp1 = in2 ^ in6; + out3 = in3 ^ in6 ^ in7; + out7 = tmp0 ^ in3; + tmp2 = tmp0 ^ in5; + out5 = tmp1 ^ in1; + out6 = tmp1 ^ out3; + out0 = tmp2 ^ in1; + out2 = tmp2 ^ out3 ^ in2; + tmp3 = out0 ^ in4; + out4 = tmp3 ^ in7; + out1 = tmp1 ^ tmp3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_98(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in5 ^ in7; + tmp1 = in1 ^ in4 ^ in7; + out1 = tmp0 ^ in2; + out0 = tmp1 ^ in6; + out2 = tmp1 ^ in3; + out6 = out0 ^ out1 ^ in1; + out5 = tmp0 ^ out2; + out3 = tmp1 ^ out6 ^ in0; + out7 = out0 ^ out5 ^ in0; + out4 = out6 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_99(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + out5 = in1 ^ in3 ^ in4; + out6 = in2 ^ in4 ^ in5; + out4 = tmp0 ^ in2; + tmp1 = tmp0 ^ in6; + tmp2 = out5 ^ in7; + out7 = tmp1 ^ in5; + out0 = tmp1 ^ tmp2; + out2 = tmp2 ^ in2; + out3 = out0 ^ out6 ^ in3; + out1 = tmp1 ^ out3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9A(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3 ^ in4; + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in6; + out5 = in1 ^ in3 ^ in5; + tmp2 = tmp0 ^ in7; + out3 = tmp0 ^ tmp1; + out0 = tmp1 ^ in4; + out7 = tmp2 ^ in3; + out1 = tmp2 ^ in2; + out6 = out0 ^ in1 ^ in2; + out4 = out1 ^ in4 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9B(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in1 ^ in3; + tmp0 = in3 ^ in5; + out6 = in2 ^ in4; + out4 = in0 ^ in2 ^ in7; + out7 = tmp0 ^ in0; + out2 = out6 ^ in3; + out1 = out4 ^ in1 ^ in5; + out3 = out7 ^ in1 ^ in6; + out0 = tmp0 ^ out3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9C(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out1 = in2 ^ in5; + tmp0 = in0 ^ in3 ^ in6; + out3 = out1 ^ in0; + out6 = out1 ^ in6; + out7 = tmp0 ^ in7; + out4 = out7 ^ in4; + out2 = out4 ^ in1; + out0 = tmp0 ^ out2; + out5 = out0 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9D(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out6 = in2 ^ in5; + tmp0 = in0 ^ in3; + out5 = in1 ^ in4 ^ in7; + out1 = out6 ^ in1; + out3 = tmp0 ^ out6; + out7 = tmp0 ^ in6; + out0 = out5 ^ in0; + out4 = out7 ^ in7; + out2 = out5 ^ out7 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9E(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1 ^ in4; + tmp0 = in0 ^ in5; + out6 = in2 ^ in6; + out7 = in0 ^ in3 ^ in7; + out4 = in0 ^ in4 ^ in6; + out5 = in1 ^ in5 ^ in7; + out1 = tmp0 ^ in2; + out3 = tmp0 ^ in7; + out2 = out4 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_9F(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out6 = in2; + out7 = in0 ^ in3; + tmp0 = in0 ^ in1; + out4 = in0 ^ in6; + out5 = in1 ^ in7; + out1 = tmp0 ^ in2 ^ in5; + out2 = out7 ^ in2 ^ in4 ^ in6; + out3 = out7 ^ in5 ^ in7; + out0 = tmp0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in6; + out2 = tmp0 ^ in7; + tmp1 = tmp0 ^ in5; + out6 = out2 ^ in3 ^ in4; + out0 = tmp1 ^ in3; + tmp2 = out0 ^ in2; + out3 = tmp2 ^ in7; + tmp3 = tmp2 ^ in1; + out5 = tmp3 ^ in0; + out4 = tmp3 ^ out6; + out7 = out5 ^ out6 ^ in1; + out1 = tmp1 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = tmp0 ^ in1; + tmp2 = tmp0 ^ in4; + out4 = tmp1 ^ in7; + out7 = tmp2 ^ in0; + out6 = tmp2 ^ out4 ^ in3; + out3 = out4 ^ in6; + out2 = out3 ^ in5; + out1 = out2 ^ in4; + out5 = out1 ^ out6 ^ in0; + out0 = tmp1 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in6; + tmp0 = in1 ^ in3 ^ in5; + out3 = tmp0 ^ in6; + out4 = tmp0 ^ in2 ^ in4; + out0 = out3 ^ in7; + out6 = out0 ^ in4; + out1 = out0 ^ out4 ^ in0; + out7 = out1 ^ in5; + out5 = out7 ^ in3 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in6; + out3 = in1 ^ in5 ^ in6; + tmp0 = out2 ^ in0; + out4 = out2 ^ out3 ^ in3; + tmp1 = tmp0 ^ in4; + out0 = tmp0 ^ out4 ^ in7; + out5 = tmp1 ^ in3; + out7 = tmp1 ^ in5; + out1 = tmp1 ^ in1 ^ in7; + out6 = tmp1 ^ out0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in3; + tmp1 = in2 ^ in4; + tmp2 = in2 ^ in5; + tmp3 = in0 ^ in7; + out0 = tmp0 ^ in5; + out6 = tmp0 ^ in6 ^ in7; + out1 = tmp1 ^ in6; + out7 = tmp1 ^ tmp3; + out3 = tmp2 ^ in3; + tmp4 = tmp2 ^ out1; + out2 = tmp3 ^ in1; + out5 = tmp4 ^ out7; + out4 = tmp4 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2 ^ in5; + tmp0 = in1 ^ in6; + tmp1 = in0 ^ in1; + tmp2 = in2 ^ in4; + out6 = in1 ^ in3 ^ in7; + out4 = tmp0 ^ in5; + out1 = tmp0 ^ tmp2; + out0 = tmp1 ^ in3 ^ in5; + out2 = tmp1 ^ in2 ^ in7; + out7 = tmp2 ^ in0; + out5 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0; + out3 = in3 ^ in5 ^ in7; + out1 = in0 ^ in2 ^ in4 ^ in6; + out0 = out3 ^ in1; + out7 = out1 ^ in7; + out6 = out0 ^ in6; + out5 = out7 ^ in5; + out4 = out6 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in2; + out3 = in5 ^ in7; + out7 = out2 ^ in4 ^ in6; + out6 = out3 ^ in1 ^ in3; + out1 = out7 ^ in1; + out5 = out7 ^ in7; + out0 = out6 ^ in0; + out4 = out6 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in4; + tmp1 = in1 ^ in6; + tmp2 = in0 ^ in2 ^ in7; + out1 = tmp0 ^ in7; + out4 = tmp0 ^ in6; + out0 = tmp1 ^ in3; + out2 = tmp1 ^ in5; + out6 = tmp1 ^ in4; + out7 = tmp2 ^ in5; + out3 = tmp2 ^ out0 ^ in6; + out5 = out7 ^ in2 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_A9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2 ^ in6; + out6 = in1 ^ in4; + out7 = in0 ^ in2 ^ in5; + out5 = in0 ^ in3 ^ in7; + out2 = out4 ^ in1 ^ in5; + out1 = out6 ^ in2 ^ in7; + out0 = out2 ^ out7 ^ in3; + out3 = out1 ^ in0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = in1 ^ in3; + tmp2 = in6 ^ in7; + out1 = tmp0 ^ in4 ^ in7; + out3 = tmp1 ^ in0; + out0 = tmp1 ^ tmp2; + out2 = tmp2 ^ in5; + out7 = tmp0 ^ out2; + out6 = out1 ^ out7 ^ in1; + out5 = out0 ^ out6 ^ in0; + out4 = out5 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in1; + tmp0 = in1 ^ in4; + tmp1 = in0 ^ in7; + out6 = tmp0 ^ in5; + out1 = tmp0 ^ tmp1 ^ in2; + out5 = tmp1 ^ in3 ^ in4; + out0 = tmp0 ^ out5 ^ in6; + out4 = out0 ^ out3 ^ in2; + out2 = out4 ^ in3 ^ in5; + out7 = tmp1 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1 ^ in3; + out1 = in2 ^ in4; + tmp0 = in0 ^ in2; + out4 = in4 ^ in7; + out5 = in0 ^ in5; + out6 = in1 ^ in6; + out7 = tmp0 ^ in7; + out3 = tmp0 ^ in3 ^ in6; + out2 = out5 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in7; + out5 = in0; + out6 = in1; + out7 = in0 ^ in2; + out0 = in0 ^ in1 ^ in3; + out2 = out7 ^ in1 ^ in5; + out1 = in1 ^ in2 ^ in4; + out3 = out7 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in3 ^ in4; + tmp0 = in0 ^ in4; + tmp1 = in0 ^ in7; + out0 = in1 ^ in3 ^ in7; + out1 = tmp0 ^ in2; + out5 = tmp0 ^ in5; + tmp2 = tmp1 ^ in6; + out2 = tmp1 ^ in5; + out3 = tmp2 ^ in3; + out7 = tmp2 ^ in2; + out6 = tmp2 ^ out2 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_AF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in3; + tmp0 = in0 ^ in7; + out5 = in0 ^ in4; + out6 = in1 ^ in5; + out7 = in0 ^ in2 ^ in6; + out0 = tmp0 ^ in1 ^ in3; + out3 = tmp0 ^ in6; + out2 = tmp0 ^ in2 ^ in5; + out1 = out5 ^ in1 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in4; + tmp1 = in3 ^ in6; + out2 = tmp0 ^ in7; + tmp2 = tmp0 ^ tmp1; + out0 = tmp2 ^ in5; + out3 = tmp2 ^ in2; + out6 = out3 ^ in6; + tmp3 = out6 ^ in0 ^ in1; + out7 = tmp3 ^ in5; + out5 = tmp3 ^ out2; + out1 = out0 ^ out5 ^ in0; + out4 = tmp1 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in4; + out2 = tmp0 ^ in2 ^ in7; + tmp1 = out2 ^ in6; + out1 = tmp1 ^ in5; + out3 = tmp1 ^ in7; + out4 = tmp1 ^ in0; + out6 = out3 ^ in3; + out0 = out6 ^ in0 ^ in2 ^ in5; + out5 = tmp1 ^ out0 ^ in1; + out7 = tmp0 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in4; + tmp0 = in4 ^ in7; + tmp1 = in1 ^ in3 ^ in6; + out3 = tmp0 ^ tmp1; + tmp2 = tmp1 ^ in0; + out0 = out3 ^ in5; + out4 = tmp2 ^ in2; + tmp3 = out4 ^ in6; + out5 = tmp0 ^ tmp3; + out1 = tmp3 ^ out0; + tmp4 = out1 ^ in7; + out7 = tmp4 ^ in3; + out6 = tmp2 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in4; + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in6; + out3 = tmp1 ^ in4 ^ in7; + tmp2 = tmp0 ^ out3; + out0 = tmp2 ^ in3; + out1 = tmp2 ^ in2; + out5 = out0 ^ in2 ^ in6; + out7 = tmp1 ^ out5; + out4 = out7 ^ in1 ^ in5 ^ in7; + out6 = tmp0 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in1; + out5 = out4 ^ in2; + tmp0 = out4 ^ in4; + out6 = out5 ^ in0 ^ in3; + out7 = tmp0 ^ out6; + out2 = tmp0 ^ in6 ^ in7; + out3 = out7 ^ in0 ^ in7; + out0 = out5 ^ out7 ^ in5; + out1 = out0 ^ out6 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in2 ^ in4; + out4 = tmp0 ^ in4; + out3 = tmp1 ^ in7; + tmp2 = out4 ^ in5; + out7 = out3 ^ in0 ^ in3; + out0 = tmp2 ^ in3; + out2 = tmp0 ^ out3 ^ in6; + out5 = tmp1 ^ tmp2; + out6 = out2 ^ out7 ^ in2; + out1 = tmp0 ^ out0 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in3 ^ in4; + tmp0 = in1 ^ in2; + tmp1 = in0 ^ in4; + tmp2 = in3 ^ in5; + tmp3 = out3 ^ in1 ^ in7; + out5 = tmp0 ^ tmp1; + out6 = tmp0 ^ tmp2; + out2 = tmp1 ^ in6; + out4 = tmp1 ^ tmp3; + out0 = tmp3 ^ in5; + out1 = out2 ^ in2 ^ in5; + out7 = tmp2 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in4; + tmp0 = in0 ^ in4; + out2 = tmp0 ^ in2 ^ in6; + tmp1 = out2 ^ in7; + out1 = out2 ^ in1 ^ in5; + out7 = tmp1 ^ in3; + out5 = out1 ^ in6; + out6 = tmp0 ^ out1 ^ in3; + out0 = tmp1 ^ out6; + out4 = out0 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in4; + tmp1 = in2 ^ in5; + out2 = tmp0 ^ in5; + out4 = tmp1 ^ in0; + tmp2 = tmp1 ^ in7; + out6 = tmp2 ^ out2; + out7 = out4 ^ in3; + out1 = tmp2 ^ in4; + out3 = tmp0 ^ out7; + out0 = out3 ^ out4 ^ in6; + out5 = out0 ^ in0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_B9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = in4 ^ in5; + out4 = tmp0 ^ tmp1; + tmp2 = tmp0 ^ in3 ^ in7; + out3 = out4 ^ in1; + out7 = tmp2 ^ in5; + out2 = out3 ^ in0; + out1 = out2 ^ in7; + out6 = out1 ^ in5 ^ in6; + out0 = tmp2 ^ out6; + out5 = tmp1 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in5 ^ in7; + out2 = tmp0 ^ in4; + tmp1 = out2 ^ in2; + out1 = tmp1 ^ in0; + out6 = tmp1 ^ in1; + out4 = out1 ^ in3 ^ in4; + tmp2 = out4 ^ out6; + out7 = out4 ^ in6 ^ in7; + out5 = tmp2 ^ in6; + out3 = tmp0 ^ tmp2; + out0 = out6 ^ out7 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in2 ^ in4 ^ in5 ^ in7; + tmp0 = out2 ^ in1; + out4 = out2 ^ in0 ^ in3; + out1 = tmp0 ^ in0; + out6 = tmp0 ^ in6; + out3 = out1 ^ in2; + tmp1 = out4 ^ out6 ^ in4; + out0 = tmp1 ^ in7; + out5 = tmp1 ^ in5; + out7 = tmp0 ^ tmp1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = in2 ^ in4; + out0 = in1 ^ in3 ^ in4; + out6 = in1 ^ in2 ^ in7; + out7 = tmp0 ^ in3; + out5 = tmp0 ^ out6 ^ in6; + out1 = tmp1 ^ in5; + tmp2 = out1 ^ out5 ^ in1; + out3 = tmp2 ^ in3; + out4 = tmp1 ^ tmp2; + out2 = tmp2 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in1 ^ in4; + out0 = tmp0 ^ tmp1; + out7 = tmp0 ^ in2 ^ in7; + out1 = tmp1 ^ in2 ^ in5; + tmp2 = out1 ^ in0; + out2 = tmp2 ^ in6; + out3 = out2 ^ in1 ^ in7; + out4 = out3 ^ in2; + out5 = tmp1 ^ out4; + out6 = tmp2 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3 ^ in6; + out4 = tmp0 ^ in5; + out7 = tmp0 ^ in2; + out3 = out4 ^ in4; + out1 = out3 ^ out7 ^ in0; + out2 = out3 ^ in3 ^ in7; + out0 = out2 ^ out4 ^ in1; + out5 = tmp0 ^ out0; + out6 = out1 ^ out5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_BF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in4; + out3 = tmp0 ^ in5 ^ in6; + out4 = out3 ^ in3; + tmp1 = out3 ^ in7; + out2 = tmp1 ^ in2; + out5 = tmp1 ^ in1; + tmp2 = out2 ^ in5; + out7 = tmp2 ^ in3 ^ in4; + tmp3 = tmp0 ^ out5; + out0 = tmp3 ^ out4; + out1 = tmp2 ^ tmp3; + out6 = tmp3 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in2 ^ in5; + tmp0 = in1 ^ in4; + tmp1 = in3 ^ in6; + out0 = out5 ^ in1; + out4 = tmp0 ^ in7; + out3 = tmp0 ^ tmp1; + out1 = tmp1 ^ in2; + out6 = tmp1 ^ in0; + out7 = out4 ^ in0; + out2 = out4 ^ out5 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in2; + tmp0 = in0 ^ in1; + out4 = in1 ^ in7; + out6 = in0 ^ in3; + out3 = in1 ^ in4 ^ in6; + tmp1 = tmp0 ^ in2; + out7 = tmp0 ^ in4; + out0 = tmp1 ^ in5; + out1 = tmp1 ^ out6 ^ in6; + out2 = out6 ^ out7 ^ in5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in3 ^ in4; + tmp0 = in0 ^ in3 ^ in6; + out5 = in2 ^ in4 ^ in5; + tmp1 = out4 ^ in7; + out1 = tmp0 ^ in2; + out6 = tmp0 ^ in5; + out2 = out5 ^ in3; + out7 = tmp0 ^ tmp1; + out3 = tmp1 ^ in2 ^ in6; + out0 = tmp1 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in1 ^ in3; + tmp0 = in0 ^ in2; + tmp1 = in3 ^ in5; + out5 = in2 ^ in4; + tmp2 = tmp0 ^ out4; + out2 = tmp1 ^ in4; + out6 = tmp1 ^ in0; + out0 = tmp1 ^ tmp2 ^ in7; + out1 = tmp2 ^ in6; + out7 = out1 ^ out5 ^ in3; + out3 = tmp0 ^ out7 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in7; + out3 = tmp0 ^ in4; + tmp1 = tmp0 ^ in2; + out1 = tmp1 ^ in6; + out5 = tmp1 ^ in5; + out4 = out1 ^ out3 ^ in1; + out0 = out4 ^ in4 ^ in5; + out2 = out0 ^ out3 ^ in0; + out7 = out1 ^ out2 ^ in7; + out6 = tmp1 ^ out0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in4 ^ in7; + tmp0 = in3 ^ in7; + out4 = in1 ^ in2 ^ in6; + out6 = in0 ^ in3 ^ in4; + out5 = tmp0 ^ in2; + out1 = tmp0 ^ out4; + out0 = out4 ^ in0 ^ in5; + out2 = out0 ^ out5 ^ in4; + out7 = tmp0 ^ out2 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in5 ^ in6; + tmp1 = in1 ^ in7; + tmp2 = tmp0 ^ in0; + tmp3 = tmp0 ^ tmp1; + tmp4 = tmp2 ^ in4; + out0 = tmp3 ^ in2; + out6 = tmp4 ^ in3; + out2 = out6 ^ in2; + out7 = tmp1 ^ tmp4; + out3 = tmp2 ^ out2; + tmp5 = out3 ^ in5; + out5 = tmp5 ^ in7; + out4 = tmp3 ^ tmp5; + out1 = tmp4 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2 ^ in4; + tmp0 = in3 ^ in5; + tmp1 = out3 ^ in7; + out6 = tmp0 ^ in0 ^ in4; + out5 = tmp1 ^ in3; + out2 = out6 ^ in6; + out7 = out2 ^ in1 ^ in3; + out0 = tmp1 ^ out7; + out1 = tmp0 ^ out0; + out4 = out1 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out0 = in1 ^ in2; + out1 = in2 ^ in3; + tmp0 = in5 ^ in6; + tmp1 = in0 ^ in7; + out2 = out1 ^ in1 ^ in4; + out4 = tmp0 ^ in4; + out5 = tmp0 ^ in7; + out6 = tmp1 ^ in6; + out7 = tmp1 ^ in1; + out3 = out2 ^ in0 ^ in2 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_C9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in5 ^ in6; + out7 = in0 ^ in1; + tmp0 = in1 ^ in3; + out5 = in6 ^ in7; + out6 = in0 ^ in7; + out0 = out7 ^ in2; + out3 = out7 ^ in4 ^ in5; + out1 = tmp0 ^ in2; + out2 = tmp0 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in7; + tmp1 = in2 ^ in7; + tmp2 = tmp0 ^ in6; + out0 = tmp1 ^ in1; + tmp3 = tmp1 ^ in3; + out6 = tmp2 ^ in5; + out7 = tmp2 ^ in1; + out2 = tmp3 ^ in4; + out5 = out6 ^ in0 ^ in4; + out4 = out5 ^ in3; + out1 = tmp0 ^ tmp3; + out3 = tmp3 ^ out5 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in7; + tmp1 = in5 ^ in7; + out7 = in0 ^ in1 ^ in6; + out5 = tmp0 ^ in6; + out2 = tmp0 ^ in3; + out6 = tmp1 ^ in0; + out4 = tmp1 ^ in3 ^ in6; + tmp2 = out5 ^ out7 ^ in2; + out1 = tmp2 ^ out2; + out0 = tmp2 ^ in4; + out3 = tmp2 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5; + tmp1 = in1 ^ in6; + out1 = in2 ^ in3 ^ in7; + out5 = tmp0 ^ in6; + out0 = tmp1 ^ in2; + tmp2 = out5 ^ in0 ^ in7; + out3 = tmp2 ^ in4; + out6 = tmp0 ^ out3; + out7 = tmp1 ^ tmp2 ^ in3; + tmp3 = out1 ^ out6; + out4 = tmp2 ^ tmp3; + out2 = tmp3 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in3 ^ in6; + tmp0 = in0 ^ in1; + tmp1 = in2 ^ in7; + out6 = in0 ^ in4 ^ in7; + out2 = tmp0 ^ out5 ^ in4; + out7 = tmp0 ^ in5; + out0 = tmp0 ^ in2 ^ in6; + out4 = tmp1 ^ in5; + out1 = tmp1 ^ in1 ^ in3; + out3 = out6 ^ in5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in5; + tmp1 = tmp0 ^ in3; + out4 = tmp1 ^ in4; + tmp2 = out4 ^ in6; + out3 = tmp2 ^ in0; + out5 = tmp2 ^ in2; + out2 = out3 ^ in5 ^ in7; + out6 = tmp1 ^ out2; + out7 = out2 ^ out4 ^ in1; + out1 = tmp2 ^ out6; + out0 = tmp0 ^ out7 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_CF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in6; + tmp1 = in0 ^ in1 ^ in5; + out4 = in2 ^ in3 ^ in5; + out5 = tmp0 ^ in4; + out7 = tmp1 ^ in6; + out1 = tmp1 ^ out4 ^ in7; + tmp2 = out5 ^ in0; + out2 = tmp2 ^ in7; + out3 = tmp2 ^ out4; + out6 = tmp0 ^ out2 ^ in5; + out0 = tmp0 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + tmp1 = in1 ^ in4; + tmp2 = in2 ^ in5; + out7 = tmp0 ^ tmp1; + out0 = tmp1 ^ tmp2; + tmp3 = tmp2 ^ in3; + out1 = tmp3 ^ in6; + tmp4 = out1 ^ in1; + out2 = tmp4 ^ in7; + out3 = out2 ^ in2; + out4 = tmp0 ^ out3; + out5 = tmp3 ^ out3; + out6 = tmp4 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5 ^ in6; + tmp1 = tmp0 ^ in1; + out1 = tmp1 ^ in2; + out2 = tmp1 ^ in7; + out3 = out2 ^ in3; + out5 = out3 ^ in2; + tmp2 = out3 ^ in0; + out4 = tmp2 ^ in4; + out7 = tmp0 ^ out4; + out6 = tmp2 ^ out1 ^ in6; + out0 = out2 ^ out6 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in5 ^ in6; + out2 = tmp0 ^ in2 ^ in3; + out1 = out2 ^ in0; + out3 = out2 ^ in1; + out4 = out1 ^ in1 ^ in2; + out6 = out1 ^ in6 ^ in7; + out7 = out4 ^ in4 ^ in5; + out5 = out4 ^ out6 ^ in4; + out0 = tmp0 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in3 ^ in5 ^ in6; + tmp0 = out2 ^ in2; + tmp1 = tmp0 ^ in1; + out1 = tmp1 ^ in0; + out3 = tmp1 ^ in3; + out4 = out1 ^ in2 ^ in4; + tmp2 = out4 ^ in5; + out7 = tmp2 ^ in7; + out0 = tmp0 ^ out7; + tmp3 = out0 ^ in0; + out5 = tmp3 ^ in6; + out6 = tmp2 ^ tmp3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in3 ^ in5; + tmp0 = in1 ^ in5; + tmp1 = tmp0 ^ in2; + out4 = tmp1 ^ in0; + tmp2 = tmp1 ^ in6; + out2 = out4 ^ in3 ^ in7; + out0 = tmp2 ^ in4; + out5 = tmp2 ^ out3; + out1 = tmp0 ^ out5 ^ in7; + out6 = tmp0 ^ out2 ^ in4; + out7 = tmp1 ^ out6 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in5; + tmp0 = in0 ^ in4; + tmp1 = tmp0 ^ in1 ^ in5; + out4 = tmp1 ^ in2; + out0 = out4 ^ in6; + tmp2 = tmp0 ^ out0; + out5 = tmp2 ^ in3; + out1 = out5 ^ in7; + out6 = tmp1 ^ out1; + out7 = tmp2 ^ out6; + out2 = out7 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2 ^ in4 ^ in6; + out5 = tmp0 ^ in3; + out0 = tmp0 ^ in5 ^ in7; + out3 = out0 ^ out5 ^ in2; + tmp1 = out3 ^ in0; + out1 = tmp1 ^ in6; + out2 = tmp1 ^ in7; + out4 = tmp1 ^ in1; + out6 = tmp1 ^ in4; + out7 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in3; + out3 = in2 ^ in5 ^ in7; + out2 = tmp0 ^ in5; + tmp1 = tmp0 ^ out3 ^ in1; + out1 = tmp1 ^ in6; + out4 = tmp1 ^ in4; + tmp2 = out1 ^ in4; + out6 = tmp2 ^ in1; + out7 = tmp2 ^ in2; + out0 = tmp2 ^ in3; + out5 = tmp2 ^ in0 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0; + out5 = in1; + tmp0 = in1 ^ in2; + out6 = in0 ^ in2; + out0 = tmp0 ^ in4; + tmp1 = tmp0 ^ in3; + out7 = tmp1 ^ out6; + out2 = tmp1 ^ in6; + out3 = out7 ^ in7; + out1 = tmp1 ^ in1 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_D9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in0 ^ in4; + out5 = in1 ^ in5; + out2 = in1 ^ in3 ^ in6; + out3 = in0 ^ in1 ^ in7; + out6 = in0 ^ in2 ^ in6; + out0 = out4 ^ in1 ^ in2; + out1 = out5 ^ in2 ^ in3; + out7 = out3 ^ in3; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out5 = in1 ^ in4; + tmp0 = in2 ^ in7; + tmp1 = in0 ^ in2 ^ in3; + out0 = tmp0 ^ out5; + out4 = tmp0 ^ tmp1; + out2 = tmp0 ^ in3 ^ in6; + out1 = tmp1 ^ in5; + out3 = tmp1 ^ in1; + out6 = out1 ^ in3; + out7 = out3 ^ in2 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in1 ^ in5; + tmp2 = in3 ^ in7; + out3 = tmp0 ^ in2; + out5 = tmp1 ^ in4; + out6 = tmp1 ^ out3 ^ in6; + out2 = tmp2 ^ in6; + tmp3 = tmp2 ^ in4; + tmp4 = out3 ^ in3; + out4 = tmp3 ^ in0; + out1 = tmp4 ^ in5; + out0 = tmp3 ^ tmp4; + out7 = tmp0 ^ out2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + tmp1 = in0 ^ in3; + out6 = tmp0 ^ in4; + tmp2 = tmp0 ^ in7; + out3 = tmp1 ^ in6; + tmp3 = tmp1 ^ in1; + out1 = tmp1 ^ tmp2 ^ in5; + out4 = tmp2 ^ in6; + out2 = tmp3 ^ in2; + out7 = tmp3 ^ in5; + out5 = tmp2 ^ out2; + out0 = out2 ^ out3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in6; + out2 = in0 ^ in1 ^ in3; + out6 = out3 ^ in2 ^ in4; + out7 = out2 ^ in5 ^ in7; + out0 = out6 ^ in1; + out4 = out6 ^ in7; + out5 = out7 ^ in0; + out1 = out5 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3 ^ in6; + tmp1 = in3 ^ in4 ^ in7; + out4 = tmp0 ^ in0; + out5 = tmp1 ^ in1; + out3 = out4 ^ in7; + out2 = out3 ^ in6; + out1 = out2 ^ in5; + out6 = tmp1 ^ out1; + out0 = tmp0 ^ out5; + out7 = out0 ^ out1 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_DF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in3 ^ in7; + tmp0 = out2 ^ in1 ^ in5; + out1 = tmp0 ^ in2; + out7 = tmp0 ^ in6; + out5 = tmp0 ^ in0 ^ in4; + tmp1 = out1 ^ out5 ^ in6; + out4 = tmp1 ^ in3; + out6 = tmp1 ^ in5; + tmp2 = tmp1 ^ in7; + out0 = tmp2 ^ in1; + out3 = tmp2 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in7; + tmp0 = in2 ^ in4; + out4 = out3 ^ in3 ^ in5; + out2 = tmp0 ^ in1; + tmp1 = tmp0 ^ in6; + out0 = out4 ^ in2; + out6 = out4 ^ in0; + out1 = tmp1 ^ in3; + out5 = tmp1 ^ in0; + out7 = out5 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1 ^ in4; + tmp0 = in1 ^ in7; + out3 = tmp0 ^ in3; + tmp1 = out3 ^ in5; + out4 = tmp1 ^ in4; + tmp2 = tmp1 ^ in0; + out0 = tmp2 ^ in2; + out6 = tmp2 ^ in6; + tmp3 = out0 ^ out4 ^ in6; + out5 = tmp3 ^ in5; + out7 = tmp0 ^ tmp3; + out1 = tmp2 ^ out5 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in1 ^ in2; + out4 = in1 ^ in5; + out2 = in2 ^ in4 ^ in7; + out5 = in0 ^ in2 ^ in6; + out0 = out3 ^ in3 ^ in5; + out7 = out3 ^ in0 ^ in4; + out6 = out2 ^ out7 ^ in3; + out1 = out5 ^ in3 ^ in4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in4 ^ in7; + tmp0 = in1 ^ in3; + out3 = tmp0 ^ in2; + tmp1 = out3 ^ in0; + out0 = tmp1 ^ in5; + tmp2 = tmp1 ^ in4; + out1 = tmp2 ^ in6; + tmp3 = tmp2 ^ in3; + out7 = tmp3 ^ in7; + out6 = out1 ^ out2 ^ in2; + tmp4 = tmp0 ^ out0; + out5 = tmp4 ^ in6; + out4 = tmp3 ^ tmp4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in6; + tmp0 = in0 ^ in4; + tmp1 = tmp0 ^ in2 ^ in6; + out2 = tmp1 ^ in1; + out7 = out2 ^ in5; + tmp2 = tmp0 ^ out7; + out4 = tmp2 ^ in3; + out0 = out4 ^ in7; + out6 = tmp1 ^ out0; + out5 = tmp2 ^ out6; + out1 = out5 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in3 ^ in6; + tmp0 = in0 ^ in1; + tmp1 = in5 ^ in7; + out2 = tmp0 ^ in4 ^ in6; + tmp2 = tmp1 ^ out2; + out6 = tmp2 ^ in3; + out7 = tmp2 ^ in2; + out0 = out6 ^ in2 ^ in4; + out5 = out6 ^ in1 ^ in2; + out1 = tmp0 ^ out5 ^ in5; + out4 = tmp1 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in2 ^ in6 ^ in7; + out2 = out3 ^ in0 ^ in4; + out4 = out3 ^ in1 ^ in5; + out1 = out2 ^ in3; + out7 = out2 ^ out4 ^ in2; + out0 = out4 ^ in3 ^ in7; + out5 = out1 ^ in4; + out6 = out0 ^ out2 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in3; + out3 = tmp0 ^ in6 ^ in7; + tmp1 = out3 ^ in0; + out5 = tmp1 ^ in5; + tmp2 = tmp1 ^ in4; + tmp3 = out5 ^ in7; + out1 = tmp2 ^ in1; + out0 = tmp3 ^ in1; + out6 = out1 ^ in2; + out2 = tmp0 ^ tmp2; + tmp4 = tmp3 ^ out6; + out4 = tmp4 ^ in6; + out7 = tmp4 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in3 ^ in6; + tmp0 = in4 ^ in7; + out1 = in2 ^ in3 ^ in4; + out5 = tmp0 ^ in0; + tmp1 = tmp0 ^ in1; + tmp2 = tmp1 ^ in5; + out0 = tmp1 ^ out1; + out2 = tmp2 ^ in2; + out6 = tmp2 ^ out5; + tmp3 = out6 ^ in6; + out3 = tmp3 ^ in7; + out7 = tmp3 ^ in2 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_E9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = in3 ^ in6; + tmp2 = tmp0 ^ in6; + out4 = tmp1 ^ in4; + out6 = tmp2 ^ in5; + out7 = tmp2 ^ in2 ^ in7; + out3 = out6 ^ in3 ^ in7; + out0 = tmp1 ^ out7; + out2 = out3 ^ out4 ^ in0; + out5 = tmp0 ^ out2; + out1 = out0 ^ out5 ^ in5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in6 ^ in7; + out5 = in0 ^ in7; + out6 = in0 ^ in1; + out0 = in1 ^ in2 ^ in3; + out2 = in2 ^ in4 ^ in5; + out7 = out6 ^ in2; + out1 = out0 ^ out6 ^ in4; + out3 = out7 ^ in5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in4 ^ in5; + tmp0 = in0 ^ in1; + out4 = in4 ^ in6 ^ in7; + out5 = in0 ^ in5 ^ in7; + out6 = tmp0 ^ in6; + tmp1 = tmp0 ^ in2; + out0 = tmp1 ^ in3; + out7 = tmp1 ^ in7; + out1 = out0 ^ in4; + out3 = out0 ^ in5 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out3 = in0 ^ in5; + out4 = in2 ^ in3 ^ in7; + out5 = in0 ^ in3 ^ in4; + out6 = out3 ^ in1 ^ in4; + out1 = out4 ^ in4; + out0 = out4 ^ in1 ^ in6; + out2 = out0 ^ out5 ^ in5; + out7 = out2 ^ in4 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_ED(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in2 ^ in4; + tmp1 = in3 ^ in5; + out4 = tmp0 ^ in3 ^ in7; + out3 = tmp1 ^ in0; + out1 = out4 ^ in1; + out5 = out3 ^ in4; + out7 = out1 ^ out5 ^ in6; + out2 = tmp0 ^ out7; + out0 = tmp1 ^ out7; + out6 = out2 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2; + tmp0 = in0 ^ in1; + out5 = in0 ^ in3; + tmp1 = tmp0 ^ in2; + out6 = tmp0 ^ in4; + tmp2 = tmp1 ^ out5; + out7 = tmp1 ^ in5; + out1 = tmp2 ^ out6 ^ in7; + out0 = tmp2 ^ in6; + tmp3 = out7 ^ in1; + out3 = tmp3 ^ in7; + out2 = tmp3 ^ in4 ^ in6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_EF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out4 = in2 ^ in4; + tmp0 = in0 ^ in5; + tmp1 = in4 ^ in6; + out5 = tmp0 ^ in3; + out2 = tmp0 ^ tmp1; + out6 = tmp1 ^ in0 ^ in1; + out3 = out5 ^ in2 ^ in7; + out7 = out3 ^ in1 ^ in3; + out0 = out4 ^ out6 ^ in3; + out1 = tmp1 ^ out0 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F0(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in4 ^ in5; + out2 = tmp0 ^ in6; + out3 = tmp1 ^ in1; + tmp2 = tmp1 ^ in7; + out1 = out2 ^ out3 ^ in3; + tmp3 = tmp0 ^ tmp2; + out0 = tmp3 ^ in3; + out5 = tmp3 ^ in0; + out4 = out1 ^ out5 ^ in4; + out7 = out4 ^ in2; + out6 = tmp2 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F1(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in1 ^ in6; + tmp0 = in3 ^ in5; + out3 = tmp0 ^ in1 ^ in4; + tmp1 = out3 ^ in2; + out1 = tmp1 ^ in6; + tmp2 = tmp1 ^ in0; + tmp3 = out1 ^ in5; + out0 = tmp2 ^ in7; + out6 = tmp2 ^ in4; + out7 = tmp3 ^ in0; + out5 = tmp0 ^ out0; + out4 = tmp3 ^ out5 ^ in1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F2(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in4 ^ in5; + out2 = in2 ^ in6 ^ in7; + tmp1 = tmp0 ^ in1; + tmp2 = tmp1 ^ in2; + out0 = tmp2 ^ in3; + out3 = tmp2 ^ in7; + out5 = out3 ^ in0 ^ in4; + tmp3 = tmp0 ^ out5; + out7 = tmp3 ^ in3; + out4 = tmp3 ^ out2; + out1 = out0 ^ out4 ^ in4; + out6 = tmp1 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F3(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in6 ^ in7; + tmp0 = in0 ^ in1; + out4 = tmp0 ^ in6; + tmp1 = tmp0 ^ in2; + out5 = tmp1 ^ in7; + out6 = tmp1 ^ in3; + out7 = out6 ^ in4; + out0 = out7 ^ in5; + out1 = out0 ^ in6; + out3 = out0 ^ in0 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F4(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in1 ^ in2; + tmp0 = out2 ^ in3; + out4 = tmp0 ^ in4; + out5 = out4 ^ in5; + out6 = out5 ^ in6; + out7 = out6 ^ in7; + out0 = out7 ^ in0; + out1 = out0 ^ in1; + out3 = tmp0 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F5(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in1; + tmp0 = out2 ^ in2; + out4 = tmp0 ^ in3; + out5 = out4 ^ in4; + out6 = out5 ^ in5; + out7 = out6 ^ in6; + out0 = out7 ^ in7; + out1 = out0 ^ in0; + out3 = tmp0 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F6(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in7; + out2 = tmp0 ^ in2; + out4 = out2 ^ in1 ^ in4; + out7 = out4 ^ in3 ^ in5; + out5 = out7 ^ in4 ^ in7; + out0 = tmp0 ^ out7 ^ in6; + tmp1 = out0 ^ in1; + out6 = out0 ^ in0 ^ in5; + out3 = tmp1 ^ in3; + out1 = tmp0 ^ tmp1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F7(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in7; + tmp0 = out2 ^ in1; + out4 = tmp0 ^ in2; + out5 = out4 ^ in3 ^ in7; + out6 = out5 ^ in4; + out7 = out6 ^ in5; + out0 = out7 ^ in6; + out1 = out0 ^ in7; + out3 = tmp0 ^ out1; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F8(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in4; + tmp1 = in3 ^ in5; + tmp2 = tmp0 ^ in6; + out4 = tmp0 ^ tmp1; + out1 = tmp1 ^ in2 ^ in4; + out3 = tmp2 ^ in1; + out5 = out3 ^ in5; + out7 = out1 ^ out5 ^ in7; + out6 = tmp1 ^ out7; + out0 = tmp2 ^ out7; + out2 = out6 ^ in0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_F9(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in3 ^ in5; + tmp1 = in0 ^ in6; + out4 = tmp0 ^ in0; + tmp2 = tmp1 ^ in4; + tmp3 = tmp1 ^ in2; + out5 = tmp2 ^ in1; + out3 = out5 ^ in3; + tmp4 = tmp3 ^ out3; + out1 = tmp4 ^ in5; + out0 = tmp4 ^ in0 ^ in7; + out6 = tmp0 ^ out0 ^ in4; + out7 = tmp2 ^ tmp4; + out2 = tmp3 ^ out6; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FA(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in1; + tmp1 = tmp0 ^ in2; + tmp2 = tmp0 ^ in5; + tmp3 = tmp1 ^ in7; + out5 = tmp2 ^ in6; + out6 = tmp3 ^ in6; + out7 = tmp3 ^ in3; + out3 = out6 ^ in4; + out2 = tmp1 ^ out5; + out4 = out2 ^ out3 ^ in1; + out0 = out4 ^ out7 ^ in5; + out1 = tmp2 ^ out0; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FB(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in5 ^ in6; + tmp0 = in0 ^ in1; + out4 = in0 ^ in5 ^ in7; + out5 = tmp0 ^ in6; + tmp1 = tmp0 ^ in2; + out6 = tmp1 ^ in7; + out7 = tmp1 ^ in3; + out0 = out7 ^ in4; + out1 = out0 ^ in5; + out3 = out0 ^ in6 ^ in7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FC(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in1 ^ in2; + tmp1 = in0 ^ in7; + out2 = tmp0 ^ tmp1 ^ in5; + out3 = tmp1 ^ in4; + tmp2 = out2 ^ in6; + out6 = tmp2 ^ in4; + out7 = tmp2 ^ in3; + out4 = out6 ^ in1 ^ in3; + tmp3 = out4 ^ in0; + out1 = tmp3 ^ in6; + out0 = tmp3 ^ in1 ^ in5; + out5 = tmp0 ^ out4; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FD(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in5; + tmp1 = in1 ^ in7; + out2 = tmp0 ^ tmp1; + out6 = out2 ^ in2 ^ in4; + tmp2 = out6 ^ in0; + out1 = tmp2 ^ in3; + out0 = tmp0 ^ out1 ^ in6; + out5 = out0 ^ in2; + tmp3 = out5 ^ in1; + out3 = tmp3 ^ in6; + out7 = tmp2 ^ tmp3; + out4 = tmp1 ^ out7; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FE(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3, tmp4; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + tmp0 = in0 ^ in2; + out2 = tmp0 ^ in5; + out3 = tmp0 ^ in4; + tmp1 = out3 ^ in6; + out4 = tmp1 ^ in5; + tmp2 = tmp1 ^ in1; + out6 = tmp2 ^ in7; + tmp3 = tmp2 ^ in0; + out0 = tmp3 ^ in3; + tmp4 = out0 ^ out4 ^ in7; + out5 = tmp4 ^ in6; + out7 = tmp4 ^ in2; + out1 = tmp3 ^ out5; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void +gf8_muladd_FF(void *out, void *in) +{ + unsigned int i; + uint64_t *in_ptr = (uint64_t *)in; + uint64_t *out_ptr = (uint64_t *)out; + + for (i = 0; i < WIDTH; i++) { + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + uint64_t tmp0, tmp1, tmp2, tmp3; + + uint64_t in0 = out_ptr[0]; + uint64_t in1 = out_ptr[WIDTH]; + uint64_t in2 = out_ptr[WIDTH * 2]; + uint64_t in3 = out_ptr[WIDTH * 3]; + uint64_t in4 = out_ptr[WIDTH * 4]; + uint64_t in5 = out_ptr[WIDTH * 5]; + uint64_t in6 = out_ptr[WIDTH * 6]; + uint64_t in7 = out_ptr[WIDTH * 7]; + + out2 = in0 ^ in5; + tmp0 = in4 ^ in7; + tmp1 = out2 ^ in2; + out4 = tmp1 ^ in6; + out7 = tmp1 ^ in1 ^ in3; + out1 = tmp0 ^ out7; + tmp2 = out1 ^ in5; + out6 = tmp2 ^ in3; + tmp3 = tmp2 ^ in7; + out0 = tmp3 ^ in6; + out3 = tmp3 ^ in1; + out5 = tmp0 ^ out0 ^ in2; + + out_ptr[0] = out0 ^ in_ptr[0]; + out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH]; + out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2]; + out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3]; + out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4]; + out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5]; + out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6]; + out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7]; + + in_ptr++; + out_ptr++; + } +} + +static void (*gf8_muladd[])(void *out, void *in) = { + gf8_muladd_00, gf8_muladd_01, gf8_muladd_02, gf8_muladd_03, gf8_muladd_04, + gf8_muladd_05, gf8_muladd_06, gf8_muladd_07, gf8_muladd_08, gf8_muladd_09, + gf8_muladd_0A, gf8_muladd_0B, gf8_muladd_0C, gf8_muladd_0D, gf8_muladd_0E, + gf8_muladd_0F, gf8_muladd_10, gf8_muladd_11, gf8_muladd_12, gf8_muladd_13, + gf8_muladd_14, gf8_muladd_15, gf8_muladd_16, gf8_muladd_17, gf8_muladd_18, + gf8_muladd_19, gf8_muladd_1A, gf8_muladd_1B, gf8_muladd_1C, gf8_muladd_1D, + gf8_muladd_1E, gf8_muladd_1F, gf8_muladd_20, gf8_muladd_21, gf8_muladd_22, + gf8_muladd_23, gf8_muladd_24, gf8_muladd_25, gf8_muladd_26, gf8_muladd_27, + gf8_muladd_28, gf8_muladd_29, gf8_muladd_2A, gf8_muladd_2B, gf8_muladd_2C, + gf8_muladd_2D, gf8_muladd_2E, gf8_muladd_2F, gf8_muladd_30, gf8_muladd_31, + gf8_muladd_32, gf8_muladd_33, gf8_muladd_34, gf8_muladd_35, gf8_muladd_36, + gf8_muladd_37, gf8_muladd_38, gf8_muladd_39, gf8_muladd_3A, gf8_muladd_3B, + gf8_muladd_3C, gf8_muladd_3D, gf8_muladd_3E, gf8_muladd_3F, gf8_muladd_40, + gf8_muladd_41, gf8_muladd_42, gf8_muladd_43, gf8_muladd_44, gf8_muladd_45, + gf8_muladd_46, gf8_muladd_47, gf8_muladd_48, gf8_muladd_49, gf8_muladd_4A, + gf8_muladd_4B, gf8_muladd_4C, gf8_muladd_4D, gf8_muladd_4E, gf8_muladd_4F, + gf8_muladd_50, gf8_muladd_51, gf8_muladd_52, gf8_muladd_53, gf8_muladd_54, + gf8_muladd_55, gf8_muladd_56, gf8_muladd_57, gf8_muladd_58, gf8_muladd_59, + gf8_muladd_5A, gf8_muladd_5B, gf8_muladd_5C, gf8_muladd_5D, gf8_muladd_5E, + gf8_muladd_5F, gf8_muladd_60, gf8_muladd_61, gf8_muladd_62, gf8_muladd_63, + gf8_muladd_64, gf8_muladd_65, gf8_muladd_66, gf8_muladd_67, gf8_muladd_68, + gf8_muladd_69, gf8_muladd_6A, gf8_muladd_6B, gf8_muladd_6C, gf8_muladd_6D, + gf8_muladd_6E, gf8_muladd_6F, gf8_muladd_70, gf8_muladd_71, gf8_muladd_72, + gf8_muladd_73, gf8_muladd_74, gf8_muladd_75, gf8_muladd_76, gf8_muladd_77, + gf8_muladd_78, gf8_muladd_79, gf8_muladd_7A, gf8_muladd_7B, gf8_muladd_7C, + gf8_muladd_7D, gf8_muladd_7E, gf8_muladd_7F, gf8_muladd_80, gf8_muladd_81, + gf8_muladd_82, gf8_muladd_83, gf8_muladd_84, gf8_muladd_85, gf8_muladd_86, + gf8_muladd_87, gf8_muladd_88, gf8_muladd_89, gf8_muladd_8A, gf8_muladd_8B, + gf8_muladd_8C, gf8_muladd_8D, gf8_muladd_8E, gf8_muladd_8F, gf8_muladd_90, + gf8_muladd_91, gf8_muladd_92, gf8_muladd_93, gf8_muladd_94, gf8_muladd_95, + gf8_muladd_96, gf8_muladd_97, gf8_muladd_98, gf8_muladd_99, gf8_muladd_9A, + gf8_muladd_9B, gf8_muladd_9C, gf8_muladd_9D, gf8_muladd_9E, gf8_muladd_9F, + gf8_muladd_A0, gf8_muladd_A1, gf8_muladd_A2, gf8_muladd_A3, gf8_muladd_A4, + gf8_muladd_A5, gf8_muladd_A6, gf8_muladd_A7, gf8_muladd_A8, gf8_muladd_A9, + gf8_muladd_AA, gf8_muladd_AB, gf8_muladd_AC, gf8_muladd_AD, gf8_muladd_AE, + gf8_muladd_AF, gf8_muladd_B0, gf8_muladd_B1, gf8_muladd_B2, gf8_muladd_B3, + gf8_muladd_B4, gf8_muladd_B5, gf8_muladd_B6, gf8_muladd_B7, gf8_muladd_B8, + gf8_muladd_B9, gf8_muladd_BA, gf8_muladd_BB, gf8_muladd_BC, gf8_muladd_BD, + gf8_muladd_BE, gf8_muladd_BF, gf8_muladd_C0, gf8_muladd_C1, gf8_muladd_C2, + gf8_muladd_C3, gf8_muladd_C4, gf8_muladd_C5, gf8_muladd_C6, gf8_muladd_C7, + gf8_muladd_C8, gf8_muladd_C9, gf8_muladd_CA, gf8_muladd_CB, gf8_muladd_CC, + gf8_muladd_CD, gf8_muladd_CE, gf8_muladd_CF, gf8_muladd_D0, gf8_muladd_D1, + gf8_muladd_D2, gf8_muladd_D3, gf8_muladd_D4, gf8_muladd_D5, gf8_muladd_D6, + gf8_muladd_D7, gf8_muladd_D8, gf8_muladd_D9, gf8_muladd_DA, gf8_muladd_DB, + gf8_muladd_DC, gf8_muladd_DD, gf8_muladd_DE, gf8_muladd_DF, gf8_muladd_E0, + gf8_muladd_E1, gf8_muladd_E2, gf8_muladd_E3, gf8_muladd_E4, gf8_muladd_E5, + gf8_muladd_E6, gf8_muladd_E7, gf8_muladd_E8, gf8_muladd_E9, gf8_muladd_EA, + gf8_muladd_EB, gf8_muladd_EC, gf8_muladd_ED, gf8_muladd_EE, gf8_muladd_EF, + gf8_muladd_F0, gf8_muladd_F1, gf8_muladd_F2, gf8_muladd_F3, gf8_muladd_F4, + gf8_muladd_F5, gf8_muladd_F6, gf8_muladd_F7, gf8_muladd_F8, gf8_muladd_F9, + gf8_muladd_FA, gf8_muladd_FB, gf8_muladd_FC, gf8_muladd_FD, gf8_muladd_FE, + gf8_muladd_FF}; + +static uint64_t zero[EC_METHOD_WORD_SIZE * 8] = { + 0, +}; + +void +ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count) +{ + uint32_t i, last, tmp; + + last = 1; + for (i = count; i > 0; i--) { + if (values[i - 1] != 0) { + tmp = values[i - 1]; + values[i - 1] = ec_gf_div(gf, tmp, last); + last = tmp; + } + } +} + +void +ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values, + uint32_t count) +{ + src += offset; + gf8_muladd_00(dst, src); + while (--count > 0) { + src += EC_METHOD_CHUNK_SIZE; + gf8_muladd[*values](dst, src); + values++; + } +} + +void +ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values, + uint32_t count) +{ + uint32_t i, last, tmp; + + i = 0; + while ((last = *values++) == 0) { + i++; + } + gf8_muladd_00(dst, src[i++] + offset); + while (i < count) { + tmp = *values++; + if (tmp != 0) { + gf8_muladd[last](dst, src[i] + offset); + last = tmp; + } + i++; + } + gf8_muladd[last](dst, zero); +} diff --git a/xlators/cluster/ec/src/ec-code-c.h b/xlators/cluster/ec/src/ec-code-c.h new file mode 100644 index 00000000000..42b5a064eb8 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-c.h @@ -0,0 +1,27 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_C_H__ +#define __EC_CODE_C_H__ + +#include "ec-types.h" + +void +ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count); + +void +ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values, + uint32_t count); + +void +ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values, + uint32_t count); + +#endif /* __EC_CODE_C_H__ */ diff --git a/xlators/cluster/ec/src/ec-code-intel.c b/xlators/cluster/ec/src/ec-code-intel.c new file mode 100644 index 00000000000..f1c4e13e321 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-intel.c @@ -0,0 +1,594 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <inttypes.h> +#include <string.h> +#include <errno.h> + +#include "ec-code-intel.h" + +static void +ec_code_intel_init(ec_code_intel_t *intel) +{ + memset(intel, 0, sizeof(ec_code_intel_t)); +} + +static void +ec_code_intel_prefix(ec_code_intel_t *intel, uint8_t prefix) +{ + intel->prefix.data[intel->prefix.bytes++] = prefix; +} + +static void +ec_code_intel_rex(ec_code_intel_t *intel, gf_boolean_t w) +{ + gf_boolean_t present = _gf_false; + + if (w) { + intel->rex.w = 1; + present = _gf_true; + } + if (intel->modrm.present) { + if (intel->modrm.reg > 7) { + intel->modrm.reg &= 7; + intel->rex.r = 1; + present = _gf_true; + } + if (intel->sib.present) { + if (intel->sib.index > 7) { + intel->sib.index &= 7; + intel->rex.x = 1; + present = _gf_true; + } + if (intel->sib.base > 7) { + intel->sib.base &= 7; + intel->rex.b = 1; + present = _gf_true; + } + } else if (intel->modrm.rm > 7) { + intel->modrm.rm &= 7; + intel->rex.b = 1; + present = _gf_true; + } + } else if (intel->reg > 7) { + intel->reg &= 7; + intel->rex.b = 1; + present = _gf_true; + } + intel->rex.present = present; +} + +static void +ec_code_intel_vex(ec_code_intel_t *intel, gf_boolean_t w, gf_boolean_t l, + ec_code_vex_opcode_t opcode, ec_code_vex_prefix_t prefix, + uint32_t reg) +{ + ec_code_intel_rex(intel, w); + if (((intel->rex.w == 1) || (intel->rex.x == 0) || (intel->rex.b == 0)) || + ((opcode != VEX_OPCODE_NONE) && (opcode != VEX_OPCODE_0F))) { + intel->rex.present = _gf_false; + + intel->vex.bytes = 3; + intel->vex.data[0] = 0xC4; + intel->vex.data[1] = ((intel->rex.r << 7) | (intel->rex.x << 6) | + (intel->rex.b << 5) | opcode) ^ + 0xE0; + intel->vex.data[2] = (intel->rex.w << 7) | ((~reg & 0x0F) << 3) | + (l ? 0x04 : 0x00) | prefix; + } else { + intel->vex.bytes = 2; + intel->vex.data[0] = 0xC5; + intel->vex.data[1] = (intel->rex.r << 7) | ((~reg & 0x0F) << 3) | + (l ? 0x04 : 0x00) | prefix; + } +} + +static void +ec_code_intel_modrm_reg(ec_code_intel_t *intel, uint32_t rm, uint32_t reg) +{ + intel->modrm.present = _gf_true; + intel->modrm.mod = 3; + intel->modrm.rm = rm; + intel->modrm.reg = reg; +} + +static void +ec_code_intel_modrm_mem(ec_code_intel_t *intel, uint32_t reg, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset) +{ + if (index == REG_SP) { + intel->invalid = _gf_true; + return; + } + if ((index != REG_NULL) && (scale != 1) && (scale != 2) && (scale != 4) && + (scale != 8)) { + intel->invalid = _gf_true; + return; + } + scale >>= 1; + if (scale == 4) { + scale = 3; + } + + intel->modrm.present = _gf_true; + intel->modrm.reg = reg; + + intel->offset.value = offset; + if ((offset == 0) && (base != REG_BP)) { + intel->modrm.mod = 0; + intel->offset.bytes = 0; + } else if ((offset >= -128) && (offset <= 127)) { + intel->modrm.mod = 1; + intel->offset.bytes = 1; + } else { + intel->modrm.mod = 2; + intel->offset.bytes = 4; + } + + intel->modrm.rm = base; + if ((index != REG_NULL) || (base == REG_SP)) { + intel->modrm.rm = 4; + intel->sib.present = _gf_true; + intel->sib.index = index; + if (index == REG_NULL) { + intel->sib.index = 4; + } + intel->sib.scale = scale; + intel->sib.base = base; + if (base == REG_NULL) { + intel->sib.base = 5; + intel->modrm.mod = 0; + intel->offset.bytes = 4; + } + } else if (base == REG_NULL) { + intel->modrm.mod = 0; + intel->modrm.rm = 5; + intel->offset.bytes = 4; + } +} + +static void +ec_code_intel_op_1(ec_code_intel_t *intel, uint8_t opcode, uint32_t reg) +{ + intel->reg = reg; + intel->opcode.bytes = 1; + intel->opcode.data[0] = opcode; +} + +static void +ec_code_intel_op_2(ec_code_intel_t *intel, uint8_t opcode1, uint8_t opcode2, + uint32_t reg) +{ + intel->reg = reg; + intel->opcode.bytes = 2; + intel->opcode.data[0] = opcode1; + intel->opcode.data[1] = opcode2; +} + +static void +ec_code_intel_immediate_1(ec_code_intel_t *intel, uint32_t value) +{ + intel->immediate.bytes = 1; + intel->immediate.value = value; +} + +static void +ec_code_intel_immediate_2(ec_code_intel_t *intel, uint32_t value) +{ + intel->immediate.bytes = 2; + intel->immediate.value = value; +} + +static void +ec_code_intel_immediate_4(ec_code_intel_t *intel, uint32_t value) +{ + intel->immediate.bytes = 4; + intel->immediate.value = value; +} + +static void +ec_code_intel_emit(ec_code_builder_t *builder, ec_code_intel_t *intel) +{ + uint8_t insn[15]; + uint32_t i, count; + + if (intel->invalid) { + ec_code_error(builder, EINVAL); + return; + } + + count = 0; + for (i = 0; i < intel->prefix.bytes; i++) { + insn[count++] = intel->prefix.data[i]; + } + for (i = 0; i < intel->vex.bytes; i++) { + insn[count++] = intel->vex.data[i]; + } + if (intel->rex.present) { + insn[count++] = 0x40 | (intel->rex.w << 3) | (intel->rex.r << 2) | + (intel->rex.x << 1) | (intel->rex.b << 0); + } + for (i = 0; i < intel->opcode.bytes; i++) { + insn[count++] = intel->opcode.data[i]; + } + if (intel->modrm.present) { + insn[count++] = (intel->modrm.mod << 6) | (intel->modrm.reg << 3) | + (intel->modrm.rm << 0); + if (intel->sib.present) { + insn[count++] = (intel->sib.scale << 6) | (intel->sib.index << 3) | + (intel->sib.base << 0); + } + } + for (i = 0; i < intel->offset.bytes; i++) { + insn[count++] = intel->offset.data[i]; + } + for (i = 0; i < intel->immediate.bytes; i++) { + insn[count++] = intel->immediate.data[i]; + } + + ec_code_emit(builder, insn, count); +} + +void +ec_code_intel_op_push_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_op_1(&intel, 0x50 | (reg & 7), reg); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_pop_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_op_1(&intel, 0x58 | (reg & 7), reg); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + if (size == 0) { + ec_code_intel_op_1(&intel, 0xC3, 0); + } else { + ec_code_intel_immediate_2(&intel, size); + ec_code_intel_op_1(&intel, 0xC2, 0); + } + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_reg(&intel, dst, src); + ec_code_intel_op_1(&intel, 0x89, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_r2m(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x89, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, ec_code_intel_reg_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x8B, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_reg(&intel, dst, src); + ec_code_intel_op_1(&intel, 0x31, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, ec_code_intel_reg_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x33, 0); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value, + ec_code_intel_reg_t reg) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + if ((value >= -128) && (value < 128)) { + ec_code_intel_modrm_reg(&intel, reg, 0); + ec_code_intel_op_1(&intel, 0x83, 0); + ec_code_intel_immediate_1(&intel, value); + } else { + if (reg == REG_AX) { + ec_code_intel_op_1(&intel, 0x05, reg); + } else { + ec_code_intel_modrm_reg(&intel, reg, 0); + ec_code_intel_op_1(&intel, 0x81, 0); + } + ec_code_intel_immediate_4(&intel, value); + } + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value, + ec_code_intel_reg_t reg) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + if (reg == REG_AX) { + ec_code_intel_op_1(&intel, 0xA9, reg); + } else { + ec_code_intel_modrm_reg(&intel, reg, 0); + ec_code_intel_op_1(&intel, 0xF7, 0); + } + ec_code_intel_immediate_4(&intel, value); + ec_code_intel_rex(&intel, _gf_true); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address) +{ + ec_code_intel_t intel; + int32_t rel; + + ec_code_intel_init(&intel); + + rel = address - builder->address - 2; + if ((rel >= -128) && (rel < 128)) { + ec_code_intel_op_1(&intel, 0x75, 0); + ec_code_intel_immediate_1(&intel, rel); + } else { + rel -= 4; + ec_code_intel_op_2(&intel, 0x0F, 0x85, 0); + ec_code_intel_immediate_4(&intel, rel); + } + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src, + uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_reg(&intel, src, dst); + ec_code_intel_op_2(&intel, 0x0F, 0x6F, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset); + ec_code_intel_op_2(&intel, 0x0F, 0x7F, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_2(&intel, 0x0F, 0x6F, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src, + uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_reg(&intel, src, dst); + ec_code_intel_op_2(&intel, 0x0F, 0xEF, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_prefix(&intel, 0x66); + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_2(&intel, 0x0F, 0xEF, 0); + ec_code_intel_rex(&intel, _gf_false); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src, + uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_reg(&intel, src, dst); + ec_code_intel_op_1(&intel, 0x6F, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + VEX_REG_NONE); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x7F, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + VEX_REG_NONE); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0x6F, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + VEX_REG_NONE); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src, + uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_reg(&intel, src, dst); + ec_code_intel_op_1(&intel, 0xEF, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + dst); + + ec_code_intel_emit(builder, &intel); +} + +void +ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst) +{ + ec_code_intel_t intel; + + ec_code_intel_init(&intel); + + ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset); + ec_code_intel_op_1(&intel, 0xEF, 0); + ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66, + dst); + + ec_code_intel_emit(builder, &intel); +} diff --git a/xlators/cluster/ec/src/ec-code-intel.h b/xlators/cluster/ec/src/ec-code-intel.h new file mode 100644 index 00000000000..3fa4a174765 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-intel.h @@ -0,0 +1,191 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_INTEL_H__ +#define __EC_CODE_INTEL_H__ + +#include "ec-code.h" + +#define VEX_REG_NONE 0 + +enum _ec_code_intel_reg; +typedef enum _ec_code_intel_reg ec_code_intel_reg_t; + +enum _ec_code_vex_prefix; +typedef enum _ec_code_vex_prefix ec_code_vex_prefix_t; + +enum _ec_code_vex_opcode; +typedef enum _ec_code_vex_opcode ec_code_vex_opcode_t; + +struct _ec_code_intel_buffer; +typedef struct _ec_code_intel_buffer ec_code_intel_buffer_t; + +struct _ec_code_intel_sib; +typedef struct _ec_code_intel_sib ec_code_intel_sib_t; + +struct _ec_code_intel_modrm; +typedef struct _ec_code_intel_modrm ec_code_intel_modrm_t; + +struct _ec_code_intel_rex; +typedef struct _ec_code_intel_rex ec_code_intel_rex_t; + +struct _ec_code_intel; +typedef struct _ec_code_intel ec_code_intel_t; + +enum _ec_code_intel_reg { + REG_NULL = -1, + REG_AX, + REG_CX, + REG_DX, + REG_BX, + REG_SP, + REG_BP, + REG_SI, + REG_DI, + REG_8, + REG_9, + REG_10, + REG_11, + REG_12, + REG_13, + REG_14, + REG_15 +}; + +enum _ec_code_vex_prefix { + VEX_PREFIX_NONE = 0, + VEX_PREFIX_66, + VEX_PREFIX_F3, + VEX_PREFIX_F2 +}; + +enum _ec_code_vex_opcode { + VEX_OPCODE_NONE = 0, + VEX_OPCODE_0F, + VEX_OPCODE_0F_38, + VEX_OPCODE_0F_3A +}; + +struct _ec_code_intel_buffer { + uint32_t bytes; + union { + uint8_t data[4]; + uint32_t value; + }; +}; + +struct _ec_code_intel_sib { + gf_boolean_t present; + uint32_t base; + uint32_t index; + uint32_t scale; +}; + +struct _ec_code_intel_modrm { + gf_boolean_t present; + uint32_t mod; + uint32_t rm; + uint32_t reg; +}; + +struct _ec_code_intel_rex { + gf_boolean_t present; + uint32_t w; + uint32_t r; + uint32_t x; + uint32_t b; +}; + +struct _ec_code_intel { + gf_boolean_t invalid; + ec_code_intel_buffer_t prefix; + ec_code_intel_buffer_t opcode; + ec_code_intel_buffer_t offset; + ec_code_intel_buffer_t immediate; + ec_code_intel_buffer_t vex; + ec_code_intel_rex_t rex; + ec_code_intel_modrm_t modrm; + ec_code_intel_sib_t sib; + uint32_t reg; +}; + +void +ec_code_intel_op_push_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg); +void +ec_code_intel_op_pop_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg); +void +ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size); + +void +ec_code_intel_op_mov_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t dst); +void +ec_code_intel_op_mov_r2m(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset); +void +ec_code_intel_op_mov_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, ec_code_intel_reg_t dst); +void +ec_code_intel_op_xor_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src, + ec_code_intel_reg_t dst); +void +ec_code_intel_op_xor_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, ec_code_intel_reg_t dst); +void +ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value, + ec_code_intel_reg_t reg); +void +ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value, + ec_code_intel_reg_t reg); +void +ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address); + +void +ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src, + uint32_t dst); +void +ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset); +void +ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst); +void +ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src, + uint32_t dst); +void +ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst); + +void +ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src, + uint32_t dst); +void +ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src, + ec_code_intel_reg_t base, ec_code_intel_reg_t index, + uint32_t scale, int32_t offset); +void +ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst); +void +ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src, + uint32_t dst); +void +ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base, + ec_code_intel_reg_t index, uint32_t scale, + int32_t offset, uint32_t dst); + +#endif /* __EC_CODE_INTEL_H__ */ diff --git a/xlators/cluster/ec/src/ec-code-sse.c b/xlators/cluster/ec/src/ec-code-sse.c new file mode 100644 index 00000000000..e11e7ff8400 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-sse.c @@ -0,0 +1,101 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <errno.h> + +#include "ec-code-intel.h" + +static void +ec_code_sse_prolog(ec_code_builder_t *builder) +{ + builder->loop = builder->address; +} + +static void +ec_code_sse_epilog(ec_code_builder_t *builder) +{ + ec_code_intel_op_add_i2r(builder, 16, REG_DX); + ec_code_intel_op_add_i2r(builder, 16, REG_DI); + ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX); + ec_code_intel_op_jne(builder, builder->loop); + + ec_code_intel_op_ret(builder, 0); +} + +static void +ec_code_sse_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + if (builder->linear) { + ec_code_intel_op_mov_m2sse( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_AX); + builder->base = idx; + } + ec_code_intel_op_mov_m2sse(builder, REG_AX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static void +ec_code_sse_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit) +{ + ec_code_intel_op_mov_sse2m(builder, src, REG_DI, REG_NULL, 0, + bit * builder->width); +} + +static void +ec_code_sse_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_intel_op_mov_sse2sse(builder, src, dst); +} + +static void +ec_code_sse_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_intel_op_xor_sse2sse(builder, src, dst); +} + +static void +ec_code_sse_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + if (builder->linear) { + ec_code_intel_op_xor_m2sse( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_AX); + builder->base = idx; + } + ec_code_intel_op_xor_m2sse(builder, REG_AX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static char *ec_code_sse_needed_flags[] = {"sse2", NULL}; + +ec_code_gen_t ec_code_gen_sse = {.name = "sse", + .flags = ec_code_sse_needed_flags, + .width = 16, + .prolog = ec_code_sse_prolog, + .epilog = ec_code_sse_epilog, + .load = ec_code_sse_load, + .store = ec_code_sse_store, + .copy = ec_code_sse_copy, + .xor2 = ec_code_sse_xor2, + .xor3 = NULL, + .xorm = ec_code_sse_xorm}; diff --git a/xlators/cluster/ec/src/ec-code-sse.h b/xlators/cluster/ec/src/ec-code-sse.h new file mode 100644 index 00000000000..f1acbcf894b --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-sse.h @@ -0,0 +1,18 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_SSE_H__ +#define __EC_CODE_SSE_H__ + +#include "ec-code.h" + +extern ec_code_gen_t ec_code_gen_sse; + +#endif /* __EC_CODE_SSE_H__ */ diff --git a/xlators/cluster/ec/src/ec-code-x64.c b/xlators/cluster/ec/src/ec-code-x64.c new file mode 100644 index 00000000000..26565b4493f --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-x64.c @@ -0,0 +1,144 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <errno.h> + +#include "ec-code-intel.h" + +static ec_code_intel_reg_t ec_code_x64_regmap[] = { + REG_AX, REG_CX, REG_BP, REG_8, REG_9, REG_10, + REG_11, REG_12, REG_13, REG_14, REG_15}; + +static void +ec_code_x64_prolog(ec_code_builder_t *builder) +{ + uint32_t i; + + ec_code_intel_op_push_r(builder, REG_BP); + if (!builder->linear) { + ec_code_intel_op_push_r(builder, REG_BX); + } + if (builder->regs > 11) { + ec_code_error(builder, EINVAL); + return; + } + for (i = 7; i < builder->regs; i++) { + ec_code_intel_op_push_r(builder, ec_code_x64_regmap[i]); + } + + builder->loop = builder->address; +} + +static void +ec_code_x64_epilog(ec_code_builder_t *builder) +{ + uint32_t i; + + ec_code_intel_op_add_i2r(builder, 8, REG_DX); + ec_code_intel_op_add_i2r(builder, 8, REG_DI); + ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX); + ec_code_intel_op_jne(builder, builder->loop); + + if (builder->regs > 11) { + ec_code_error(builder, EINVAL); + return; + } + for (i = builder->regs; i > 7; i--) { + ec_code_intel_op_pop_r(builder, ec_code_x64_regmap[i - 1]); + } + if (!builder->linear) { + ec_code_intel_op_pop_r(builder, REG_BX); + } + ec_code_intel_op_pop_r(builder, REG_BP); + ec_code_intel_op_ret(builder, 0); +} + +static void +ec_code_x64_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + dst = ec_code_x64_regmap[dst]; + + if (builder->linear) { + ec_code_intel_op_mov_m2r( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_BX); + builder->base = idx; + } + ec_code_intel_op_mov_m2r(builder, REG_BX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static void +ec_code_x64_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit) +{ + src = ec_code_x64_regmap[src]; + + ec_code_intel_op_mov_r2m(builder, src, REG_DI, REG_NULL, 0, + bit * builder->width); +} + +static void +ec_code_x64_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + dst = ec_code_x64_regmap[dst]; + src = ec_code_x64_regmap[src]; + + ec_code_intel_op_mov_r2r(builder, src, dst); +} + +static void +ec_code_x64_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + dst = ec_code_x64_regmap[dst]; + src = ec_code_x64_regmap[src]; + + ec_code_intel_op_xor_r2r(builder, src, dst); +} + +static void +ec_code_x64_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx, + uint32_t bit) +{ + dst = ec_code_x64_regmap[dst]; + + if (builder->linear) { + ec_code_intel_op_xor_m2r( + builder, REG_SI, REG_DX, 1, + idx * builder->width * builder->bits + bit * builder->width, dst); + } else { + if (builder->base != idx) { + ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8, + REG_BX); + builder->base = idx; + } + ec_code_intel_op_xor_m2r(builder, REG_BX, REG_DX, 1, + bit * builder->width, dst); + } +} + +static char *ec_code_x64_needed_flags[] = {NULL}; + +ec_code_gen_t ec_code_gen_x64 = {.name = "x64", + .flags = ec_code_x64_needed_flags, + .width = sizeof(uint64_t), + .prolog = ec_code_x64_prolog, + .epilog = ec_code_x64_epilog, + .load = ec_code_x64_load, + .store = ec_code_x64_store, + .copy = ec_code_x64_copy, + .xor2 = ec_code_x64_xor2, + .xor3 = NULL, + .xorm = ec_code_x64_xorm}; diff --git a/xlators/cluster/ec/src/ec-code-x64.h b/xlators/cluster/ec/src/ec-code-x64.h new file mode 100644 index 00000000000..bd8174e4bf5 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code-x64.h @@ -0,0 +1,18 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_X64_H__ +#define __EC_CODE_X64_H__ + +#include "ec-code.h" + +extern ec_code_gen_t ec_code_gen_x64; + +#endif /* __EC_CODE_X64_H__ */ diff --git a/xlators/cluster/ec/src/ec-code.c b/xlators/cluster/ec/src/ec-code.c new file mode 100644 index 00000000000..03162ae05a9 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code.c @@ -0,0 +1,1060 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <string.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <ctype.h> + +#include <glusterfs/syscall.h> + +#include "ec-mem-types.h" +#include "ec-code.h" +#include "ec-messages.h" +#include "ec-code-c.h" +#include "ec-helpers.h" + +#ifdef USE_EC_DYNAMIC_X64 +#include "ec-code-x64.h" +#endif + +#ifdef USE_EC_DYNAMIC_SSE +#include "ec-code-sse.h" +#endif + +#ifdef USE_EC_DYNAMIC_AVX +#include "ec-code-avx.h" +#endif + +#define EC_CODE_SIZE (1024 * 64) +#define EC_CODE_ALIGN 4096 + +#define EC_CODE_CHUNK_MIN_SIZE 512 + +#define EC_PROC_BUFFER_SIZE 4096 + +#define PROC_CPUINFO "/proc/cpuinfo" + +struct _ec_code_proc; +typedef struct _ec_code_proc ec_code_proc_t; + +struct _ec_code_proc { + int32_t fd; + gf_boolean_t eof; + gf_boolean_t error; + gf_boolean_t skip; + ssize_t size; + ssize_t pos; + char buffer[EC_PROC_BUFFER_SIZE]; +}; + +static ec_code_gen_t *ec_code_gen_table[] = { +#ifdef USE_EC_DYNAMIC_AVX + &ec_code_gen_avx, +#endif +#ifdef USE_EC_DYNAMIC_SSE + &ec_code_gen_sse, +#endif +#ifdef USE_EC_DYNAMIC_X64 + &ec_code_gen_x64, +#endif + NULL}; + +static void +ec_code_arg_set(ec_code_arg_t *arg, uint32_t value) +{ + arg->value = value; +} + +static void +ec_code_arg_assign(ec_code_builder_t *builder, ec_code_op_t *op, + ec_code_arg_t *arg, uint32_t reg) +{ + arg->value = reg; + + if (builder->regs <= reg) { + builder->regs = reg + 1; + } +} + +static void +ec_code_arg_use(ec_code_builder_t *builder, ec_code_op_t *op, + ec_code_arg_t *arg, uint32_t reg) +{ + arg->value = reg; +} + +static void +ec_code_arg_update(ec_code_builder_t *builder, ec_code_op_t *op, + ec_code_arg_t *arg, uint32_t reg) +{ + arg->value = reg; +} + +static ec_code_op_t * +ec_code_op_next(ec_code_builder_t *builder) +{ + ec_code_op_t *op; + + op = &builder->ops[builder->count++]; + memset(op, 0, sizeof(ec_code_op_t)); + + return op; +} + +static void +ec_code_load(ec_code_builder_t *builder, uint32_t bit, uint32_t offset) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_LOAD; + ec_code_arg_assign(builder, op, &op->arg1, builder->map[bit]); + ec_code_arg_set(&op->arg2, offset); + ec_code_arg_set(&op->arg3, bit); +} + +static void +ec_code_store(ec_code_builder_t *builder, uint32_t reg, uint32_t bit) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_STORE; + ec_code_arg_use(builder, op, &op->arg1, builder->map[reg]); + ec_code_arg_set(&op->arg2, 0); + ec_code_arg_set(&op->arg3, bit); +} + +static void +ec_code_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_COPY; + ec_code_arg_assign(builder, op, &op->arg1, builder->map[dst]); + ec_code_arg_use(builder, op, &op->arg2, builder->map[src]); + ec_code_arg_set(&op->arg3, 0); +} + +static void +ec_code_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_XOR2; + ec_code_arg_update(builder, op, &op->arg1, builder->map[dst]); + ec_code_arg_use(builder, op, &op->arg2, builder->map[src]); + ec_code_arg_set(&op->arg3, 0); +} + +static void +ec_code_xor3(ec_code_builder_t *builder, uint32_t dst, uint32_t src1, + uint32_t src2) +{ + ec_code_op_t *op; + + if (builder->code->gen->xor3 == NULL) { + ec_code_copy(builder, dst, src1); + ec_code_xor2(builder, dst, src2); + + return; + } + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_XOR3; + ec_code_arg_assign(builder, op, &op->arg1, builder->map[dst]); + ec_code_arg_use(builder, op, &op->arg2, builder->map[src1]); + ec_code_arg_use(builder, op, &op->arg3, builder->map[src2]); +} + +static void +ec_code_xorm(ec_code_builder_t *builder, uint32_t bit, uint32_t offset) +{ + ec_code_op_t *op; + + op = ec_code_op_next(builder); + + op->op = EC_GF_OP_XORM; + ec_code_arg_update(builder, op, &op->arg1, builder->map[bit]); + ec_code_arg_set(&op->arg2, offset); + ec_code_arg_set(&op->arg3, bit); +} + +static void +ec_code_dup(ec_code_builder_t *builder, ec_gf_op_t *op) +{ + switch (op->op) { + case EC_GF_OP_COPY: + ec_code_copy(builder, op->arg1, op->arg2); + break; + case EC_GF_OP_XOR2: + ec_code_xor2(builder, op->arg1, op->arg2); + break; + case EC_GF_OP_XOR3: + ec_code_xor3(builder, op->arg1, op->arg2, op->arg3); + break; + default: + break; + } +} + +static void +ec_code_gf_load(ec_code_builder_t *builder, uint32_t offset) +{ + uint32_t i; + + for (i = 0; i < builder->code->gf->bits; i++) { + ec_code_load(builder, i, offset); + } +} + +static void +ec_code_gf_load_xor(ec_code_builder_t *builder, uint32_t offset) +{ + uint32_t i; + + for (i = 0; i < builder->code->gf->bits; i++) { + ec_code_xorm(builder, i, offset); + } +} + +static void +ec_code_gf_store(ec_code_builder_t *builder) +{ + uint32_t i; + + for (i = 0; i < builder->code->gf->bits; i++) { + ec_code_store(builder, i, i); + } +} + +static void +ec_code_gf_clear(ec_code_builder_t *builder) +{ + uint32_t i; + + ec_code_xor2(builder, 0, 0); + for (i = 0; i < builder->code->gf->bits; i++) { + ec_code_store(builder, 0, i); + } +} + +static void +ec_code_gf_mul(ec_code_builder_t *builder, uint32_t value) +{ + ec_gf_mul_t *mul; + ec_gf_op_t *op; + uint32_t map[EC_GF_MAX_REGS]; + int32_t i; + + mul = builder->code->gf->table[value]; + for (op = mul->ops; op->op != EC_GF_OP_END; op++) { + ec_code_dup(builder, op); + } + + for (i = 0; i < mul->regs; i++) { + map[i] = builder->map[mul->map[i]]; + } + memcpy(builder->map, map, sizeof(uint32_t) * mul->regs); +} + +static ec_code_builder_t * +ec_code_prepare(ec_code_t *code, uint32_t count, uint32_t width, + gf_boolean_t linear) +{ + ec_code_builder_t *builder; + uint32_t i; + + count *= code->gf->bits + code->gf->max_ops; + count += code->gf->bits; + builder = GF_MALLOC( + sizeof(ec_code_builder_t) + sizeof(ec_code_op_t) * count, + ec_mt_ec_code_builder_t); + if (builder == NULL) { + return EC_ERR(ENOMEM); + } + + builder->address = 0; + builder->code = code; + builder->size = 0; + builder->count = 0; + builder->regs = 0; + builder->error = 0; + builder->bits = code->gf->bits; + builder->width = width; + builder->data = NULL; + builder->linear = linear; + builder->base = -1; + + for (i = 0; i < EC_GF_MAX_REGS; i++) { + builder->map[i] = i; + } + + return builder; +} + +static size_t +ec_code_space_size(void) +{ + return (sizeof(ec_code_space_t) + 15) & ~15; +} + +static size_t +ec_code_chunk_size(void) +{ + return (sizeof(ec_code_chunk_t) + 15) & ~15; +} + +static ec_code_chunk_t * +ec_code_chunk_from_space(ec_code_space_t *space) +{ + return (ec_code_chunk_t *)((uintptr_t)space + ec_code_space_size()); +} + +static void * +ec_code_to_executable(ec_code_space_t *space, void *addr) +{ + return (void *)((uintptr_t)addr - (uintptr_t)space + + (uintptr_t)space->exec); +} + +static void * +ec_code_from_executable(ec_code_space_t *space, void *addr) +{ + return (void *)((uintptr_t)addr - (uintptr_t)space->exec + + (uintptr_t)space); +} + +static void * +ec_code_func_from_chunk(ec_code_chunk_t *chunk, void **exec) +{ + void *addr; + + addr = (void *)((uintptr_t)chunk + ec_code_chunk_size()); + + *exec = ec_code_to_executable(chunk->space, addr); + + return addr; +} + +static ec_code_chunk_t * +ec_code_chunk_from_func(ec_code_func_linear_t func) +{ + ec_code_chunk_t *chunk; + + chunk = (ec_code_chunk_t *)((uintptr_t)func - ec_code_chunk_size()); + + return ec_code_from_executable(chunk->space, chunk); +} + +static ec_code_chunk_t * +ec_code_chunk_split(ec_code_chunk_t *chunk, size_t size) +{ + ec_code_chunk_t *extra; + ssize_t avail; + + avail = chunk->size - size - ec_code_chunk_size(); + if (avail > 0) { + extra = (ec_code_chunk_t *)((uintptr_t)chunk + chunk->size - avail); + extra->space = chunk->space; + extra->size = avail; + list_add(&extra->list, &chunk->list); + chunk->size = size; + } + list_del_init(&chunk->list); + + return chunk; +} + +static gf_boolean_t +ec_code_chunk_touch(ec_code_chunk_t *prev, ec_code_chunk_t *next) +{ + uintptr_t end; + + end = (uintptr_t)prev + ec_code_chunk_size() + prev->size; + return (end == (uintptr_t)next); +} + +static ec_code_space_t * +ec_code_space_create(ec_code_t *code, size_t size) +{ + char path[] = GLUSTERFS_LIBEXECDIR "/ec-code-dynamic.XXXXXX"; + ec_code_space_t *space; + void *exec; + int32_t fd, err; + + /* We need to create memory areas to store the generated dynamic code. + * Obviously these areas need to be written to be able to create the + * code and they also need to be executable to execute it. + * + * However it's a bad practice to have a memory region that is both + * writable *and* executable. In fact, selinux forbids this and causes + * attempts to do so to fail (unless specifically configured). + * + * To solve the problem we'll use two distinct memory areas mapped to + * the same physical storage. One of the memory areas will have write + * permission, and the other will have execute permission. Both areas + * will have the same contents. The physical storage will be a regular + * file that will be mmapped to both areas. + */ + + /* We need to create a temporary file as the backend storage for the + * memory mapped areas. */ + /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */ + fd = mkstemp(path); + if (fd < 0) { + err = errno; + gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED, + "Unable to create a temporary file for the ec dynamic " + "code"); + space = EC_ERR(err); + goto done; + } + /* Once created we don't need to keep it in the file system. It will + * still exist until we close the last file descriptor or unmap the + * memory areas bound to the file. */ + sys_unlink(path); + + size = (size + EC_CODE_ALIGN - 1) & ~(EC_CODE_ALIGN - 1); + if (sys_ftruncate(fd, size) < 0) { + err = errno; + gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED, + "Unable to resize the file for the ec dynamic code"); + space = EC_ERR(err); + goto done_close; + } + + /* This creates an executable memory area to be able to run the + * generated fragments of code. */ + exec = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0); + if (exec == MAP_FAILED) { + err = errno; + gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED, + "Unable to map the executable area for the ec dynamic " + "code"); + space = EC_ERR(err); + goto done_close; + } + /* It's not important to check the return value of mlock(). If it fails + * everything will continue to work normally. */ + mlock(exec, size); + + /* This maps a read/write memory area to be able to create the dynamici + * code. */ + space = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (space == MAP_FAILED) { + err = errno; + gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED, + "Unable to map the writable area for the ec dynamic " + "code"); + space = EC_ERR(err); + + munmap(exec, size); + + goto done_close; + } + + space->exec = exec; + space->size = size; + space->code = code; + list_add_tail(&space->list, &code->spaces); + INIT_LIST_HEAD(&space->chunks); + +done_close: + /* If everything has succeeded, we already have the memory areas + * mapped. We don't need the file descriptor anymore because the + * backend storage will be there until the mmap()'d regions are + * unmapped. */ + sys_close(fd); +done: + return space; +} + +static void +ec_code_space_destroy(ec_code_space_t *space) +{ + list_del_init(&space->list); + + munmap(space->exec, space->size); + munmap(space, space->size); +} + +static void +ec_code_chunk_merge(ec_code_chunk_t *chunk) +{ + ec_code_chunk_t *item, *tmp; + + list_for_each_entry_safe(item, tmp, &chunk->space->chunks, list) + { + if ((uintptr_t)item > (uintptr_t)chunk) { + list_add_tail(&chunk->list, &item->list); + if (ec_code_chunk_touch(chunk, item)) { + chunk->size += item->size + ec_code_chunk_size(); + list_del_init(&item->list); + } + + goto check; + } + if (ec_code_chunk_touch(item, chunk)) { + item->size += chunk->size + ec_code_chunk_size(); + list_del_init(&item->list); + chunk = item; + } + } + list_add_tail(&chunk->list, &chunk->space->chunks); + +check: + if (chunk->size == + chunk->space->size - ec_code_space_size() - ec_code_chunk_size()) { + ec_code_space_destroy(chunk->space); + } +} + +static ec_code_chunk_t * +ec_code_space_alloc(ec_code_t *code, size_t size) +{ + ec_code_space_t *space; + ec_code_chunk_t *chunk; + size_t map_size; + + /* To minimize fragmentation, we only allocate chunks of sizes multiples + * of EC_CODE_CHUNK_MIN_SIZE. */ + size = ((size + ec_code_chunk_size() + EC_CODE_CHUNK_MIN_SIZE - 1) & + ~(EC_CODE_CHUNK_MIN_SIZE - 1)) - + ec_code_chunk_size(); + list_for_each_entry(space, &code->spaces, list) + { + list_for_each_entry(chunk, &space->chunks, list) + { + if (chunk->size >= size) { + goto out; + } + } + } + + map_size = EC_CODE_SIZE - ec_code_space_size() - ec_code_chunk_size(); + if (map_size < size) { + map_size = size; + } + space = ec_code_space_create(code, map_size); + if (EC_IS_ERR(space)) { + return (ec_code_chunk_t *)space; + } + + chunk = ec_code_chunk_from_space(space); + chunk->size = map_size - ec_code_space_size() - ec_code_chunk_size(); + list_add(&chunk->list, &space->chunks); + +out: + chunk->space = space; + + return ec_code_chunk_split(chunk, size); +} + +static ec_code_chunk_t * +ec_code_alloc(ec_code_t *code, uint32_t size) +{ + ec_code_chunk_t *chunk; + + LOCK(&code->lock); + + chunk = ec_code_space_alloc(code, size); + + UNLOCK(&code->lock); + + return chunk; +} + +static void +ec_code_free(ec_code_chunk_t *chunk) +{ + gf_lock_t *lock; + + lock = &chunk->space->code->lock; + LOCK(lock); + + ec_code_chunk_merge(chunk); + + UNLOCK(lock); +} + +static int32_t +ec_code_write(ec_code_builder_t *builder) +{ + ec_code_gen_t *gen; + ec_code_op_t *op; + uint32_t i; + + builder->error = 0; + builder->size = 0; + builder->address = 0; + builder->base = -1; + + gen = builder->code->gen; + gen->prolog(builder); + for (i = 0; i < builder->count; i++) { + op = &builder->ops[i]; + switch (op->op) { + case EC_GF_OP_LOAD: + gen->load(builder, op->arg1.value, op->arg2.value, + op->arg3.value); + break; + case EC_GF_OP_STORE: + gen->store(builder, op->arg1.value, op->arg3.value); + break; + case EC_GF_OP_COPY: + gen->copy(builder, op->arg1.value, op->arg2.value); + break; + case EC_GF_OP_XOR2: + gen->xor2(builder, op->arg1.value, op->arg2.value); + break; + case EC_GF_OP_XOR3: + gen->xor3(builder, op->arg1.value, op->arg2.value, + op->arg3.value); + break; + case EC_GF_OP_XORM: + gen->xorm(builder, op->arg1.value, op->arg2.value, + op->arg3.value); + break; + default: + break; + } + } + gen->epilog(builder); + + return builder->error; +} + +static void * +ec_code_compile(ec_code_builder_t *builder) +{ + ec_code_chunk_t *chunk; + void *func; + int32_t err; + + err = ec_code_write(builder); + if (err != 0) { + return EC_ERR(err); + } + + chunk = ec_code_alloc(builder->code, builder->size); + if (EC_IS_ERR(chunk)) { + return chunk; + } + builder->data = ec_code_func_from_chunk(chunk, &func); + + err = ec_code_write(builder); + if (err != 0) { + ec_code_free(chunk); + + return EC_ERR(err); + } + + GF_FREE(builder); + + return func; +} + +ec_code_t * +ec_code_create(ec_gf_t *gf, ec_code_gen_t *gen) +{ + ec_code_t *code; + + code = GF_MALLOC(sizeof(ec_code_t), ec_mt_ec_code_t); + if (code == NULL) { + return EC_ERR(ENOMEM); + } + memset(code, 0, sizeof(ec_code_t)); + INIT_LIST_HEAD(&code->spaces); + LOCK_INIT(&code->lock); + + code->gf = gf; + code->gen = gen; + + return code; +} + +void +ec_code_destroy(ec_code_t *code) +{ + if (!list_empty(&code->spaces)) { + } + + LOCK_DESTROY(&code->lock); + + GF_FREE(code); +} + +static uint32_t +ec_code_value_next(uint32_t *values, uint32_t count, uint32_t *offset) +{ + uint32_t i, next; + + next = 0; + for (i = *offset + 1; i < count; i++) { + next = values[i]; + if (next != 0) { + break; + } + } + *offset = i; + + return next; +} + +static void * +ec_code_build_dynamic(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count, gf_boolean_t linear) +{ + ec_code_builder_t *builder; + uint32_t offset, val, next; + + builder = ec_code_prepare(code, count, width, linear); + if (EC_IS_ERR(builder)) { + return builder; + } + + offset = -1; + next = ec_code_value_next(values, count, &offset); + if (next != 0) { + ec_code_gf_load(builder, offset); + do { + val = next; + next = ec_code_value_next(values, count, &offset); + if (next != 0) { + ec_code_gf_mul(builder, ec_gf_div(code->gf, val, next)); + ec_code_gf_load_xor(builder, offset); + } + } while (next != 0); + ec_code_gf_mul(builder, val); + ec_code_gf_store(builder); + } else { + ec_code_gf_clear(builder); + } + + return ec_code_compile(builder); +} + +static void * +ec_code_build(ec_code_t *code, uint32_t width, uint32_t *values, uint32_t count, + gf_boolean_t linear) +{ + void *func; + + if (code->gen != NULL) { + func = ec_code_build_dynamic(code, width, values, count, linear); + if (!EC_IS_ERR(func)) { + return func; + } + + gf_msg_debug(THIS->name, GF_LOG_DEBUG, + "Unable to generate dynamic code. Falling back " + "to precompiled code"); + + /* The dynamic code generation shouldn't fail in normal + * conditions, but if it fails at some point, it's very + * probable that it will fail again, so we completely disable + * dynamic code generation. */ + code->gen = NULL; + } + + ec_code_c_prepare(code->gf, values, count); + + if (linear) { + return ec_code_c_linear; + } + + return ec_code_c_interleaved; +} + +ec_code_func_linear_t +ec_code_build_linear(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count) +{ + return (ec_code_func_linear_t)ec_code_build(code, width, values, count, + _gf_true); +} + +ec_code_func_interleaved_t +ec_code_build_interleaved(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count) +{ + return (ec_code_func_interleaved_t)ec_code_build(code, width, values, count, + _gf_false); +} + +void +ec_code_release(ec_code_t *code, ec_code_func_t *func) +{ + if ((func->linear != ec_code_c_linear) && + (func->interleaved != ec_code_c_interleaved)) { + ec_code_free(ec_code_chunk_from_func(func->linear)); + } +} + +void +ec_code_error(ec_code_builder_t *builder, int32_t error) +{ + if (builder->error == 0) { + gf_msg(THIS->name, GF_LOG_ERROR, error, EC_MSG_DYN_CODEGEN_FAILED, + "Failed to generate dynamic code"); + builder->error = error; + } +} + +void +ec_code_emit(ec_code_builder_t *builder, uint8_t *bytes, uint32_t count) +{ + if (builder->error != 0) { + return; + } + + if (builder->data != NULL) { + memcpy(builder->data + builder->size, bytes, count); + } + + builder->size += count; + builder->address += count; +} + +static char * +ec_code_proc_trim_left(char *text, ssize_t *length) +{ + ssize_t len; + + for (len = *length; (len > 0) && isspace(*text); len--) { + text++; + } + *length = len; + + return text; +} + +static char * +ec_code_proc_trim_right(char *text, ssize_t *length, char sep) +{ + char *last; + ssize_t len; + + len = *length; + + last = text; + for (len = *length; (len > 0) && (*text != sep); len--) { + if (!isspace(*text)) { + last = text + 1; + } + text++; + } + *last = 0; + *length = len; + + return text; +} + +static char * +ec_code_proc_line_parse(ec_code_proc_t *file, ssize_t *length) +{ + char *text, *end; + ssize_t len; + + len = file->size - file->pos; + text = ec_code_proc_trim_left(file->buffer + file->pos, &len); + end = ec_code_proc_trim_right(text, &len, '\n'); + if (len == 0) { + if (!file->eof) { + if (text == file->buffer) { + file->size = file->pos = 0; + file->skip = _gf_true; + } else { + file->size = file->pos = end - text; + memmove(file->buffer, text, file->pos + 1); + } + len = sys_read(file->fd, file->buffer + file->pos, + sizeof(file->buffer) - file->pos - 1); + if (len > 0) { + file->size += len; + } + file->error = len < 0; + file->eof = len <= 0; + + return NULL; + } + file->size = file->pos = 0; + } else { + file->pos = end - file->buffer + 1; + } + + *length = end - text; + + if (file->skip) { + file->skip = _gf_false; + text = NULL; + } + + return text; +} + +static char * +ec_code_proc_line(ec_code_proc_t *file, ssize_t *length) +{ + char *text; + + text = NULL; + while (!file->eof) { + text = ec_code_proc_line_parse(file, length); + if (text != NULL) { + break; + } + } + + return text; +} + +static char * +ec_code_proc_split(char *text, ssize_t *length, char sep) +{ + text = ec_code_proc_trim_right(text, length, sep); + if (*length == 0) { + return NULL; + } + (*length)--; + text++; + + return ec_code_proc_trim_left(text, length); +} + +static uint32_t +ec_code_cpu_check(uint32_t idx, char *list, uint32_t count) +{ + ec_code_gen_t *gen; + char **ptr; + char *table[count + 1]; + uint32_t i; + + for (i = 0; i < count; i++) { + table[i] = list; + list += strlen(list) + 1; + } + + gen = ec_code_gen_table[idx]; + while (gen != NULL) { + for (ptr = gen->flags; *ptr != NULL; ptr++) { + for (i = 0; i < count; i++) { + if (strcmp(*ptr, table[i]) == 0) { + break; + } + } + if (i >= count) { + gen = ec_code_gen_table[++idx]; + break; + } + } + if (*ptr == NULL) { + break; + } + } + + return idx; +} + +ec_code_gen_t * +ec_code_detect(xlator_t *xl, const char *def) +{ + ec_code_proc_t file; + ec_code_gen_t *gen = NULL; + char *line, *data, *list; + ssize_t length; + uint32_t count, base, select; + + if (strcmp(def, "none") == 0) { + gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION_NONE, + "Not using any cpu extensions"); + + return NULL; + } + + file.fd = sys_open(PROC_CPUINFO, O_RDONLY, 0); + if (file.fd < 0) { + goto out; + } + file.size = file.pos = 0; + file.eof = file.error = file.skip = _gf_false; + + select = 0; + if (strcmp(def, "auto") != 0) { + while (ec_code_gen_table[select] != NULL) { + if (strcmp(ec_code_gen_table[select]->name, def) == 0) { + break; + } + select++; + } + if (ec_code_gen_table[select] == NULL) { + gf_msg(xl->name, GF_LOG_WARNING, EINVAL, EC_MSG_EXTENSION_UNKNOWN, + "CPU extension '%s' is not known. Not using any cpu " + "extensions", + def); + + return NULL; + } + } else { + def = NULL; + } + + while ((line = ec_code_proc_line(&file, &length)) != NULL) { + data = ec_code_proc_split(line, &length, ':'); + if ((data != NULL) && (strcmp(line, "flags") == 0)) { + list = data; + count = 0; + while ((data != NULL) && (*data != 0)) { + count++; + data = ec_code_proc_split(data, &length, ' '); + } + base = select; + select = ec_code_cpu_check(select, list, count); + if ((base != select) && (def != NULL)) { + gf_msg(xl->name, GF_LOG_WARNING, ENOTSUP, + EC_MSG_EXTENSION_UNSUPPORTED, + "CPU extension '%s' is not supported", def); + def = NULL; + } + } + } + + if (file.error) { + gf_msg(xl->name, GF_LOG_WARNING, 0, EC_MSG_EXTENSION_FAILED, + "Unable to determine supported CPU extensions. Not using any " + "cpu extensions"); + + gen = NULL; + } else { + gen = ec_code_gen_table[select]; + if (gen == NULL) { + gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION_NONE, + "Not using any cpu extensions"); + } else { + gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION, + "Using '%s' CPU extensions", gen->name); + } + } + + sys_close(file.fd); + +out: + return gen; +} diff --git a/xlators/cluster/ec/src/ec-code.h b/xlators/cluster/ec/src/ec-code.h new file mode 100644 index 00000000000..75fb35d93e3 --- /dev/null +++ b/xlators/cluster/ec/src/ec-code.h @@ -0,0 +1,44 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_CODE_H__ +#define __EC_CODE_H__ + +#include <glusterfs/xlator.h> +#include <glusterfs/list.h> + +#include "ec-types.h" +#include "ec-galois.h" + +ec_code_gen_t * +ec_code_detect(xlator_t *xl, const char *def); + +ec_code_t * +ec_code_create(ec_gf_t *gf, ec_code_gen_t *gen); + +void +ec_code_destroy(ec_code_t *code); + +ec_code_func_linear_t +ec_code_build_linear(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count); +ec_code_func_interleaved_t +ec_code_build_interleaved(ec_code_t *code, uint32_t width, uint32_t *values, + uint32_t count); +void +ec_code_release(ec_code_t *code, ec_code_func_t *func); + +void +ec_code_error(ec_code_builder_t *builder, int32_t error); + +void +ec_code_emit(ec_code_builder_t *builder, uint8_t *bytes, uint32_t count); + +#endif /* __EC_CODE_H__ */ diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c new file mode 100644 index 00000000000..703a30e2485 --- /dev/null +++ b/xlators/cluster/ec/src/ec-combine.c @@ -0,0 +1,995 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <fnmatch.h> + +#include "libxlator.h" +#include <glusterfs/byte-order.h> + +#include "ec-types.h" +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-combine.h" +#include "ec-messages.h" +#include <glusterfs/quota-common-utils.h> + +#define EC_QUOTA_PREFIX "trusted.glusterfs.quota." + +#define EC_MISSING_DATA ((data_t *)1ULL) + +struct _ec_dict_info; +typedef struct _ec_dict_info ec_dict_info_t; + +struct _ec_dict_combine; +typedef struct _ec_dict_combine ec_dict_combine_t; + +struct _ec_dict_info { + dict_t *dict; + int32_t count; +}; + +struct _ec_dict_combine { + ec_cbk_data_t *cbk; + int32_t which; +}; + +int32_t +ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + int valid = 0; + + if (!fop || !dst || !src) + return 0; + + switch (fop->id) { + case GF_FOP_REMOVEXATTR: + case GF_FOP_FREMOVEXATTR: + case GF_FOP_SETXATTR: + case GF_FOP_FSETXATTR: + return 1; + + case GF_FOP_SYMLINK: + case GF_FOP_LINK: + case GF_FOP_CREATE: + case GF_FOP_MKNOD: + case GF_FOP_MKDIR: + valid = 3; + break; + case GF_FOP_UNLINK: + case GF_FOP_RMDIR: + case GF_FOP_SETATTR: + case GF_FOP_FSETATTR: + case GF_FOP_TRUNCATE: + case GF_FOP_FTRUNCATE: + case GF_FOP_WRITE: + case GF_FOP_FALLOCATE: + case GF_FOP_DISCARD: + case GF_FOP_ZEROFILL: + valid = 2; + break; + case GF_FOP_RENAME: + valid = 5; + break; + default: + gf_msg_callingfn(fop->xl->name, GF_LOG_WARNING, EINVAL, + EC_MSG_INVALID_FOP, "Invalid fop %d", fop->id); + return 0; + break; + } + + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, valid)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of '%s'", + gf_fop_list[fop->id]); + return 0; + } + return 1; +} + +void +ec_iatt_time_merge(int64_t *dst_sec, uint32_t *dst_nsec, int64_t src_sec, + uint32_t src_nsec) +{ + if ((*dst_sec < src_sec) || + ((*dst_sec == src_sec) && (*dst_nsec < src_nsec))) { + *dst_sec = src_sec; + *dst_nsec = src_nsec; + } +} + +static gf_boolean_t +ec_iatt_is_trusted(ec_fop_data_t *fop, struct iatt *iatt) +{ + uint64_t ino; + int32_t i; + + /* Only the top level fop will have fop->locks filled. */ + while (fop->parent != NULL) { + fop = fop->parent; + } + + /* Lookups are special requests always done without locks taken but they + * require to be able to identify differences between bricks. Special + * handling of these differences is already done in lookup specific code + * so we shouldn't ignore any difference here and consider all iatt + * structures as trusted. */ + if (fop->id == GF_FOP_LOOKUP) { + return _gf_true; + } + + /* Check if the iatt references an inode locked by the current fop */ + for (i = 0; i < fop->lock_count; i++) { + ino = gfid_to_ino(fop->locks[i].lock->loc.inode->gfid); + if (iatt->ia_ino == ino) { + return _gf_true; + } + } + + return _gf_false; +} + +int32_t +ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src, + int32_t count) +{ + int32_t i; + gf_boolean_t failed = _gf_false; + + for (i = 0; i < count; i++) { + /* Check for basic fields. These fields must be equal always, even if + * the inode is not locked because in these cases the parent inode + * will be locked and differences in these fields require changes in + * the parent directory. */ + if ((dst[i].ia_ino != src[i].ia_ino) || + (((dst[i].ia_type == IA_IFBLK) || (dst[i].ia_type == IA_IFCHR)) && + (dst[i].ia_rdev != src[i].ia_rdev)) || + (gf_uuid_compare(dst[i].ia_gfid, src[i].ia_gfid) != 0)) { + failed = _gf_true; + } + /* Check for not so stable fields. These fields can change if the + * inode is not locked. */ + if (!failed && ((dst[i].ia_uid != src[i].ia_uid) || + (dst[i].ia_gid != src[i].ia_gid) || + (st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type) != + st_mode_from_ia(src[i].ia_prot, src[i].ia_type)))) { + if (ec_iatt_is_trusted(fop, dst)) { + /* If the iatt contains information from an inode that is + * locked, these differences are real problems, so we need to + * report them. Otherwise we ignore them and don't care which + * data is returned. */ + failed = _gf_true; + } else { + gf_msg_debug(fop->xl->name, 0, + "Ignoring iatt differences because inode is not " + "locked"); + } + } + if (failed) { + gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_IATT_COMBINE_FAIL, + "Failed to combine iatt (inode: %" PRIu64 "-%" PRIu64 + ", " + "links: %u-%u, uid: %u-%u, gid: %u-%u, " + "rdev: %" PRIu64 "-%" PRIu64 ", size: %" PRIu64 "-%" PRIu64 + ", " + "mode: %o-%o), %s", + dst[i].ia_ino, src[i].ia_ino, dst[i].ia_nlink, + src[i].ia_nlink, dst[i].ia_uid, src[i].ia_uid, dst[i].ia_gid, + src[i].ia_gid, dst[i].ia_rdev, src[i].ia_rdev, + dst[i].ia_size, src[i].ia_size, + st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type), + st_mode_from_ia(src[i].ia_prot, dst[i].ia_type), + ec_msg_str(fop)); + + return 0; + } + } + + while (count-- > 0) { + dst[count].ia_blocks += src[count].ia_blocks; + if (dst[count].ia_blksize < src[count].ia_blksize) { + dst[count].ia_blksize = src[count].ia_blksize; + } + + ec_iatt_time_merge(&dst[count].ia_atime, &dst[count].ia_atime_nsec, + src[count].ia_atime, src[count].ia_atime_nsec); + ec_iatt_time_merge(&dst[count].ia_mtime, &dst[count].ia_mtime_nsec, + src[count].ia_mtime, src[count].ia_mtime_nsec); + ec_iatt_time_merge(&dst[count].ia_ctime, &dst[count].ia_ctime_nsec, + src[count].ia_ctime, src[count].ia_ctime_nsec); + } + + return 1; +} + +void +ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers) +{ + uint64_t blocks; + + while (count-- > 0) { + blocks = iatt[count].ia_blocks * ec->fragments + answers - 1; + blocks /= answers; + iatt[count].ia_blocks = blocks; + } +} + +gf_boolean_t +ec_xattr_match(dict_t *dict, char *key, data_t *value, void *arg) +{ + if ((fnmatch(GF_XATTR_STIME_PATTERN, key, 0) == 0) || + (strcmp(key, GET_LINK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)) { + return _gf_false; + } + + return _gf_true; +} + +gf_boolean_t +ec_value_ignore(char *key) +{ + if ((strcmp(key, GF_CONTENT_KEY) == 0) || + (strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) || + (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0) || + (strcmp(key, GF_XATTR_LOCKINFO_KEY) == 0) || + (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) || + (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) || + (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) || + (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) || + (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) || + (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) || + (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) || + (XATTR_IS_NODE_UUID(key))) { + return _gf_true; + } + + return _gf_false; +} + +int32_t +ec_dict_compare(dict_t *dict1, dict_t *dict2) +{ + if (are_dicts_equal(dict1, dict2, ec_xattr_match, ec_value_ignore)) + return 1; + return 0; +} + +static uint32_t +ec_dict_list(data_t **list, ec_cbk_data_t *cbk, int32_t which, char *key, + gf_boolean_t global) +{ + ec_t *ec = cbk->fop->xl->private; + ec_cbk_data_t *ans = NULL; + dict_t *dict = NULL; + data_t *data; + uint32_t count; + int32_t i; + + for (i = 0; i < ec->nodes; i++) { + /* We initialize the list with EC_MISSING_DATA if we are + * returning a global list or the current subvolume belongs + * to the group of the accepted answer. Note that if some + * subvolume is known to be down before issuing the request, + * we won't have any answer from it, so we set here the + * appropriate default value. */ + if (global || ((cbk->mask & (1ULL << i)) != 0)) { + list[i] = EC_MISSING_DATA; + } else { + list[i] = NULL; + } + } + + count = 0; + list_for_each_entry(ans, &cbk->fop->answer_list, answer_list) + { + if (global || ((cbk->mask & ans->mask) != 0)) { + dict = (which == EC_COMBINE_XDATA) ? ans->xdata : ans->dict; + data = dict_get(dict, key); + if (data != NULL) { + list[ans->idx] = data; + count++; + } + } + } + + return count; +} + +int32_t +ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post, + const char *fmt, va_list args) +{ + char *tmp; + int32_t len; + + len = gf_vasprintf(str, fmt, args); + if (len < 0) { + return -ENOMEM; + } + + tmp = strchr(*str, '{'); + if (tmp == NULL) { + goto out; + } + *tmp++ = 0; + *sep = tmp; + tmp = strchr(tmp, '}'); + if (tmp == NULL) { + goto out; + } + *tmp++ = 0; + *post = tmp; + + return 0; + +out: + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_FORMAT, + "Invalid concat format"); + + GF_FREE(*str); + + return -EINVAL; +} + +static int32_t +ec_dict_data_concat(ec_cbk_data_t *cbk, int32_t which, char *key, char *new_key, + const char *def, gf_boolean_t global, const char *fmt, ...) +{ + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + char *str = NULL, *pre = NULL, *sep, *post; + dict_t *dict; + va_list args; + int32_t i, num, len, deflen, prelen, postlen, seplen, tmp; + int32_t err; + + ec_dict_list(data, cbk, which, key, global); + + va_start(args, fmt); + err = ec_concat_prepare(cbk->fop->xl, &pre, &sep, &post, fmt, args); + va_end(args); + + if (err != 0) { + return err; + } + + prelen = strlen(pre); + seplen = strlen(sep); + postlen = strlen(post); + + deflen = 0; + if (def != NULL) { + deflen = strlen(def); + } + + len = prelen + postlen + 1; + num = -1; + for (i = 0; i < ec->nodes; i++) { + if (data[i] == NULL) { + continue; + } + if (data[i] == EC_MISSING_DATA) { + if (def == NULL) { + continue; + } + len += deflen; + } else { + len += data[i]->len - 1; + } + if (num >= 0) { + len += seplen; + } + num++; + } + + err = -ENOMEM; + + str = GF_MALLOC(len, gf_common_mt_char); + if (str == NULL) { + goto out; + } + + memcpy(str, pre, prelen); + len = prelen; + for (i = 0; i < ec->nodes; i++) { + if (data[i] == NULL) { + continue; + } + if (data[i] == EC_MISSING_DATA) { + if (deflen == 0) { + continue; + } + tmp = deflen; + memcpy(str + len, def, tmp); + } else { + tmp = data[i]->len - 1; + memcpy(str + len, data[i]->data, tmp); + } + len += tmp; + if (i < num) { + memcpy(str + len, sep, seplen); + len += seplen; + } + } + memcpy(str + len, post, postlen + 1); + + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + if (new_key) { + key = new_key; + } + err = dict_set_dynstr(dict, key, str); + if (err != 0) { + goto out; + } + + str = NULL; + +out: + GF_FREE(str); + GF_FREE(pre); + + return err; +} + +int32_t +ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key) +{ + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict, *lockinfo, *tmp = NULL; + char *ptr = NULL; + int32_t i, len; + int32_t err; + + ec_dict_list(data, cbk, which, key, _gf_false); + + lockinfo = dict_new(); + if (lockinfo == NULL) { + return -ENOMEM; + } + + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } + + tmp = dict_new(); + if (tmp == NULL) { + err = -ENOMEM; + + goto out; + } + err = dict_unserialize(data[i]->data, data[i]->len, &tmp); + if (err != 0) { + goto out; + } + if (dict_copy(tmp, lockinfo) == NULL) { + err = -ENOMEM; + + goto out; + } + + dict_unref(tmp); + } + + tmp = NULL; + + err = dict_allocate_and_serialize(lockinfo, (char **)&ptr, + (unsigned int *)&len); + if (err != 0) { + goto out; + } + + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + err = dict_set_dynptr(dict, key, ptr, len); + if (err != 0) { + goto out; + } + + ptr = NULL; + +out: + GF_FREE(ptr); + dict_unref(lockinfo); + if (tmp != NULL) { + dict_unref(tmp); + } + + return err; +} + +int32_t +ec_dict_data_uuid(ec_cbk_data_t *cbk, int32_t which, char *key) +{ + ec_cbk_data_t *ans, *min; + dict_t *src, *dst; + data_t *data; + + min = cbk; + for (ans = cbk->next; ans != NULL; ans = ans->next) { + if (ans->idx < min->idx) { + min = ans; + } + } + + if (min != cbk) { + src = (which == EC_COMBINE_XDATA) ? min->xdata : min->dict; + dst = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + + data = dict_get(src, key); + if (data == NULL) { + return -ENOENT; + } + if (dict_set(dst, key, data) != 0) { + return -ENOMEM; + } + } + + return 0; +} + +int32_t +ec_dict_data_iatt(ec_cbk_data_t *cbk, int32_t which, char *key) +{ + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict; + struct iatt *stbuf, *tmp; + int32_t i, ret; + + ec_dict_list(data, cbk, which, key, _gf_false); + + stbuf = NULL; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } + tmp = data_to_iatt(data[i], key); + if (tmp == NULL) { + ret = -EINVAL; + goto out; + } + if (stbuf == NULL) { + stbuf = GF_MALLOC(sizeof(struct iatt), gf_common_mt_char); + if (stbuf == NULL) { + ret = -ENOMEM; + goto out; + } + *stbuf = *tmp; + } else { + if (!ec_iatt_combine(cbk->fop, stbuf, tmp, 1)) { + ret = -EINVAL; + goto out; + } + } + } + + if ((stbuf != NULL) && (stbuf->ia_type == IA_IFREG)) { + ec_iatt_rebuild(ec, stbuf, 1, cbk->count); + /* TODO: not sure if an iatt could come in xdata from a fop that takes + * no locks. */ + if (!ec_get_inode_size(cbk->fop, cbk->fop->locks[0].lock->loc.inode, + &stbuf->ia_size)) { + ret = -EINVAL; + goto out; + } + } + + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + ret = dict_set_iatt(dict, key, stbuf, false); + if (ret >= 0) { + stbuf = NULL; + } + +out: + GF_FREE(stbuf); + + return ret; +} + +int32_t +ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key) +{ + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict; + int32_t i; + uint32_t max, tmp; + + ec_dict_list(data, cbk, which, key, _gf_false); + + max = 0; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } + + tmp = data_to_uint32(data[i]); + if (max < tmp) { + max = tmp; + } + } + + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + return dict_set_uint32(dict, key, max); +} + +int32_t +ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key) +{ + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict; + int32_t i; + uint64_t max, tmp; + + ec_dict_list(data, cbk, which, key, _gf_false); + + max = 0; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } + + tmp = data_to_uint64(data[i]); + if (max < tmp) { + max = tmp; + } + } + + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + return dict_set_uint64(dict, key, max); +} + +int32_t +ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key) +{ + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict = NULL; + int32_t i = 0; + quota_meta_t size = { + 0, + }; + quota_meta_t max_size = { + 0, + }; + + if (ec_dict_list(data, cbk, which, key, _gf_false) == 0) { + return 0; + } + + /* Quota size xattr is managed outside of the control of the ec xlator. + * This means that it might not be updated at the same time on all + * bricks and we can receive slightly different values. If that's the + * case, we take the maximum of all received values. + */ + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA) || + (quota_data_to_meta(data[i], &size) < 0)) { + continue; + } + + if (size.size > max_size.size) + max_size.size = size.size; + if (size.file_count > max_size.file_count) + max_size.file_count = size.file_count; + if (size.dir_count > max_size.dir_count) + max_size.dir_count = size.dir_count; + } + + max_size.size *= ec->fragments; + + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + return quota_dict_set_meta(dict, key, &max_size, IA_IFDIR); +} + +int32_t +ec_dict_data_stime(ec_cbk_data_t *cbk, int32_t which, char *key) +{ + ec_t *ec = cbk->fop->xl->private; + data_t *data[ec->nodes]; + dict_t *dict; + int32_t i, err; + + ec_dict_list(data, cbk, which, key, _gf_false); + + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + for (i = 0; i < ec->nodes; i++) { + if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) { + continue; + } + err = gf_get_max_stime(cbk->fop->xl, dict, key, data[i]); + if (err != 0) { + gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err, + EC_MSG_STIME_COMBINE_FAIL, "STIME combination failed"); + + return err; + } + } + + return 0; +} + +int32_t +ec_dict_data_combine(dict_t *dict, char *key, data_t *value, void *arg) +{ + ec_dict_combine_t *data = arg; + + if ((strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) || + (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0)) { + return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL, + _gf_false, _gf_false, "(<EC:%s> { })", + data->cbk->fop->xl->name); + } + + if (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) { + return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL, + _gf_false, "{\n}"); + } + + if (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) == 0) { + return ec_dict_data_merge(data->cbk, data->which, key); + } + + if (strcmp(key, GET_LINK_COUNT) == 0) { + return ec_dict_data_max32(data->cbk, data->which, key); + } + + if (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) { + return ec_dict_data_max32(data->cbk, data->which, key); + } + if ((strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) || + (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0)) { + return ec_dict_data_max32(data->cbk, data->which, key); + } + + if (strcmp(key, QUOTA_SIZE_KEY) == 0) { + return ec_dict_data_quota(data->cbk, data->which, key); + } + /* Ignore all other quota attributes */ + if (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) { + return 0; + } + + if (XATTR_IS_NODE_UUID(key)) { + if (data->cbk->fop->int32) { + /* List of node uuid is requested */ + return ec_dict_data_concat(data->cbk, data->which, key, + GF_XATTR_LIST_NODE_UUIDS_KEY, UUID0_STR, + _gf_true, "{ }"); + } else { + return ec_dict_data_uuid(data->cbk, data->which, key); + } + } + + if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) { + return ec_dict_data_stime(data->cbk, data->which, key); + } + + if (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, FNM_NOESCAPE) == 0) { + return ec_dict_data_max64(data->cbk, data->which, key); + } + + if (strcmp(key, GF_PRESTAT) == 0 || strcmp(key, GF_POSTSTAT) == 0) { + return ec_dict_data_iatt(data->cbk, data->which, key); + } + + return 0; +} + +int32_t +ec_dict_combine(ec_cbk_data_t *cbk, int32_t which) +{ + dict_t *dict = NULL; + ec_dict_combine_t data; + int32_t err = 0; + + data.cbk = cbk; + data.which = which; + + dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict; + if (dict != NULL) { + err = dict_foreach(dict, ec_dict_data_combine, &data); + if (err != 0) { + gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err, + EC_MSG_DICT_COMBINE_FAIL, "Dictionary combination failed"); + + return err; + } + } + + return 0; +} + +int32_t +ec_vector_compare(struct iovec *dst_vector, int32_t dst_count, + struct iovec *src_vector, int32_t src_count) +{ + int32_t dst_size = 0, src_size = 0; + + if (dst_count > 0) { + dst_size = iov_length(dst_vector, dst_count); + } + if (src_count > 0) { + src_size = iov_length(src_vector, src_count); + } + + return (dst_size == src_size); +} + +int32_t +ec_flock_compare(struct gf_flock *dst, struct gf_flock *src) +{ + if ((dst->l_type != src->l_type) || (dst->l_whence != src->l_whence) || + (dst->l_start != src->l_start) || (dst->l_len != src->l_len) || + (dst->l_pid != src->l_pid) || + !is_same_lkowner(&dst->l_owner, &src->l_owner)) { + return 0; + } + + return 1; +} + +void +ec_statvfs_combine(struct statvfs *dst, struct statvfs *src) +{ + if (dst->f_bsize < src->f_bsize) { + dst->f_bsize = src->f_bsize; + } + + if (dst->f_frsize < src->f_frsize) { + dst->f_blocks *= dst->f_frsize; + dst->f_blocks /= src->f_frsize; + + dst->f_bfree *= dst->f_frsize; + dst->f_bfree /= src->f_frsize; + + dst->f_bavail *= dst->f_frsize; + dst->f_bavail /= src->f_frsize; + + dst->f_frsize = src->f_frsize; + } else if (dst->f_frsize > src->f_frsize) { + src->f_blocks *= src->f_frsize; + src->f_blocks /= dst->f_frsize; + + src->f_bfree *= src->f_frsize; + src->f_bfree /= dst->f_frsize; + + src->f_bavail *= src->f_frsize; + src->f_bavail /= dst->f_frsize; + } + if (dst->f_blocks > src->f_blocks) { + dst->f_blocks = src->f_blocks; + } + if (dst->f_bfree > src->f_bfree) { + dst->f_bfree = src->f_bfree; + } + if (dst->f_bavail > src->f_bavail) { + dst->f_bavail = src->f_bavail; + } + + if (dst->f_files < src->f_files) { + dst->f_files = src->f_files; + } + if (dst->f_ffree > src->f_ffree) { + dst->f_ffree = src->f_ffree; + } + if (dst->f_favail > src->f_favail) { + dst->f_favail = src->f_favail; + } + if (dst->f_namemax > src->f_namemax) { + dst->f_namemax = src->f_namemax; + } + + if (dst->f_flag != src->f_flag) { + gf_msg_debug(THIS->name, 0, + "Mismatching file system flags " + "(%lX, %lX)", + dst->f_flag, src->f_flag); + } + dst->f_flag &= src->f_flag; +} + +int32_t +ec_combine_check(ec_cbk_data_t *dst, ec_cbk_data_t *src, ec_combine_f combine) +{ + ec_fop_data_t *fop = dst->fop; + + if (dst->op_ret != src->op_ret) { + gf_msg_debug(fop->xl->name, 0, + "Mismatching return code in " + "answers of '%s': %d <-> %d", + ec_fop_name(fop->id), dst->op_ret, src->op_ret); + + return 0; + } + if (dst->op_ret < 0) { + if (dst->op_errno != src->op_errno) { + gf_msg_debug(fop->xl->name, 0, + "Mismatching errno code in " + "answers of '%s': %d <-> %d", + ec_fop_name(fop->id), dst->op_errno, src->op_errno); + + return 0; + } + } + + if (!ec_dict_compare(dst->xdata, src->xdata)) { + gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_XDATA_MISMATCH, + "Mismatching xdata in answers " + "of '%s'", + ec_fop_name(fop->id)); + + return 0; + } + + if ((dst->op_ret >= 0) && (combine != NULL)) { + return combine(fop, dst, src); + } + + return 1; +} + +void +ec_combine(ec_cbk_data_t *newcbk, ec_combine_f combine) +{ + ec_fop_data_t *fop = newcbk->fop; + ec_cbk_data_t *cbk = NULL, *tmp = NULL; + struct list_head *item = NULL; + int32_t needed = 0; + char str[32]; + + LOCK(&fop->lock); + + fop->received |= newcbk->mask; + + item = fop->cbk_list.prev; + list_for_each_entry(cbk, &fop->cbk_list, list) + { + if (ec_combine_check(newcbk, cbk, combine)) { + newcbk->count += cbk->count; + newcbk->mask |= cbk->mask; + + item = cbk->list.prev; + while (item != &fop->cbk_list) { + tmp = list_entry(item, ec_cbk_data_t, list); + if (tmp->count >= newcbk->count) { + break; + } + item = item->prev; + } + list_del(&cbk->list); + + newcbk->next = cbk; + + break; + } + } + list_add(&newcbk->list, item); + + ec_trace("ANSWER", fop, "combine=%s[%d]", + ec_bin(str, sizeof(str), newcbk->mask, 0), newcbk->count); + + cbk = list_entry(fop->cbk_list.next, ec_cbk_data_t, list); + if ((fop->mask ^ fop->remaining) == fop->received) { + needed = fop->minimum - cbk->count; + } + + UNLOCK(&fop->lock); + + if (needed > 0) { + ec_dispatch_next(fop, newcbk->idx); + } +} diff --git a/xlators/cluster/ec/src/ec-combine.h b/xlators/cluster/ec/src/ec-combine.h new file mode 100644 index 00000000000..1010cc3be26 --- /dev/null +++ b/xlators/cluster/ec/src/ec-combine.h @@ -0,0 +1,44 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_COMBINE_H__ +#define __EC_COMBINE_H__ + +#define EC_COMBINE_DICT 0 +#define EC_COMBINE_XDATA 1 + +typedef int32_t (*ec_combine_f)(ec_fop_data_t *fop, ec_cbk_data_t *dst, + ec_cbk_data_t *src); + +void +ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers); + +int32_t +ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src, + int32_t count); +int32_t +ec_dict_compare(dict_t *dict1, dict_t *dict2); +int32_t +ec_vector_compare(struct iovec *dst_vector, int32_t dst_count, + struct iovec *src_vector, int32_t src_count); +int32_t +ec_flock_compare(struct gf_flock *dst, struct gf_flock *src); +void +ec_statvfs_combine(struct statvfs *dst, struct statvfs *src); + +int32_t +ec_dict_combine(ec_cbk_data_t *cbk, int32_t which); + +void +ec_combine(ec_cbk_data_t *cbk, ec_combine_f combine); + +int32_t +ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src); +#endif /* __EC_COMBINE_H__ */ diff --git a/xlators/cluster/ec/src/ec-common.c b/xlators/cluster/ec/src/ec-common.c new file mode 100644 index 00000000000..b955efd8c2d --- /dev/null +++ b/xlators/cluster/ec/src/ec-common.c @@ -0,0 +1,3042 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <glusterfs/byte-order.h> +#include <glusterfs/hashfn.h> + +#include "ec-mem-types.h" +#include "ec-types.h" +#include "ec-helpers.h" +#include "ec-combine.h" +#include "ec-common.h" +#include "ec-fops.h" +#include "ec-method.h" +#include "ec.h" +#include "ec-messages.h" + +#define EC_INVALID_INDEX UINT32_MAX + +void +ec_update_fd_status(fd_t *fd, xlator_t *xl, int idx, int32_t ret_status) +{ + ec_fd_t *fd_ctx; + + if (fd == NULL) + return; + + LOCK(&fd->lock); + { + fd_ctx = __ec_fd_get(fd, xl); + if (fd_ctx) { + if (ret_status >= 0) + fd_ctx->fd_status[idx] = EC_FD_OPENED; + else + fd_ctx->fd_status[idx] = EC_FD_NOT_OPENED; + } + } + UNLOCK(&fd->lock); +} + +static uintptr_t +ec_fd_ctx_need_open(fd_t *fd, xlator_t *this, uintptr_t mask) +{ + int i = 0; + int count = 0; + ec_t *ec = NULL; + ec_fd_t *fd_ctx = NULL; + uintptr_t need_open = 0; + + ec = this->private; + + fd_ctx = ec_fd_get(fd, this); + if (!fd_ctx) + return count; + + LOCK(&fd->lock); + { + for (i = 0; i < ec->nodes; i++) { + if ((fd_ctx->fd_status[i] == EC_FD_NOT_OPENED) && + ((ec->xl_up & (1 << i)) != 0) && ((mask & (1 << i)) != 0)) { + fd_ctx->fd_status[i] = EC_FD_OPENING; + need_open |= (1 << i); + count++; + } + } + } + UNLOCK(&fd->lock); + + /* If fd needs to open on minimum number of nodes + * then ignore fixing the fd as it has been + * requested from heal operation. + */ + if (count >= ec->fragments) { + need_open = 0; + } + + return need_open; +} + +static gf_boolean_t +ec_is_fd_fixable(fd_t *fd) +{ + if (!fd || !fd->inode) + return _gf_false; + else if (fd_is_anonymous(fd)) + return _gf_false; + else if (gf_uuid_is_null(fd->inode->gfid)) + return _gf_false; + + return _gf_true; +} + +static void +ec_fix_open(ec_fop_data_t *fop, uintptr_t mask) +{ + uintptr_t need_open = 0; + int ret = 0; + int32_t flags = 0; + loc_t loc = { + 0, + }; + + if (!ec_is_fd_fixable(fop->fd)) + goto out; + + /* Evaluate how many remote fd's to be opened */ + need_open = ec_fd_ctx_need_open(fop->fd, fop->xl, mask); + if (need_open == 0) { + goto out; + } + + loc.inode = inode_ref(fop->fd->inode); + gf_uuid_copy(loc.gfid, fop->fd->inode->gfid); + ret = loc_path(&loc, NULL); + if (ret < 0) { + goto out; + } + + flags = fop->fd->flags & (~(O_TRUNC | O_APPEND | O_CREAT | O_EXCL)); + if (IA_IFDIR == fop->fd->inode->ia_type) { + ec_opendir(fop->frame, fop->xl, need_open, + EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL, + &fop->loc[0], fop->fd, NULL); + } else { + ec_open(fop->frame, fop->xl, need_open, + EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL, &loc, + flags, fop->fd, NULL); + } + +out: + loc_wipe(&loc); +} + +static off_t +ec_range_end_get(off_t fl_start, uint64_t fl_size) +{ + if (fl_size > 0) { + if (fl_size >= EC_RANGE_FULL) { + /* Infinity */ + fl_start = LLONG_MAX; + } else { + fl_start += fl_size - 1; + if (fl_start < 0) { + /* Overflow */ + fl_start = LLONG_MAX; + } + } + } + + return fl_start; +} + +static gf_boolean_t +ec_is_range_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2) +{ + return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start)); +} + +static gf_boolean_t +ec_lock_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2) +{ + ec_t *ec = l1->fop->xl->private; + + /* Fops like access/stat won't have to worry what the other fops are + * modifying as the fop is wound only to one brick. So it can be + * executed in parallel*/ + if (l1->fop->minimum == EC_MINIMUM_ONE || + l2->fop->minimum == EC_MINIMUM_ONE) + return _gf_false; + + if ((l1->fop->flags & EC_FLAG_LOCK_SHARED) && + (l2->fop->flags & EC_FLAG_LOCK_SHARED)) + return _gf_false; + + if (!ec->parallel_writes) { + return _gf_true; + } + + return ec_is_range_conflict(l1, l2); +} + +uint32_t +ec_select_first_by_read_policy(ec_t *ec, ec_fop_data_t *fop) +{ + if (ec->read_policy == EC_ROUND_ROBIN) { + return ec->idx; + } else if (ec->read_policy == EC_GFID_HASH) { + if (fop->use_fd) { + return SuperFastHash((char *)fop->fd->inode->gfid, + sizeof(fop->fd->inode->gfid)) % + ec->nodes; + } else { + if (gf_uuid_is_null(fop->loc[0].gfid)) + loc_gfid(&fop->loc[0], fop->loc[0].gfid); + return SuperFastHash((char *)fop->loc[0].gfid, + sizeof(fop->loc[0].gfid)) % + ec->nodes; + } + } + return 0; +} + +static gf_boolean_t +ec_child_valid(ec_t *ec, ec_fop_data_t *fop, uint32_t idx) +{ + return (idx < ec->nodes) && (((fop->remaining >> idx) & 1) == 1); +} + +static uint32_t +ec_child_next(ec_t *ec, ec_fop_data_t *fop, uint32_t idx) +{ + while (!ec_child_valid(ec, fop, idx)) { + if (++idx >= ec->nodes) { + idx = 0; + } + if (idx == fop->first) { + return EC_INVALID_INDEX; + } + } + + return idx; +} + +int32_t +ec_heal_report(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, uintptr_t mask, uintptr_t good, + uintptr_t bad, uint32_t pending, dict_t *xdata) +{ + if (op_ret < 0) { + gf_msg(this->name, GF_LOG_DEBUG, op_errno, EC_MSG_HEAL_FAIL, + "Heal failed"); + } else { + if ((mask & ~good) != 0) { + gf_msg(this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_SUCCESS, + "Heal succeeded on %d/%d " + "subvolumes", + gf_bits_count(mask & ~(good | bad)), + gf_bits_count(mask & ~good)); + } + } + + return 0; +} + +static uintptr_t +ec_fop_needs_name_heal(ec_fop_data_t *fop) +{ + ec_t *ec = NULL; + ec_cbk_data_t *cbk = NULL; + ec_cbk_data_t *enoent_cbk = NULL; + + ec = fop->xl->private; + if (fop->id != GF_FOP_LOOKUP) + return 0; + + if (!fop->loc[0].name || strlen(fop->loc[0].name) == 0) + return 0; + + list_for_each_entry(cbk, &fop->cbk_list, list) + { + if (cbk->op_ret < 0 && cbk->op_errno == ENOENT) { + enoent_cbk = cbk; + break; + } + } + + if (!enoent_cbk) + return 0; + + return ec->xl_up & ~enoent_cbk->mask; +} + +int32_t +ec_fop_needs_heal(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + + if (fop->lock_count == 0) { + /* + * if fop->lock_count is zero that means it saw version mismatch + * without any locks so it can't be trusted. If we launch a heal + * based on this it will lead to INODELKs which will affect I/O + * performance. Considering self-heal-daemon and operations on + * the inode from client which take locks can still trigger the + * heal we can choose to not attempt a heal when fop->lock_count + * is zero. + */ + return 0; + } + return (ec->xl_up & ~(fop->remaining | fop->good)) != 0; +} + +void +ec_check_status(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + int32_t partial = 0; + char str1[32], str2[32], str3[32], str4[32], str5[32]; + + if (!ec_fop_needs_name_heal(fop) && !ec_fop_needs_heal(fop)) { + return; + } + + if (fop->answer && fop->answer->op_ret >= 0) { + if ((fop->id == GF_FOP_LOOKUP) || (fop->id == GF_FOP_STAT) || + (fop->id == GF_FOP_FSTAT)) { + partial = fop->answer->iatt[0].ia_type == IA_IFDIR; + } else if (fop->id == GF_FOP_OPENDIR) { + partial = 1; + } + } + + gf_msg( + fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_FAIL_ON_SUBVOLS, + "Operation failed on %d of %d subvolumes.(up=%s, mask=%s, " + "remaining=%s, good=%s, bad=%s," + "(Least significant bit represents first client/brick of subvol), %s)", + gf_bits_count(ec->xl_up & ~(fop->remaining | fop->good)), ec->nodes, + ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), fop->mask, ec->nodes), + ec_bin(str3, sizeof(str3), fop->remaining, ec->nodes), + ec_bin(str4, sizeof(str4), fop->good, ec->nodes), + ec_bin(str5, sizeof(str5), ec->xl_up & ~(fop->remaining | fop->good), + ec->nodes), + ec_msg_str(fop)); + if (fop->use_fd) { + if (fop->fd != NULL) { + ec_fheal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL, + fop->fd, partial, NULL); + } + } else { + ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL, + &fop->loc[0], partial, NULL); + + if (fop->loc[1].inode != NULL) { + ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL, + &fop->loc[1], partial, NULL); + } + } +} + +void +ec_update_good(ec_fop_data_t *fop, uintptr_t good) +{ + fop->good = good; + + /* Fops that are executed only on one brick do not have enough information + * to decide if healing is needed or not. */ + if ((fop->expected != 1) && (fop->parent == NULL)) { + ec_check_status(fop); + } +} + +void +ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop) +{ + /* Fops that are executed only on one brick do not have enough information + * to update the global mask of good bricks. */ + if (fop->expected == 1) { + return; + } + + /* When updating the good mask of the lock, we only take into consideration + * those bits corresponding to the bricks where the fop has been executed. + * Bad bricks are removed from good_mask, but once marked as bad it's never + * set to good until the lock is released and reacquired */ + + lock->good_mask &= fop->good | fop->remaining; +} + +void +__ec_fop_set_error(ec_fop_data_t *fop, int32_t error) +{ + if ((error != 0) && (fop->error == 0)) { + fop->error = error; + } +} + +void +ec_fop_set_error(ec_fop_data_t *fop, int32_t error) +{ + LOCK(&fop->lock); + + __ec_fop_set_error(fop, error); + + UNLOCK(&fop->lock); +} + +gf_boolean_t +ec_cbk_set_error(ec_cbk_data_t *cbk, int32_t error, gf_boolean_t ro) +{ + if ((error != 0) && (cbk->op_ret >= 0)) { + /* If cbk->op_errno was 0, it means that the fop succeeded and this + * error has happened while processing the answer. If the operation was + * read-only, there's no problem (i.e. we simply return the generated + * error code). However if it caused a modification, we must return EIO + * to indicate that the operation has been partially executed. */ + cbk->op_errno = ro ? error : EIO; + cbk->op_ret = -1; + + ec_fop_set_error(cbk->fop, cbk->op_errno); + } + + return (cbk->op_ret < 0); +} + +ec_cbk_data_t * +ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro) +{ + ec_cbk_data_t *cbk; + int32_t err; + + cbk = fop->answer; + if (cbk == NULL) { + ec_fop_set_error(fop, EIO); + + return NULL; + } + + if (cbk->op_ret < 0) { + ec_fop_set_error(fop, cbk->op_errno); + } + + err = ec_dict_combine(cbk, EC_COMBINE_XDATA); + if (ec_cbk_set_error(cbk, -err, ro)) { + return NULL; + } + + return cbk; +} + +void +ec_sleep(ec_fop_data_t *fop) +{ + LOCK(&fop->lock); + + GF_ASSERT(fop->refs > 0); + fop->refs++; + fop->jobs++; + + UNLOCK(&fop->lock); +} + +int32_t +ec_check_complete(ec_fop_data_t *fop, ec_resume_f resume) +{ + int32_t error = -1; + + LOCK(&fop->lock); + + GF_ASSERT(fop->resume == NULL); + + if (--fop->jobs != 0) { + ec_trace("WAIT", fop, "resume=%p", resume); + + fop->resume = resume; + } else { + error = fop->error; + fop->error = 0; + } + + UNLOCK(&fop->lock); + + return error; +} + +void +ec_resume(ec_fop_data_t *fop, int32_t error) +{ + ec_resume_f resume = NULL; + + LOCK(&fop->lock); + + __ec_fop_set_error(fop, error); + + if (--fop->jobs == 0) { + resume = fop->resume; + fop->resume = NULL; + if (resume != NULL) { + ec_trace("RESUME", fop, "error=%d", error); + + if (fop->error != 0) { + error = fop->error; + } + fop->error = 0; + } + } + + UNLOCK(&fop->lock); + + if (resume != NULL) { + resume(fop, error); + } + + ec_fop_data_release(fop); +} + +void +ec_resume_parent(ec_fop_data_t *fop) +{ + ec_fop_data_t *parent; + int32_t error = 0; + + parent = fop->parent; + if (parent != NULL) { + if ((fop->fop_flags & EC_FOP_NO_PROPAGATE_ERROR) == 0) { + error = fop->error; + } + ec_trace("RESUME_PARENT", fop, "error=%u", error); + fop->parent = NULL; + ec_resume(parent, error); + } +} + +gf_boolean_t +ec_is_recoverable_error(int32_t op_errno) +{ + switch (op_errno) { + case ENOTCONN: + case ESTALE: + case ENOENT: + case EBADFD: /*Opened fd but brick is disconnected*/ + case EIO: /*Backend-fs crash like XFS/ext4 etc*/ + return _gf_true; + } + return _gf_false; +} + +void +ec_complete(ec_fop_data_t *fop) +{ + ec_cbk_data_t *cbk = NULL; + int32_t resume = 0, update = 0; + int healing_count = 0; + + LOCK(&fop->lock); + + ec_trace("COMPLETE", fop, ""); + + if (--fop->winds == 0) { + if (fop->answer == NULL) { + if (!list_empty(&fop->cbk_list)) { + cbk = list_entry(fop->cbk_list.next, ec_cbk_data_t, list); + healing_count = gf_bits_count(cbk->mask & fop->healing); + /* fop shouldn't be treated as success if it is not + * successful on at least fop->minimum good copies*/ + if ((cbk->count - healing_count) >= fop->minimum) { + fop->answer = cbk; + + update = 1; + } + } + + resume = 1; + } + } + + UNLOCK(&fop->lock); + + /* ec_update_good() locks inode->lock. This may cause deadlocks with + fop->lock when used in another order. Since ec_update_good() will not + be called more than once for each fop, it can be called from outside + the fop->lock locked region. */ + if (update) { + ec_update_good(fop, cbk->mask); + } + + if (resume) { + ec_resume(fop, 0); + } + + ec_fop_data_release(fop); +} + +/* There could be already granted locks sitting on the bricks, unlock for which + * must be wound at all costs*/ +static gf_boolean_t +ec_must_wind(ec_fop_data_t *fop) +{ + if ((fop->id == GF_FOP_INODELK) || (fop->id == GF_FOP_FINODELK) || + (fop->id == GF_FOP_LK)) { + if (fop->flock.l_type == F_UNLCK) + return _gf_true; + } else if ((fop->id == GF_FOP_ENTRYLK) || (fop->id == GF_FOP_FENTRYLK)) { + if (fop->entrylk_cmd == ENTRYLK_UNLOCK) + return _gf_true; + } + + return _gf_false; +} + +static gf_boolean_t +ec_internal_op(ec_fop_data_t *fop) +{ + if (ec_must_wind(fop)) + return _gf_true; + if (fop->id == GF_FOP_XATTROP) + return _gf_true; + if (fop->id == GF_FOP_FXATTROP) + return _gf_true; + if (fop->id == GF_FOP_OPEN) + return _gf_true; + return _gf_false; +} + +char * +ec_msg_str(ec_fop_data_t *fop) +{ + loc_t *loc1 = NULL; + loc_t *loc2 = NULL; + char gfid1[64] = {0}; + char gfid2[64] = {0}; + ec_fop_data_t *parent = fop->parent; + + if (fop->errstr) + return fop->errstr; + if (!fop->use_fd) { + loc1 = &fop->loc[0]; + loc2 = &fop->loc[1]; + + if (fop->id == GF_FOP_RENAME) { + gf_asprintf(&fop->errstr, + "FOP : '%s' failed on '%s' and '%s' with gfids " + "%s and %s respectively. Parent FOP: %s", + ec_fop_name(fop->id), loc1->path, loc2->path, + uuid_utoa_r(loc1->gfid, gfid1), + uuid_utoa_r(loc2->gfid, gfid2), + parent ? ec_fop_name(parent->id) : "No Parent"); + } else { + gf_asprintf( + &fop->errstr, + "FOP : '%s' failed on '%s' with gfid %s. Parent FOP: %s", + ec_fop_name(fop->id), loc1->path, + uuid_utoa_r(loc1->gfid, gfid1), + parent ? ec_fop_name(parent->id) : "No Parent"); + } + } else { + gf_asprintf( + &fop->errstr, "FOP : '%s' failed on gfid %s. Parent FOP: %s", + ec_fop_name(fop->id), uuid_utoa_r(fop->fd->inode->gfid, gfid1), + parent ? ec_fop_name(parent->id) : "No Parent"); + } + return fop->errstr; +} + +static void +ec_log_insufficient_vol(ec_fop_data_t *fop, int32_t have, uint32_t need, + int32_t loglevel) +{ + ec_t *ec = fop->xl->private; + char str1[32], str2[32], str3[32]; + + gf_msg(ec->xl->name, loglevel, 0, EC_MSG_CHILDS_INSUFFICIENT, + "Insufficient available children for this request: " + "Have : %d, Need : %u : Child UP : %s " + "Mask: %s, Healing : %s : %s ", + have, need, ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), fop->mask, ec->nodes), + ec_bin(str3, sizeof(str3), fop->healing, ec->nodes), + ec_msg_str(fop)); +} + +static int32_t +ec_child_select(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + int32_t first = 0, num = 0; + + ec_fop_cleanup(fop); + + fop->mask &= ec->node_mask; + /* Wind the fop on same subvols as parent for any internal extra fops like + * head/tail read in case of writev fop. Unlocks shouldn't do this because + * unlock should go on all subvols where lock is performed*/ + if (fop->parent && !ec_internal_op(fop)) { + fop->mask &= (fop->parent->mask & ~fop->parent->healing); + if (ec_is_data_fop(fop->id)) { + fop->healing |= fop->parent->healing; + } + } + + if ((fop->mask & ~ec->xl_up) != 0) { + gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_EXEC_UNAVAIL, + "Executing operation with " + "some subvolumes unavailable. (%" PRIXPTR "). %s ", + fop->mask & ~ec->xl_up, ec_msg_str(fop)); + fop->mask &= ec->xl_up; + } + + switch (fop->minimum) { + case EC_MINIMUM_ALL: + fop->minimum = gf_bits_count(fop->mask); + if (fop->minimum >= ec->fragments) { + break; + } + case EC_MINIMUM_MIN: + fop->minimum = ec->fragments; + break; + case EC_MINIMUM_ONE: + fop->minimum = 1; + } + + if (ec->read_policy == EC_ROUND_ROBIN) { + first = ec->idx; + if (++first >= ec->nodes) { + first = 0; + } + ec->idx = first; + } + + num = gf_bits_count(fop->mask); + /*Unconditionally wind on healing subvolumes*/ + fop->mask |= fop->healing; + fop->remaining = fop->mask; + fop->received = 0; + + ec_trace("SELECT", fop, ""); + + if ((num < fop->minimum) && (num < ec->fragments)) { + ec_log_insufficient_vol(fop, num, fop->minimum, GF_LOG_ERROR); + return 0; + } + + if (!fop->parent && fop->lock_count && + (fop->locks[0].update[EC_DATA_TXN] || + fop->locks[0].update[EC_METADATA_TXN])) { + if (ec->quorum_count && (num < ec->quorum_count)) { + ec_log_insufficient_vol(fop, num, ec->quorum_count, GF_LOG_ERROR); + return 0; + } + } + + return 1; +} + +void +ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx) +{ + uint32_t i = EC_INVALID_INDEX; + ec_t *ec = fop->xl->private; + + LOCK(&fop->lock); + + i = ec_child_next(ec, fop, idx); + if (i < EC_MAX_NODES) { + idx = i; + + fop->remaining ^= 1ULL << idx; + + ec_trace("EXECUTE", fop, "idx=%d", idx); + + fop->winds++; + fop->refs++; + } + + UNLOCK(&fop->lock); + + if (i < EC_MAX_NODES) { + fop->wind(ec, fop, idx); + } +} + +void +ec_dispatch_mask(ec_fop_data_t *fop, uintptr_t mask) +{ + ec_t *ec = fop->xl->private; + int32_t count, idx; + + count = gf_bits_count(mask); + + LOCK(&fop->lock); + + ec_trace("EXECUTE", fop, "mask=%lX", mask); + + fop->remaining ^= mask; + + fop->winds += count; + fop->refs += count; + + UNLOCK(&fop->lock); + + idx = 0; + while (mask != 0) { + if ((mask & 1) != 0) { + fop->wind(ec, fop, idx); + } + idx++; + mask >>= 1; + } +} + +void +ec_dispatch_start(ec_fop_data_t *fop) +{ + fop->answer = NULL; + fop->good = 0; + + INIT_LIST_HEAD(&fop->cbk_list); + + if (fop->lock_count > 0) { + ec_owner_copy(fop->frame, &fop->req_frame->root->lk_owner); + } +} + +void +ec_dispatch_one(ec_fop_data_t *fop) +{ + ec_dispatch_start(fop); + + if (ec_child_select(fop)) { + ec_sleep(fop); + + fop->expected = 1; + fop->first = ec_select_first_by_read_policy(fop->xl->private, fop); + + ec_dispatch_next(fop, fop->first); + } +} + +gf_boolean_t +ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk) +{ + ec_cbk_data_t *tmp; + + tmp = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + *cbk = tmp; + } + if ((tmp != NULL) && (tmp->op_ret < 0) && + ec_is_recoverable_error(tmp->op_errno)) { + GF_ASSERT(fop->mask & (1ULL << tmp->idx)); + fop->mask ^= (1ULL << tmp->idx); + if (fop->mask) { + return _gf_true; + } + } + + return _gf_false; +} + +void +ec_dispatch_inc(ec_fop_data_t *fop) +{ + ec_dispatch_start(fop); + + if (ec_child_select(fop)) { + ec_sleep(fop); + + fop->expected = gf_bits_count(fop->remaining); + fop->first = 0; + + ec_dispatch_next(fop, 0); + } +} + +void +ec_dispatch_all(ec_fop_data_t *fop) +{ + ec_dispatch_start(fop); + + if (ec_child_select(fop)) { + ec_sleep(fop); + + fop->expected = gf_bits_count(fop->remaining); + fop->first = 0; + + ec_dispatch_mask(fop, fop->remaining); + } +} + +void +ec_dispatch_min(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + uintptr_t mask; + uint32_t idx; + int32_t count; + + ec_dispatch_start(fop); + + if (ec_child_select(fop)) { + ec_sleep(fop); + + fop->expected = count = ec->fragments; + fop->first = ec_select_first_by_read_policy(fop->xl->private, fop); + idx = fop->first - 1; + mask = 0; + while (count-- > 0) { + idx = ec_child_next(ec, fop, idx + 1); + if (idx < EC_MAX_NODES) + mask |= 1ULL << idx; + } + + ec_dispatch_mask(fop, mask); + } +} + +void +ec_succeed_all(ec_fop_data_t *fop) +{ + ec_dispatch_start(fop); + + if (ec_child_select(fop)) { + fop->expected = gf_bits_count(fop->remaining); + fop->first = 0; + + /* Simulate a successful execution on all bricks */ + ec_trace("SUCCEED", fop, ""); + + fop->good = fop->remaining; + fop->remaining = 0; + } +} + +ec_lock_t * +ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc) +{ + ec_t *ec = fop->xl->private; + ec_lock_t *lock; + int32_t err; + + if ((loc->inode == NULL) || + (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid))) { + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_INODE, + "Trying to lock based on an invalid " + "inode"); + + __ec_fop_set_error(fop, EINVAL); + + return NULL; + } + + lock = mem_get0(ec->lock_pool); + if (lock != NULL) { + lock->good_mask = UINTPTR_MAX; + INIT_LIST_HEAD(&lock->owners); + INIT_LIST_HEAD(&lock->waiting); + INIT_LIST_HEAD(&lock->frozen); + err = ec_loc_from_loc(fop->xl, &lock->loc, loc); + if (err != 0) { + mem_put(lock); + lock = NULL; + + __ec_fop_set_error(fop, -err); + } + } + + return lock; +} + +void +ec_lock_destroy(ec_lock_t *lock) +{ + loc_wipe(&lock->loc); + if (lock->fd != NULL) { + fd_unref(lock->fd); + } + + mem_put(lock); +} + +int32_t +ec_lock_compare(ec_lock_t *lock1, ec_lock_t *lock2) +{ + return gf_uuid_compare(lock1->loc.gfid, lock2->loc.gfid); +} + +static void +ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, uint32_t flags, loc_t *base, + off_t fl_start, uint64_t fl_size) +{ + ec_lock_link_t *link; + + /* This check is only prepared for up to 2 locks per fop. If more locks + * are needed this must be changed. */ + if ((fop->lock_count > 0) && + (ec_lock_compare(fop->locks[0].lock, lock) < 0)) { + fop->first_lock = fop->lock_count; + } else { + /* When the first lock is added to the current fop, request lock + * counts from locks xlator to be able to determine if there is + * contention and release the lock sooner. */ + if (fop->xdata == NULL) { + fop->xdata = dict_new(); + if (fop->xdata == NULL) { + ec_fop_set_error(fop, ENOMEM); + return; + } + } + if (dict_set_str(fop->xdata, GLUSTERFS_INODELK_DOM_COUNT, + fop->xl->name) != 0) { + ec_fop_set_error(fop, ENOMEM); + return; + } + } + + link = &fop->locks[fop->lock_count++]; + + link->lock = lock; + link->fop = fop; + link->update[EC_DATA_TXN] = (flags & EC_UPDATE_DATA) != 0; + link->update[EC_METADATA_TXN] = (flags & EC_UPDATE_META) != 0; + link->base = base; + link->fl_start = fl_start; + link->fl_end = ec_range_end_get(fl_start, fl_size); + + lock->refs_pending++; +} + +static void +ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc, uint32_t flags, + loc_t *base, off_t fl_start, uint64_t fl_size) +{ + ec_lock_t *lock = NULL; + ec_inode_t *ctx; + + if ((fop->parent != NULL) || (fop->error != 0) || (loc->inode == NULL)) { + return; + } + + LOCK(&loc->inode->lock); + + ctx = __ec_inode_get(loc->inode, fop->xl); + if (ctx == NULL) { + __ec_fop_set_error(fop, ENOMEM); + + goto unlock; + } + + if (ctx->inode_lock != NULL) { + lock = ctx->inode_lock; + + /* If there's another lock, make sure that it's not the same. Otherwise + * do not insert it. + * + * This can only happen on renames where source and target names are + * in the same directory. */ + if ((fop->lock_count > 0) && (fop->locks[0].lock == lock)) { + /* Combine data/meta updates */ + fop->locks[0].update[EC_DATA_TXN] |= (flags & EC_UPDATE_DATA) != 0; + fop->locks[0].update[EC_METADATA_TXN] |= (flags & EC_UPDATE_META) != + 0; + + /* Only one base inode is allowed per fop, so there shouldn't be + * overwrites here. */ + if (base != NULL) { + fop->locks[0].base = base; + } + + goto update_query; + } + + ec_trace("LOCK_INODELK", fop, + "lock=%p, inode=%p. Lock already " + "acquired", + lock, loc->inode); + + goto insert; + } + + lock = ec_lock_allocate(fop, loc); + if (lock == NULL) { + goto unlock; + } + + ec_trace("LOCK_CREATE", fop, "lock=%p", lock); + + lock->flock.l_type = F_WRLCK; + lock->flock.l_whence = SEEK_SET; + + lock->ctx = ctx; + ctx->inode_lock = lock; + +insert: + ec_lock_insert(fop, lock, flags, base, fl_start, fl_size); +update_query: + lock->query |= (flags & EC_QUERY_INFO) != 0; +unlock: + UNLOCK(&loc->inode->lock); +} + +void +ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags, + off_t fl_start, uint64_t fl_size) +{ + ec_lock_prepare_inode_internal(fop, loc, flags, NULL, fl_start, fl_size); +} + +void +ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base, + uint32_t flags) +{ + loc_t tmp; + int32_t err; + + if (fop->error != 0) { + return; + } + + err = ec_loc_parent(fop->xl, loc, &tmp); + if (err != 0) { + ec_fop_set_error(fop, -err); + + return; + } + + if ((flags & EC_INODE_SIZE) != 0) { + flags ^= EC_INODE_SIZE; + } else { + base = NULL; + } + + ec_lock_prepare_inode_internal(fop, &tmp, flags, base, 0, EC_RANGE_FULL); + + loc_wipe(&tmp); +} + +void +ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start, + uint64_t fl_size) +{ + loc_t loc; + int32_t err; + + if (fop->error != 0) { + return; + } + + err = ec_loc_from_fd(fop->xl, &loc, fd); + if (err != 0) { + ec_fop_set_error(fop, -err); + + return; + } + + ec_lock_prepare_inode_internal(fop, &loc, flags, NULL, fl_start, fl_size); + + loc_wipe(&loc); +} + +gf_boolean_t +ec_config_check(xlator_t *xl, ec_config_t *config) +{ + ec_t *ec; + + ec = xl->private; + if ((config->version != EC_CONFIG_VERSION) || + (config->algorithm != EC_CONFIG_ALGORITHM) || + (config->gf_word_size != EC_GF_BITS) || (config->bricks != ec->nodes) || + (config->redundancy != ec->redundancy) || + (config->chunk_size != EC_METHOD_CHUNK_SIZE)) { + uint32_t data_bricks; + + /* This combination of version/algorithm requires the following + values. Incorrect values for these fields are a sign of + corruption: + + redundancy > 0 + redundancy * 2 < bricks + gf_word_size must be a power of 2 + chunk_size (in bits) must be a multiple of gf_word_size * + (bricks - redundancy) */ + + data_bricks = config->bricks - config->redundancy; + if ((config->redundancy < 1) || + (config->redundancy * 2 >= config->bricks) || + !ec_is_power_of_2(config->gf_word_size) || + ((config->chunk_size * 8) % (config->gf_word_size * data_bricks) != + 0)) { + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG, + "Invalid or corrupted config"); + } else { + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG, + "Unsupported config " + "(V=%u, A=%u, W=%u, " + "N=%u, R=%u, S=%u)", + config->version, config->algorithm, config->gf_word_size, + config->bricks, config->redundancy, config->chunk_size); + } + + return _gf_false; + } + + return _gf_true; +} + +gf_boolean_t +ec_set_dirty_flag(ec_lock_link_t *link, ec_inode_t *ctx, uint64_t *dirty) +{ + gf_boolean_t set_dirty = _gf_false; + + if (link->update[EC_DATA_TXN] && !ctx->dirty[EC_DATA_TXN]) { + if (!link->optimistic_changelog) + dirty[EC_DATA_TXN] = 1; + } + + if (link->update[EC_METADATA_TXN] && !ctx->dirty[EC_METADATA_TXN]) { + if (!link->optimistic_changelog) + dirty[EC_METADATA_TXN] = 1; + } + + if (dirty[EC_METADATA_TXN] || dirty[EC_DATA_TXN]) { + set_dirty = _gf_true; + } + + return set_dirty; +} + +int32_t +ec_prepare_update_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *dict, + dict_t *xdata) +{ + struct list_head list; + ec_fop_data_t *fop = cookie, *parent, *tmp; + ec_lock_link_t *parent_link = fop->data; + ec_lock_link_t *link = NULL; + ec_lock_t *lock = NULL; + ec_inode_t *ctx; + gf_boolean_t release = _gf_false; + uint64_t provided_flags = 0; + uint64_t dirty[EC_VERSION_SIZE] = {0, 0}; + lock = parent_link->lock; + parent = parent_link->fop; + ctx = lock->ctx; + + INIT_LIST_HEAD(&list); + provided_flags = EC_PROVIDED_FLAGS(parent_link->waiting_flags); + + LOCK(&lock->loc.inode->lock); + + list_for_each_entry(link, &lock->owners, owner_list) + { + if ((link->waiting_flags & provided_flags) != 0) { + link->waiting_flags ^= (link->waiting_flags & provided_flags); + if (EC_NEEDED_FLAGS(link->waiting_flags) == 0) + list_add_tail(&link->fop->cbk_list, &list); + } + } + if (op_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_SIZE_VERS_GET_FAIL, + "Failed to get size and version : %s", ec_msg_str(fop)); + + goto unlock; + } + + if (EC_FLAGS_HAVE(provided_flags, EC_FLAG_XATTROP)) { + op_errno = -ec_dict_del_array(dict, EC_XATTR_VERSION, ctx->pre_version, + EC_VERSION_SIZE); + if (op_errno != 0) { + gf_msg(this->name, GF_LOG_ERROR, op_errno, + EC_MSG_VER_XATTR_GET_FAIL, "Unable to get version xattr. %s", + ec_msg_str(fop)); + goto unlock; + } + ctx->post_version[0] += ctx->pre_version[0]; + ctx->post_version[1] += ctx->pre_version[1]; + + ctx->have_version = _gf_true; + + if (lock->loc.inode->ia_type == IA_IFREG || + lock->loc.inode->ia_type == IA_INVAL) { + op_errno = -ec_dict_del_number(dict, EC_XATTR_SIZE, &ctx->pre_size); + if (op_errno != 0) { + if (lock->loc.inode->ia_type == IA_IFREG) { + gf_msg(this->name, GF_LOG_ERROR, op_errno, + EC_MSG_SIZE_XATTR_GET_FAIL, + "Unable to get size xattr. %s", ec_msg_str(fop)); + goto unlock; + } + } else { + ctx->post_size = ctx->pre_size; + + ctx->have_size = _gf_true; + } + + op_errno = -ec_dict_del_config(dict, EC_XATTR_CONFIG, &ctx->config); + if (op_errno != 0) { + if ((lock->loc.inode->ia_type == IA_IFREG) || + (op_errno != ENODATA)) { + gf_msg(this->name, GF_LOG_ERROR, op_errno, + EC_MSG_CONFIG_XATTR_GET_FAIL, + "Unable to get config xattr. %s", ec_msg_str(fop)); + + goto unlock; + } + } else { + if (!ec_config_check(parent->xl, &ctx->config)) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, + EC_MSG_CONFIG_XATTR_INVALID, "Invalid config xattr"); + + op_errno = EINVAL; + + goto unlock; + } + ctx->have_config = _gf_true; + } + } + ctx->have_info = _gf_true; + } + + ec_set_dirty_flag(fop->data, ctx, dirty); + if (dirty[EC_METADATA_TXN] && + (EC_FLAGS_HAVE(provided_flags, EC_FLAG_METADATA_DIRTY))) { + GF_ASSERT(!ctx->dirty[EC_METADATA_TXN]); + ctx->dirty[EC_METADATA_TXN] = 1; + } + + if (dirty[EC_DATA_TXN] && + (EC_FLAGS_HAVE(provided_flags, EC_FLAG_DATA_DIRTY))) { + GF_ASSERT(!ctx->dirty[EC_DATA_TXN]); + ctx->dirty[EC_DATA_TXN] = 1; + } + op_errno = 0; +unlock: + + lock->waiting_flags ^= provided_flags; + + if (op_errno == 0) { + /* If the fop fails on any of the good bricks, it is important to mark + * it dirty and update versions right away if dirty was not set before. + */ + if (lock->good_mask & ~(fop->good | fop->remaining)) { + release = _gf_true; + } + + if (parent_link->update[0] && !parent_link->dirty[0]) { + lock->release |= release; + } + + if (parent_link->update[1] && !parent_link->dirty[1]) { + lock->release |= release; + } + + /* We don't allow the main fop to be executed on bricks that have not + * succeeded the initial xattrop. */ + ec_lock_update_good(lock, fop); + + /*As of now only data healing marks bricks as healing*/ + lock->healing |= fop->healing; + } + + UNLOCK(&lock->loc.inode->lock); + + while (!list_empty(&list)) { + tmp = list_entry(list.next, ec_fop_data_t, cbk_list); + list_del_init(&tmp->cbk_list); + + if (op_errno == 0) { + tmp->mask &= fop->good; + + /*As of now only data healing marks bricks as healing*/ + if (ec_is_data_fop(tmp->id)) { + tmp->healing |= fop->healing; + } + } + + ec_resume(tmp, op_errno); + } + + return 0; +} + +static gf_boolean_t +ec_set_needed_flag(ec_lock_t *lock, ec_lock_link_t *link, uint64_t flag) +{ + uint64_t current; + + link->waiting_flags |= EC_FLAG_NEEDS(flag); + + current = EC_NEEDED_FLAGS(lock->waiting_flags); + if (!EC_FLAGS_HAVE(current, flag)) { + lock->waiting_flags |= EC_FLAG_NEEDS(flag); + link->waiting_flags |= EC_FLAG_PROVIDES(flag); + + return _gf_true; + } + + return _gf_false; +} + +static uint64_t +ec_set_xattrop_flags_and_params(ec_lock_t *lock, ec_lock_link_t *link, + uint64_t *dirty) +{ + uint64_t oldflags = 0; + uint64_t newflags = 0; + ec_inode_t *ctx = lock->ctx; + + oldflags = EC_NEEDED_FLAGS(lock->waiting_flags); + + if (lock->query && !ctx->have_info) { + ec_set_needed_flag(lock, link, EC_FLAG_XATTROP); + } + + if (dirty[EC_DATA_TXN]) { + if (!ec_set_needed_flag(lock, link, EC_FLAG_DATA_DIRTY)) { + dirty[EC_DATA_TXN] = 0; + } + } + + if (dirty[EC_METADATA_TXN]) { + if (!ec_set_needed_flag(lock, link, EC_FLAG_METADATA_DIRTY)) { + dirty[EC_METADATA_TXN] = 0; + } + } + newflags = EC_NEEDED_FLAGS(lock->waiting_flags); + + return oldflags ^ newflags; +} + +void +ec_get_size_version(ec_lock_link_t *link) +{ + loc_t loc; + ec_lock_t *lock; + ec_inode_t *ctx; + ec_fop_data_t *fop; + dict_t *dict = NULL; + dict_t *xdata = NULL; + ec_t *ec = NULL; + int32_t error = 0; + gf_boolean_t set_dirty = _gf_false; + uint64_t allzero[EC_VERSION_SIZE] = {0, 0}; + uint64_t dirty[EC_VERSION_SIZE] = {0, 0}; + lock = link->lock; + ctx = lock->ctx; + fop = link->fop; + ec = fop->xl->private; + uint64_t changed_flags = 0; + + if (ec->optimistic_changelog && !(ec->node_mask & ~link->lock->good_mask) && + !ec_is_data_fop(fop->id)) + link->optimistic_changelog = _gf_true; + + memset(&loc, 0, sizeof(loc)); + + LOCK(&lock->loc.inode->lock); + + set_dirty = ec_set_dirty_flag(link, ctx, dirty); + + /* If ec metadata has already been retrieved, do not try again. */ + if (ctx->have_info) { + if (ec_is_data_fop(fop->id)) { + fop->healing |= lock->healing; + } + if (!set_dirty) + goto unlock; + } + + /* Determine if there's something we need to retrieve for the current + * operation. */ + if (!set_dirty && !lock->query && (lock->loc.inode->ia_type != IA_IFREG) && + (lock->loc.inode->ia_type != IA_INVAL)) { + goto unlock; + } + + changed_flags = ec_set_xattrop_flags_and_params(lock, link, dirty); + if (link->waiting_flags) { + /* This fop needs to wait until all its flags are cleared which + * potentially can be cleared by other xattrops that are already + * wound*/ + ec_sleep(fop); + } else { + GF_ASSERT(!changed_flags); + } + +unlock: + UNLOCK(&lock->loc.inode->lock); + + if (!changed_flags) + goto out; + + dict = dict_new(); + if (dict == NULL) { + error = -ENOMEM; + goto out; + } + + if (EC_FLAGS_HAVE(changed_flags, EC_FLAG_XATTROP)) { + /* Once we know that an xattrop will be needed, + * we try to get all available information in a + * single call. */ + error = ec_dict_set_array(dict, EC_XATTR_VERSION, allzero, + EC_VERSION_SIZE); + if (error != 0) { + goto out; + } + + if (lock->loc.inode->ia_type == IA_IFREG || + lock->loc.inode->ia_type == IA_INVAL) { + error = ec_dict_set_number(dict, EC_XATTR_SIZE, 0); + if (error == 0) { + error = ec_dict_set_number(dict, EC_XATTR_CONFIG, 0); + } + if (error != 0) { + goto out; + } + + xdata = dict_new(); + if (xdata == NULL || dict_set_int32(xdata, GF_GET_SIZE, 1)) { + error = -ENOMEM; + goto out; + } + } + } + + if (memcmp(allzero, dirty, sizeof(allzero))) { + error = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); + if (error != 0) { + goto out; + } + } + + fop->frame->root->uid = 0; + fop->frame->root->gid = 0; + + /* For normal fops, ec_[f]xattrop() must succeed on at least + * EC_MINIMUM_MIN bricks, however when this is called as part of a + * self-heal operation the mask of target bricks (fop->mask) could + * contain less than EC_MINIMUM_MIN bricks, causing the xattrop to + * always fail. Thus we always use the same minimum used for the main + * fop. + */ + if (lock->fd == NULL) { + error = ec_loc_from_loc(fop->xl, &loc, &lock->loc); + if (error != 0) { + goto out; + } + if (gf_uuid_is_null(loc.pargfid)) { + if (loc.parent != NULL) { + inode_unref(loc.parent); + loc.parent = NULL; + } + GF_FREE((char *)loc.path); + loc.path = NULL; + loc.name = NULL; + } + + ec_xattrop(fop->frame, fop->xl, fop->mask, fop->minimum, + ec_prepare_update_cbk, link, &loc, GF_XATTROP_ADD_ARRAY64, + dict, xdata); + } else { + ec_fxattrop(fop->frame, fop->xl, fop->mask, fop->minimum, + ec_prepare_update_cbk, link, lock->fd, + GF_XATTROP_ADD_ARRAY64, dict, xdata); + } + + error = 0; + +out: + fop->frame->root->uid = fop->uid; + fop->frame->root->gid = fop->gid; + + loc_wipe(&loc); + + if (dict != NULL) { + dict_unref(dict); + } + + if (xdata != NULL) { + dict_unref(xdata); + } + + if (error != 0) { + ec_fop_set_error(fop, -error); + } +} + +gf_boolean_t +__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size) +{ + ec_inode_t *ctx; + gf_boolean_t found = _gf_false; + + ctx = __ec_inode_get(inode, fop->xl); + if (ctx == NULL) { + goto out; + } + + if (ctx->have_size) { + *size = ctx->post_size; + found = _gf_true; + } + +out: + return found; +} + +gf_boolean_t +ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size) +{ + gf_boolean_t found = _gf_false; + + LOCK(&inode->lock); + { + found = __ec_get_inode_size(fop, inode, size); + } + UNLOCK(&inode->lock); + + return found; +} + +gf_boolean_t +__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size) +{ + ec_inode_t *ctx; + gf_boolean_t found = _gf_false; + + ctx = __ec_inode_get(inode, fop->xl); + if (ctx == NULL) { + goto out; + } + + /* Normal fops always have ctx->have_size set. However self-heal calls this + * to prepare the inode, so ctx->have_size will be false. In this case we + * prepare both pre_size and post_size, and set have_size and have_info to + * true. */ + if (!ctx->have_size) { + ctx->pre_size = size; + ctx->have_size = ctx->have_info = _gf_true; + } + ctx->post_size = size; + + found = _gf_true; + +out: + return found; +} + +gf_boolean_t +ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size) +{ + gf_boolean_t found = _gf_false; + + LOCK(&inode->lock); + { + found = __ec_set_inode_size(fop, inode, size); + } + UNLOCK(&inode->lock); + + return found; +} + +static void +ec_release_stripe_cache(ec_inode_t *ctx) +{ + ec_stripe_list_t *stripe_cache = NULL; + ec_stripe_t *stripe = NULL; + + stripe_cache = &ctx->stripe_cache; + while (!list_empty(&stripe_cache->lru)) { + stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru); + list_del(&stripe->lru); + GF_FREE(stripe); + } + stripe_cache->count = 0; + stripe_cache->max = 0; +} + +void +ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode) +{ + ec_inode_t *ctx; + + LOCK(&inode->lock); + + ctx = __ec_inode_get(inode, fop->xl); + if (ctx == NULL) { + goto unlock; + } + + ec_release_stripe_cache(ctx); + ctx->have_info = _gf_false; + ctx->have_config = _gf_false; + ctx->have_version = _gf_false; + ctx->have_size = _gf_false; + + memset(&ctx->config, 0, sizeof(ctx->config)); + memset(ctx->pre_version, 0, sizeof(ctx->pre_version)); + memset(ctx->post_version, 0, sizeof(ctx->post_version)); + ctx->pre_size = ctx->post_size = 0; + memset(ctx->dirty, 0, sizeof(ctx->dirty)); + +unlock: + UNLOCK(&inode->lock); +} + +int32_t +ec_get_real_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, dict_t *xdata, struct iatt *postparent) +{ + ec_fop_data_t *fop = cookie; + ec_lock_link_t *link; + + if (op_ret >= 0) { + link = fop->data; + link->size = buf->ia_size; + } else { + /* Prevent failure of parent fop. */ + fop->error = 0; + } + + return 0; +} + +/* This function is used to get the trusted.ec.size xattr from a file when + * no lock is needed on the inode. This is only required to maintain iatt + * structs on fops that manipulate directory entries but do not operate + * directly on the inode, like link, rename, ... + * + * Any error processing this request is ignored. In the worst case, an invalid + * or not up to date value in the iatt could cause some cache invalidation. + */ +void +ec_get_real_size(ec_lock_link_t *link) +{ + ec_fop_data_t *fop; + dict_t *xdata; + + if (link->base == NULL || link->base->inode == NULL) { + return; + } + + if (link->base->inode->ia_type != IA_IFREG) { + return; + } + + fop = link->fop; + + if (ec_get_inode_size(fop, link->base->inode, &link->size)) { + return; + } + + xdata = dict_new(); + if (xdata == NULL) { + return; + } + if (ec_dict_set_number(xdata, EC_XATTR_SIZE, 0) != 0) { + goto out; + } + + /* Send a simple lookup. A single answer is considered ok since this value + * is only used to return an iatt struct related to an inode that is not + * locked and have not suffered any operation. */ + ec_lookup(fop->frame, fop->xl, fop->mask, 1, ec_get_real_size_cbk, link, + link->base, xdata); + +out: + if (xdata != NULL) { + dict_unref(xdata); + } +} + +static void +ec_lock_update_fd(ec_lock_t *lock, ec_fop_data_t *fop) +{ + /* If the fop has an fd available, attach it to the lock structure to be + * able to do fxattrop calls instead of xattrop. */ + if (fop->use_fd && (lock->fd == NULL)) { + lock->fd = __fd_ref(fop->fd); + } +} + +static gf_boolean_t +ec_link_has_lock_conflict(ec_lock_link_t *link, gf_boolean_t waitlist_check) +{ + ec_lock_link_t *trav_link = NULL; + + list_for_each_entry(trav_link, &link->lock->owners, owner_list) + { + if (ec_lock_conflict(trav_link, link)) + return _gf_true; + } + + if (!waitlist_check) + return _gf_false; + + list_for_each_entry(trav_link, &link->lock->waiting, wait_list) + { + if (ec_lock_conflict(trav_link, link)) + return _gf_true; + } + + return _gf_false; +} + +static void +ec_lock_wake_shared(ec_lock_t *lock, struct list_head *list) +{ + ec_fop_data_t *fop; + ec_lock_link_t *link; + gf_boolean_t conflict = _gf_false; + + while (!conflict && !list_empty(&lock->waiting)) { + link = list_entry(lock->waiting.next, ec_lock_link_t, wait_list); + fop = link->fop; + + /* If lock is not acquired, at most one fop can be assigned as owner. + * The following fops will need to wait in the lock->waiting queue + * until the lock has been fully acquired. */ + conflict = !lock->acquired; + + /* If the fop is not shareable, only this fop can be assigned as owner. + * Other fops will need to wait until this one finishes. */ + if (ec_link_has_lock_conflict(link, _gf_false)) { + conflict = _gf_true; + } + + /* If only one fop is allowed, it can be assigned as the owner of the + * lock only if there weren't any other owner. */ + if (conflict && !list_empty(&lock->owners)) { + break; + } + + list_move_tail(&link->wait_list, list); + + list_add_tail(&link->owner_list, &lock->owners); + lock->refs_owners++; + + ec_lock_update_fd(lock, fop); + } +} + +static void +ec_lock_apply(ec_lock_link_t *link) +{ + ec_fop_data_t *fop = link->fop; + + fop->mask &= link->lock->good_mask; + fop->locked++; + + ec_get_size_version(link); + ec_get_real_size(link); +} + +gf_boolean_t +ec_lock_acquire(ec_lock_link_t *link); + +static void +ec_lock_resume_shared(struct list_head *list) +{ + ec_lock_link_t *link; + + while (!list_empty(list)) { + link = list_entry(list->next, ec_lock_link_t, wait_list); + list_del_init(&link->wait_list); + + if (link->lock->acquired) { + ec_lock_apply(link); + ec_lock(link->fop); + } else { + GF_ASSERT(list_empty(list)); + + ec_lock_acquire(link); + } + + ec_resume(link->fop, 0); + } +} + +void +ec_lock_acquired(ec_lock_link_t *link) +{ + struct list_head list; + ec_lock_t *lock; + ec_fop_data_t *fop; + + lock = link->lock; + fop = link->fop; + + ec_trace("LOCKED", fop, "lock=%p", lock); + + INIT_LIST_HEAD(&list); + + LOCK(&lock->loc.inode->lock); + + lock->acquired = _gf_true; + if (lock->contention) { + lock->release = _gf_true; + lock->contention = _gf_false; + } + + ec_lock_update_fd(lock, fop); + ec_lock_wake_shared(lock, &list); + + UNLOCK(&lock->loc.inode->lock); + + ec_lock_apply(link); + + if (fop->use_fd && + (link->update[EC_DATA_TXN] || link->update[EC_METADATA_TXN])) { + /* Try to reopen closed fd's only if lock has succeeded. */ + ec_fix_open(fop, lock->mask); + } + + ec_lock_resume_shared(&list); +} + +int32_t +ec_locked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_lock_link_t *link = NULL; + ec_lock_t *lock = NULL; + + link = fop->data; + lock = link->lock; + if (op_ret >= 0) { + lock->mask = lock->good_mask = fop->good; + lock->healing = 0; + + ec_lock_acquired(link); + ec_lock(fop->parent); + } else { + LOCK(&lock->loc.inode->lock); + { + lock->contention = _gf_false; + } + UNLOCK(&lock->loc.inode->lock); + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_PREOP_LOCK_FAILED, + "Failed to complete preop lock"); + } + + return 0; +} + +gf_boolean_t +ec_lock_acquire(ec_lock_link_t *link) +{ + ec_lock_t *lock; + ec_fop_data_t *fop; + gf_lkowner_t lk_owner; + + lock = link->lock; + fop = link->fop; + + if (!lock->acquired) { + set_lk_owner_from_ptr(&lk_owner, lock); + + ec_trace("LOCK_ACQUIRE", fop, "lock=%p, inode=%p", lock, + lock->loc.inode); + + lock->flock.l_type = F_WRLCK; + ec_inodelk(fop->frame, fop->xl, &lk_owner, -1, EC_MINIMUM_ALL, + ec_locked, link, fop->xl->name, &lock->loc, F_SETLKW, + &lock->flock, NULL); + + return _gf_false; + } + + ec_trace("LOCK_REUSE", fop, "lock=%p", lock); + + ec_lock_acquired(link); + + return _gf_true; +} + +static ec_lock_link_t * +ec_lock_timer_cancel(xlator_t *xl, ec_lock_t *lock) +{ + ec_lock_link_t *timer_link; + + /* If we don't have any timer, there's nothing to cancel. */ + if (lock->timer == NULL) { + return NULL; + } + + /* We are trying to access a lock that has an unlock timer active. + * This means that the lock must be idle, i.e. no fop can be in the + * owner, waiting or frozen lists. It also means that the lock cannot + * have been marked as being released (this is done without timers). + * There should only be one owner reference, but it's possible that + * some fops are being prepared to use this lock. */ + GF_ASSERT((lock->refs_owners == 1) && list_empty(&lock->owners) && + list_empty(&lock->waiting)); + + /* We take the timer_link before cancelling the timer, since a + * successful cancellation will destroy it. It must not be NULL + * because it references the fop responsible for the delayed unlock + * that we are currently trying to cancel. */ + timer_link = lock->timer->data; + GF_ASSERT(timer_link != NULL); + + if (gf_timer_call_cancel(xl->ctx, lock->timer) < 0) { + /* It's too late to avoid the execution of the timer callback. + * Since we need to be sure that the callback has access to all + * needed resources, we cannot resume the execution of the + * timer fop now. This will be done in the callback. */ + timer_link = NULL; + } else { + /* The timer has been cancelled. The fop referenced by + * timer_link holds the last reference. The caller is + * responsible to release it when not needed anymore. */ + ec_trace("UNLOCK_CANCELLED", timer_link->fop, "lock=%p", lock); + } + + /* We have two options here: + * + * 1. The timer has been successfully cancelled. + * + * This is the easiest case and we can continue with the currently + * acquired lock. + * + * 2. The timer callback has already been fired. + * + * In this case we have not been able to cancel the timer before + * the timer callback has been fired, but we also know that + * lock->timer != NULL. This means that the timer callback is still + * trying to acquire the inode mutex that we currently own. We are + * safe until we release it. In this case we can safely clear + * lock->timer. This will cause that the timer callback does nothing + * once it acquires the mutex. + */ + lock->timer = NULL; + + return timer_link; +} + +static gf_boolean_t +ec_lock_assign_owner(ec_lock_link_t *link) +{ + ec_fop_data_t *fop; + ec_lock_t *lock; + ec_lock_link_t *timer_link = NULL; + gf_boolean_t assigned = _gf_false; + + /* The link cannot be in any list because we have just finished preparing + * it. */ + GF_ASSERT(list_empty(&link->wait_list)); + + fop = link->fop; + lock = link->lock; + + LOCK(&lock->loc.inode->lock); + + /* Since the link has just been prepared but it's not active yet, the + * refs_pending must be one at least (the ref owned by this link). */ + GF_ASSERT(lock->refs_pending > 0); + /* The link is not pending any more. It will be assigned to the owner, + * waiting or frozen list. */ + lock->refs_pending--; + + if (lock->release) { + ec_trace("LOCK_QUEUE_FREEZE", fop, "lock=%p", lock); + + /* When lock->release is set, we'll unlock the lock as soon as + * possible, meaning that we won't use a timer. */ + GF_ASSERT(lock->timer == NULL); + + /* The lock is marked to be released. We can still have owners and fops + * in the waiting ilist f they have been added before the lock has been + * marked to be released. However new fops are put into the frozen list + * to wait for the next unlock/lock cycle. */ + list_add_tail(&link->wait_list, &lock->frozen); + + goto unlock; + } + + /* The lock is not marked to be released, so the frozen list should be + * empty. */ + GF_ASSERT(list_empty(&lock->frozen)); + + timer_link = ec_lock_timer_cancel(fop->xl, lock); + + if (!list_empty(&lock->owners)) { + /* There are other owners of this lock. We can only take ownership if + * the lock is already acquired and doesn't have conflict with existing + * owners, or waiters(to prevent starvation). + * Otherwise we need to wait. + */ + if (!lock->acquired || ec_link_has_lock_conflict(link, _gf_true)) { + ec_trace("LOCK_QUEUE_WAIT", fop, "lock=%p", lock); + + list_add_tail(&link->wait_list, &lock->waiting); + + goto unlock; + } + } + + list_add_tail(&link->owner_list, &lock->owners); + + /* If timer_link is not NULL, it means that we have inherited the owner + * reference assigned to the timer fop. In this case we simply reuse it. + * Otherwise we need to increase the number of owners. */ + if (timer_link == NULL) { + lock->refs_owners++; + } + + assigned = _gf_true; + +unlock: + if (!assigned) { + /* We have not been able to take ownership of this lock. The fop must + * be put to sleep. */ + ec_sleep(fop); + } + + UNLOCK(&lock->loc.inode->lock); + + /* If we have cancelled the timer, we need to resume the fop that was + * waiting for it. */ + if (timer_link != NULL) { + ec_resume(timer_link->fop, 0); + } + + return assigned; +} + +static void +ec_lock_next_owner(ec_lock_link_t *link, ec_cbk_data_t *cbk, + gf_boolean_t release) +{ + struct list_head list; + ec_lock_t *lock = link->lock; + ec_fop_data_t *fop = link->fop; + ec_inode_t *ctx = lock->ctx; + + INIT_LIST_HEAD(&list); + + LOCK(&lock->loc.inode->lock); + + ec_trace("LOCK_DONE", fop, "lock=%p", lock); + + /* Current link must belong to the owner list of the lock. We don't + * decrement lock->refs_owners here because the inode mutex is released + * before ec_unlock() is called and we need to know when the last owner + * unlocks the lock to do proper cleanup. lock->refs_owners is used for + * this task. */ + GF_ASSERT((lock->refs_owners > 0) && !list_empty(&link->owner_list)); + list_del_init(&link->owner_list); + + lock->release |= release; + + if ((fop->error == 0) && (cbk != NULL) && (cbk->op_ret >= 0)) { + if (link->update[0]) { + ctx->post_version[0]++; + } + if (link->update[1]) { + ctx->post_version[1]++; + } + /* If the fop fails on any of the good bricks, it is important to mark + * it dirty and update versions right away. */ + if (link->update[0] || link->update[1]) { + if (lock->good_mask & ~(fop->good | fop->remaining)) { + lock->release = _gf_true; + } + } + } + + if (fop->healing) { + lock->healing = fop->healing & (fop->good | fop->remaining); + } + ec_lock_update_good(lock, fop); + + ec_lock_wake_shared(lock, &list); + + UNLOCK(&lock->loc.inode->lock); + + ec_lock_resume_shared(&list); +} + +void +ec_lock(ec_fop_data_t *fop) +{ + ec_lock_link_t *link; + + /* There is a chance that ec_resume is called on fop even before ec_sleep. + * Which can result in refs == 0 for fop leading to use after free in this + * function when it calls ec_sleep so do ec_sleep at start and ec_resume at + * the end of this function.*/ + ec_sleep(fop); + + while (fop->locked < fop->lock_count) { + /* Since there are only up to 2 locks per fop, this xor will change + * the order of the locks if fop->first_lock is 1. */ + link = &fop->locks[fop->locked ^ fop->first_lock]; + + if (!ec_lock_assign_owner(link) || !ec_lock_acquire(link)) { + break; + } + } + + ec_resume(fop, 0); +} + +void +ec_lock_unfreeze(ec_lock_link_t *link) +{ + struct list_head list; + ec_lock_t *lock; + gf_boolean_t destroy = _gf_false; + + lock = link->lock; + + INIT_LIST_HEAD(&list); + + LOCK(&lock->loc.inode->lock); + + /* The lock must be marked to be released here, since we have just released + * it and any attempt to assign it to more fops must have added them to the + * frozen list. We can only have one active reference here: the one that + * is processing this unfreeze. */ + GF_ASSERT(lock->release && (lock->refs_owners == 1)); + lock->release = _gf_false; + lock->refs_owners = 0; + + lock->acquired = _gf_false; + + /* We are unfreezing a lock. This means that the lock has already been + * released. In this state it shouldn't have a pending timer nor have any + * owner, and the waiting list should be empty. Only the frozen list can + * contain some fop. */ + GF_ASSERT((lock->timer == NULL) && list_empty(&lock->waiting) && + list_empty(&lock->owners)); + + /* We move all frozen fops to the waiting list. */ + list_splice_init(&lock->frozen, &lock->waiting); + + /* If we don't have any fop waiting nor there are any prepared fops using + * this lock, we can finally dispose it. */ + destroy = list_empty(&lock->waiting) && (lock->refs_pending == 0); + if (destroy) { + ec_trace("LOCK_DESTROY", link->fop, "lock=%p", lock); + + lock->ctx->inode_lock = NULL; + } else { + ec_trace("LOCK_UNFREEZE", link->fop, "lock=%p", lock); + + ec_lock_wake_shared(lock, &list); + } + + UNLOCK(&lock->loc.inode->lock); + + ec_lock_resume_shared(&list); + + if (destroy) { + ec_lock_destroy(lock); + } +} + +int32_t +ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_lock_link_t *link = fop->data; + + if (op_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED, + "entry/inode unlocking failed :(%s)", ec_msg_str(link->fop)); + } else { + ec_trace("UNLOCKED", link->fop, "lock=%p", link->lock); + } + + ec_lock_unfreeze(link); + + return 0; +} + +void +ec_unlock_lock(ec_lock_link_t *link) +{ + ec_lock_t *lock; + ec_fop_data_t *fop; + gf_lkowner_t lk_owner; + + lock = link->lock; + fop = link->fop; + + lock->unlock_now = _gf_false; + ec_clear_inode_info(fop, lock->loc.inode); + + if ((lock->mask != 0) && lock->acquired) { + set_lk_owner_from_ptr(&lk_owner, lock); + lock->flock.l_type = F_UNLCK; + ec_trace("UNLOCK_INODELK", fop, "lock=%p, inode=%p", lock, + lock->loc.inode); + + ec_inodelk(fop->frame, fop->xl, &lk_owner, lock->mask, EC_MINIMUM_ONE, + ec_unlocked, link, fop->xl->name, &lock->loc, F_SETLK, + &lock->flock, NULL); + } else { + ec_lock_unfreeze(link); + } +} + +void +ec_inode_bad_inc(inode_t *inode, xlator_t *xl) +{ + ec_inode_t *ctx = NULL; + + LOCK(&inode->lock); + { + ctx = __ec_inode_get(inode, xl); + if (ctx == NULL) { + goto unlock; + } + ctx->bad_version++; + } +unlock: + UNLOCK(&inode->lock); +} + +int32_t +ec_update_size_version_done(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xattr, + dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_lock_link_t *link; + ec_lock_t *lock; + ec_inode_t *ctx; + + link = fop->data; + lock = link->lock; + ctx = lock->ctx; + + if (op_ret < 0) { + if (link->lock->fd == NULL) { + ec_inode_bad_inc(link->lock->loc.inode, this); + } else { + ec_inode_bad_inc(link->lock->fd->inode, this); + } + + gf_msg(fop->xl->name, fop_log_level(fop->id, op_errno), op_errno, + EC_MSG_SIZE_VERS_UPDATE_FAIL, + "Failed to update version and size. %s", ec_msg_str(fop)); + } else { + fop->parent->good &= fop->good; + + ec_lock_update_good(lock, fop); + + if (ec_dict_del_array(xattr, EC_XATTR_VERSION, ctx->post_version, + EC_VERSION_SIZE) == 0) { + ctx->pre_version[0] = ctx->post_version[0]; + ctx->pre_version[1] = ctx->post_version[1]; + + ctx->have_version = _gf_true; + } + if (ec_dict_del_number(xattr, EC_XATTR_SIZE, &ctx->post_size) == 0) { + ctx->pre_size = ctx->post_size; + + ctx->have_size = _gf_true; + } + if ((ec_dict_del_config(xdata, EC_XATTR_CONFIG, &ctx->config) == 0) && + ec_config_check(fop->xl, &ctx->config)) { + ctx->have_config = _gf_true; + } + + ctx->have_info = _gf_true; + } + /* If we are here because of fop's and other than unlock request, + * that means we are still holding a lock. That make sure + * lock->unlock_now can not be modified. + */ + if (lock->unlock_now) { + ec_unlock_lock(fop->data); + } + + return 0; +} + +void +ec_update_size_version(ec_lock_link_t *link, uint64_t *version, uint64_t size, + uint64_t *dirty) +{ + ec_fop_data_t *fop; + ec_lock_t *lock; + ec_inode_t *ctx; + dict_t *dict = NULL; + uintptr_t update_on = 0; + int32_t err = -ENOMEM; + + fop = link->fop; + lock = link->lock; + ctx = lock->ctx; + + ec_trace("UPDATE", fop, "version=%ld/%ld, size=%ld, dirty=%ld/%ld", + version[0], version[1], size, dirty[0], dirty[1]); + + dict = dict_new(); + if (dict == NULL) { + goto out; + } + + /* If we don't have version information or it has been modified, we + * update it. */ + if (!ctx->have_version || (version[0] != 0) || (version[1] != 0)) { + err = ec_dict_set_array(dict, EC_XATTR_VERSION, version, + EC_VERSION_SIZE); + if (err != 0) { + goto out; + } + } + + if (size != 0) { + /* If size has been changed, we should already + * know the previous size of the file. */ + GF_ASSERT(ctx->have_size); + + err = ec_dict_set_number(dict, EC_XATTR_SIZE, size); + if (err != 0) { + goto out; + } + } + + if (dirty[0] || dirty[1]) { + err = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); + if (err != 0) { + goto out; + } + } + + /* If config information is not known, we request it now. */ + if ((lock->loc.inode->ia_type == IA_IFREG) && !ctx->have_config) { + /* A failure requesting this xattr is ignored because it's not + * absolutely required right now. */ + (void)ec_dict_set_number(dict, EC_XATTR_CONFIG, 0); + } + + fop->frame->root->uid = 0; + fop->frame->root->gid = 0; + + update_on = lock->good_mask | lock->healing; + + if (link->lock->fd == NULL) { + ec_xattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN, + ec_update_size_version_done, link, &link->lock->loc, + GF_XATTROP_ADD_ARRAY64, dict, NULL); + } else { + ec_fxattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN, + ec_update_size_version_done, link, link->lock->fd, + GF_XATTROP_ADD_ARRAY64, dict, NULL); + } + + fop->frame->root->uid = fop->uid; + fop->frame->root->gid = fop->gid; + + dict_unref(dict); + + return; + +out: + if (dict != NULL) { + dict_unref(dict); + } + + ec_fop_set_error(fop, -err); + + gf_msg(fop->xl->name, GF_LOG_ERROR, -err, EC_MSG_SIZE_VERS_UPDATE_FAIL, + "Unable to update version and size. %s", ec_msg_str(fop)); + + if (lock->unlock_now) { + ec_unlock_lock(fop->data); + } +} + +gf_boolean_t +ec_update_info(ec_lock_link_t *link) +{ + ec_lock_t *lock; + ec_inode_t *ctx; + uint64_t version[2] = {0, 0}; + uint64_t dirty[2] = {0, 0}; + uint64_t size; + ec_t *ec = NULL; + uintptr_t mask; + + lock = link->lock; + ctx = lock->ctx; + ec = link->fop->xl->private; + + /* pre_version[*] will be 0 if have_version is false */ + version[EC_DATA_TXN] = ctx->post_version[EC_DATA_TXN] - + ctx->pre_version[EC_DATA_TXN]; + version[EC_METADATA_TXN] = ctx->post_version[EC_METADATA_TXN] - + ctx->pre_version[EC_METADATA_TXN]; + + size = ctx->post_size - ctx->pre_size; + /* If we set the dirty flag for update fop, we have to unset it. + * If fop has failed on some bricks, leave the dirty as marked. */ + + if (lock->unlock_now) { + if (version[EC_DATA_TXN]) { + /*A data fop will have difference in post and pre version + *and for data fop we send writes on healing bricks also */ + mask = lock->good_mask | lock->healing; + } else { + mask = lock->good_mask; + } + /* Ensure that nodes are up while doing final + * metadata update.*/ + if (!(ec->node_mask & ~(mask)) && !(ec->node_mask & ~ec->xl_up)) { + if (ctx->dirty[EC_DATA_TXN] != 0) { + dirty[EC_DATA_TXN] = -1; + } + if (ctx->dirty[EC_METADATA_TXN] != 0) { + dirty[EC_METADATA_TXN] = -1; + } + /*If everything is fine and we already + *have version xattr set on entry, there + *is no need to update version again*/ + if (ctx->pre_version[EC_DATA_TXN]) { + version[EC_DATA_TXN] = 0; + } + if (ctx->pre_version[EC_METADATA_TXN]) { + version[EC_METADATA_TXN] = 0; + } + } else { + link->optimistic_changelog = _gf_false; + ec_set_dirty_flag(link, ctx, dirty); + } + memset(ctx->dirty, 0, sizeof(ctx->dirty)); + } + + if ((version[EC_DATA_TXN] != 0) || (version[EC_METADATA_TXN] != 0) || + (dirty[EC_DATA_TXN] != 0) || (dirty[EC_METADATA_TXN] != 0)) { + ec_update_size_version(link, version, size, dirty); + return _gf_true; + } + + return _gf_false; +} + +void +ec_unlock_now(ec_lock_link_t *link) +{ + ec_lock_t *lock; + lock = link->lock; + + ec_trace("UNLOCK_NOW", link->fop, "lock=%p", link->lock); + /*At this point, lock is not being used by any fop and + *can not be reused by any fop as it is going to be released. + *lock->unlock_now can not be modified at any other place. + */ + lock->unlock_now = _gf_true; + + if (!ec_update_info(link)) { + ec_unlock_lock(link); + } + + ec_resume(link->fop, 0); +} + +void +ec_lock_release(ec_t *ec, inode_t *inode) +{ + ec_lock_t *lock; + ec_inode_t *ctx; + ec_lock_link_t *timer_link = NULL; + + LOCK(&inode->lock); + + ctx = __ec_inode_get(inode, ec->xl); + if (ctx == NULL) { + goto done; + } + lock = ctx->inode_lock; + if ((lock == NULL) || lock->release) { + goto done; + } + + gf_msg_debug(ec->xl->name, 0, "Releasing inode %p due to lock contention", + inode); + + if (!lock->acquired) { + /* This happens if some bricks already got the lock while inodelk is in + * progress. Set release to true after lock is acquired*/ + lock->contention = _gf_true; + goto done; + } + + /* The lock is not marked to be released, so the frozen list should be + * empty. */ + GF_ASSERT(list_empty(&lock->frozen)); + + timer_link = ec_lock_timer_cancel(ec->xl, lock); + + /* We mark the lock to be released as soon as possible. */ + lock->release = _gf_true; + +done: + UNLOCK(&inode->lock); + + /* If we have cancelled the timer, we need to start the unlock of the + * inode. If there was a timer but we have been unable to cancel it + * because it was just triggered, the timer callback will take care + * of releasing the inode. */ + if (timer_link != NULL) { + ec_unlock_now(timer_link); + } +} + +void +ec_unlock_timer_add(ec_lock_link_t *link); + +void +ec_unlock_timer_del(ec_lock_link_t *link) +{ + ec_lock_t *lock; + inode_t *inode; + gf_boolean_t now = _gf_false; + + /* If we are here, it means that the timer has expired before having + * been cancelled. This guarantees that 'link' is still valid because + * the fop that contains it must be pending (if timer cancellation in + * ec_lock_assign_owner() fails, the fop is left sleeping). + * + * At the same time, the fop still has a reference to the lock, so + * it must also be valid. + */ + lock = link->lock; + + /* 'lock' must have a valid inode since it can only be destroyed + * when the lock itself is destroyed, but we have a reference to the + * lock to avoid this. + */ + inode = lock->loc.inode; + + LOCK(&inode->lock); + + if (lock->timer != NULL) { + ec_trace("UNLOCK_DELAYED", link->fop, "lock=%p", lock); + + /* The unlock timer has expired without anyone cancelling it. + * This means that it shouldn't have any owner, and the waiting + * and frozen lists should be empty. It must have only one + * owner reference, but there can be fops being prepared + * though. + * */ + GF_ASSERT(!lock->release && (lock->refs_owners == 1) && + list_empty(&lock->owners) && list_empty(&lock->waiting) && + list_empty(&lock->frozen)); + + gf_timer_call_cancel(link->fop->xl->ctx, lock->timer); + lock->timer = NULL; + + /* Any fop being processed from now on, will need to wait + * until the next unlock/lock cycle. */ + lock->release = now = _gf_true; + } + + UNLOCK(&inode->lock); + + if (now) { + ec_unlock_now(link); + } else { + /* The timer has been cancelled just after firing it but before + * getting here. This means that another fop has used the lock + * and everything should be handled as if this callback were + * have not been executed. However we still have an owner + * reference. + * + * We need to release our reference. If this is not the last + * reference (the most common case because another fop has + * taken another ref) we only need to decrement the counter. + * Otherwise we have been delayed enough so that the other fop + * has had time to acquire the reference, do its operation and + * release it. At the time of releasing it, the fop did found + * that the ref counter was > 1 (our reference), so the delayed + * unlock timer wasn't started. We need to start it again if we + * are the last reference. + * + * ec_unlock_timer_add() handles both cases. + */ + ec_unlock_timer_add(link); + + /* We need to resume the fop that was waiting for the delayed + * unlock. + */ + ec_resume(link->fop, 0); + } +} + +void +ec_unlock_timer_cbk(void *data) +{ + ec_unlock_timer_del(data); +} + +static gf_boolean_t +ec_eager_lock_used(ec_t *ec, ec_fop_data_t *fop) +{ + /* Fops with no locks at this point mean that they are sent as sub-fops + * of other higher level fops. In this case we simply assume that the + * parent fop will take correct care of the eager lock. */ + if (fop->lock_count == 0) { + return _gf_true; + } + + /* We may have more than one lock, but this only happens in the rename + * fop, and both locks will reference an inode of the same type (a + * directory in this case), so we only need to check the first lock. */ + if (fop->locks[0].lock->loc.inode->ia_type == IA_IFREG) { + return ec->eager_lock; + } + + return ec->other_eager_lock; +} + +static uint32_t +ec_eager_lock_timeout(ec_t *ec, ec_lock_t *lock) +{ + if (lock->loc.inode->ia_type == IA_IFREG) { + return ec->eager_lock_timeout; + } + + return ec->other_eager_lock_timeout; +} + +static gf_boolean_t +ec_lock_delay_create(ec_lock_link_t *link) +{ + struct timespec delay; + ec_fop_data_t *fop = link->fop; + ec_lock_t *lock = link->lock; + + delay.tv_sec = ec_eager_lock_timeout(fop->xl->private, lock); + delay.tv_nsec = 0; + lock->timer = gf_timer_call_after(fop->xl->ctx, delay, ec_unlock_timer_cbk, + link); + if (lock->timer == NULL) { + gf_msg(fop->xl->name, GF_LOG_WARNING, ENOMEM, + EC_MSG_UNLOCK_DELAY_FAILED, "Unable to delay an unlock"); + + return _gf_false; + } + + return _gf_true; +} + +void +ec_unlock_timer_add(ec_lock_link_t *link) +{ + ec_fop_data_t *fop = link->fop; + ec_lock_t *lock = link->lock; + gf_boolean_t now = _gf_false; + + LOCK(&lock->loc.inode->lock); + + /* We are trying to unlock the lock. We can have multiple scenarios here, + * but all of them need to have lock->timer == NULL: + * + * 1. There are other owners currently running that can call ec_unlock(). + * + * None of them can have started the timer until the last one. But this + * call should be the consequence of this lastest one. + * + * 2. There are fops in the waiting or frozen lists. + * + * These fops cannot call ec_unlock(). So we should be here. + * + * We must reach here with at least one owner reference. + */ + GF_ASSERT((lock->timer == NULL) && (lock->refs_owners > 0)); + + /* If the fop detects that a heal is needed, we mark the lock to be + * released as soon as possible. */ + lock->release |= ec_fop_needs_heal(fop); + + if (lock->refs_owners > 1) { + ec_trace("UNLOCK_SKIP", fop, "lock=%p", lock); + + /* If there are other owners we cannot do anything else with the lock. + * Note that the current fop has already been removed from the owners + * list in ec_lock_reuse(). */ + lock->refs_owners--; + + UNLOCK(&lock->loc.inode->lock); + } else if (lock->acquired) { + /* There are no other owners and the lock is acquired. If there were + * fops waiting, at least one of them should have been promoted to an + * owner, so the waiting list should be empty. */ + GF_ASSERT(list_empty(&lock->owners) && list_empty(&lock->waiting)); + + ec_t *ec = fop->xl->private; + + /* If everything goes as expected this fop will be put to sleep until + * the timer callback is executed. */ + ec_sleep(fop); + + /* If the lock needs to be released, or ec is shutting down, do not + * delay lock release. */ + if (!lock->release && !ec->shutdown) { + ec_trace("UNLOCK_DELAY", fop, "lock=%p, release=%d", lock, + lock->release); + + if (!ec_lock_delay_create(link)) { + /* We are unable to create a new timer. We immediately release + * the lock. */ + lock->release = now = _gf_true; + } + + } else { + ec_trace("UNLOCK_FORCE", fop, "lock=%p, release=%d", lock, + lock->release); + lock->release = now = _gf_true; + } + + UNLOCK(&lock->loc.inode->lock); + + if (now) { + ec_unlock_now(link); + } + } else { + /* There are no owners and the lock is not acquired. This can only + * happen if a lock attempt has failed and we get to the unlock step + * of the fop. As in the previous case, the waiting list must be + * empty. */ + GF_ASSERT(list_empty(&lock->owners) && list_empty(&lock->waiting)); + + /* We need to mark the lock to be released to correctly handle fops + * that may get in after we release the inode mutex but before + * ec_lock_unfreeze() is processed. */ + lock->release = _gf_true; + + UNLOCK(&lock->loc.inode->lock); + + ec_lock_unfreeze(link); + } +} + +void +ec_unlock(ec_fop_data_t *fop) +{ + int32_t i; + + for (i = 0; i < fop->lock_count; i++) { + ec_unlock_timer_add(&fop->locks[i]); + } +} + +void +ec_flush_size_version(ec_fop_data_t *fop) +{ + GF_ASSERT(fop->lock_count == 1); + ec_update_info(&fop->locks[0]); +} + +static void +ec_update_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache, ec_stripe_t *stripe, + ec_fop_data_t *fop) +{ + off_t base; + + /* On write fops, we only update existing fragments if the write has + * succeeded. Otherwise, we remove them from the cache. */ + if ((fop->id == GF_FOP_WRITE) && (fop->answer != NULL) && + (fop->answer->op_ret >= 0)) { + base = stripe->frag_offset - fop->frag_range.first; + base *= ec->fragments; + + /* We check if the stripe offset falls inside the real region + * modified by the write fop (a write request is allowed, + * though uncommon, to write less bytes than requested). The + * current write fop implementation doesn't allow partial + * writes of fragments, so if there's no error, we are sure + * that a full stripe has been completely modified or not + * touched at all. The value of op_ret may not be a multiple + * of the stripe size because it depends on the requested + * size by the user, so we update the stripe if the write has + * modified at least one byte (meaning ec has written the full + * stripe). */ + if (base < fop->answer->op_ret + fop->head) { + memcpy(stripe->data, fop->vector[0].iov_base + base, + ec->stripe_size); + list_move_tail(&stripe->lru, &stripe_cache->lru); + + GF_ATOMIC_INC(ec->stats.stripe_cache.updates); + } + } else { + stripe->frag_offset = -1; + list_move(&stripe->lru, &stripe_cache->lru); + + GF_ATOMIC_INC(ec->stats.stripe_cache.invals); + } +} + +static void +ec_update_cached_stripes(ec_fop_data_t *fop) +{ + uint64_t first; + uint64_t last; + ec_stripe_t *stripe = NULL; + ec_inode_t *ctx = NULL; + ec_stripe_list_t *stripe_cache = NULL; + inode_t *inode = NULL; + struct list_head *temp; + struct list_head sentinel; + + first = fop->frag_range.first; + /* 'last' represents the first stripe not touched by the operation */ + last = fop->frag_range.last; + + /* If there are no modified stripes, we don't need to do anything + * else. */ + if (last <= first) { + return; + } + + if (!fop->use_fd) { + inode = fop->loc[0].inode; + } else { + inode = fop->fd->inode; + } + + LOCK(&inode->lock); + + ctx = __ec_inode_get(inode, fop->xl); + if (ctx == NULL) { + goto out; + } + stripe_cache = &ctx->stripe_cache; + + /* Since we'll be moving elements of the list to the tail, we might + * end in an infinite loop. To avoid it, we insert a sentinel element + * into the list, so that it will be used to detect when we have + * traversed all existing elements once. */ + list_add_tail(&sentinel, &stripe_cache->lru); + temp = stripe_cache->lru.next; + while (temp != &sentinel) { + stripe = list_entry(temp, ec_stripe_t, lru); + temp = temp->next; + if ((first <= stripe->frag_offset) && (stripe->frag_offset < last)) { + ec_update_stripe(fop->xl->private, stripe_cache, stripe, fop); + } + } + list_del(&sentinel); + +out: + UNLOCK(&inode->lock); +} + +void +ec_lock_reuse(ec_fop_data_t *fop) +{ + ec_cbk_data_t *cbk; + ec_t *ec = NULL; + int32_t i, count; + gf_boolean_t release = _gf_false; + ec = fop->xl->private; + cbk = fop->answer; + + if (ec_eager_lock_used(ec, fop) && cbk != NULL) { + if (cbk->xdata != NULL) { + if ((dict_get_int32(cbk->xdata, GLUSTERFS_INODELK_COUNT, &count) == + 0) && + (count > 1)) { + release = _gf_true; + } + if (release) { + gf_msg_debug(fop->xl->name, 0, "Lock contention detected"); + } + } + } else { + /* If eager lock is disabled or if we haven't get + * an answer with enough quorum, we always release + * the lock. */ + release = _gf_true; + } + ec_update_cached_stripes(fop); + + for (i = 0; i < fop->lock_count; i++) { + ec_lock_next_owner(&fop->locks[i], cbk, release); + } +} + +void +__ec_manager(ec_fop_data_t *fop, int32_t error) +{ + ec_t *ec = fop->xl->private; + + do { + ec_trace("MANAGER", fop, "error=%d", error); + + if (!ec_must_wind(fop)) { + if (ec->xl_up_count < ec->fragments) { + error = ENOTCONN; + } + } + + if (error != 0) { + fop->error = error; + fop->state = -fop->state; + } + + if ((fop->state == EC_STATE_END) || (fop->state == -EC_STATE_END)) { + ec_fop_data_release(fop); + + break; + } + + /* At each state, fop must not be used anywhere else and there + * shouldn't be any pending subfop going on. */ + GF_ASSERT(fop->jobs == 0); + + /* While the manager is running we need to avoid that subfops launched + * from it could finish and call ec_resume() before the fop->handler + * has completed. This could lead to the same manager being executed + * by two threads concurrently. ec_check_complete() will take care of + * this reference. */ + fop->jobs = 1; + + fop->state = fop->handler(fop, fop->state); + GF_ASSERT(fop->state >= 0); + + error = ec_check_complete(fop, __ec_manager); + } while (error >= 0); +} + +void +ec_manager(ec_fop_data_t *fop, int32_t error) +{ + GF_ASSERT(fop->jobs == 0); + GF_ASSERT(fop->winds == 0); + GF_ASSERT(fop->error == 0); + + if (fop->state == EC_STATE_START) { + fop->state = EC_STATE_INIT; + } + + __ec_manager(fop, error); +} + +gf_boolean_t +__ec_is_last_fop(ec_t *ec) +{ + if ((list_empty(&ec->pending_fops)) && + (GF_ATOMIC_GET(ec->async_fop_count) == 0)) { + return _gf_true; + } + return _gf_false; +} diff --git a/xlators/cluster/ec/src/ec-common.h b/xlators/cluster/ec/src/ec-common.h new file mode 100644 index 00000000000..51493612ac6 --- /dev/null +++ b/xlators/cluster/ec/src/ec-common.h @@ -0,0 +1,234 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_COMMON_H__ +#define __EC_COMMON_H__ + +#include "glusterfs/compat-errno.h" // for ENODATA on BSD +#include "ec-data.h" + +typedef enum { EC_DATA_TXN, EC_METADATA_TXN } ec_txn_t; + +#define EC_FOP_HEAL -1 +#define EC_FOP_FHEAL -2 + +#define EC_CONFIG_VERSION 0 + +#define EC_CONFIG_ALGORITHM 0 + +#define EC_FLAG_LOCK_SHARED 0x0001 + +#define QUORUM_CBK(fn, fop, frame, cookie, this, op_ret, op_errno, params...) \ + do { \ + ec_t *__ec = fop->xl->private; \ + int32_t __op_ret = 0; \ + int32_t __op_errno = 0; \ + int32_t __success_count = gf_bits_count(fop->good); \ + \ + __op_ret = op_ret; \ + __op_errno = op_errno; \ + if (!fop->parent && frame && \ + (GF_CLIENT_PID_SELF_HEALD != frame->root->pid) && \ + __ec->quorum_count && (__success_count < __ec->quorum_count) && \ + op_ret >= 0) { \ + __op_ret = -1; \ + __op_errno = EIO; \ + gf_msg(__ec->xl->name, GF_LOG_ERROR, 0, \ + EC_MSG_CHILDS_INSUFFICIENT, \ + "Insufficient available children for this request " \ + "(have %d, need %d). %s", \ + __success_count, __ec->quorum_count, ec_msg_str(fop)); \ + } \ + fn(frame, cookie, this, __op_ret, __op_errno, params); \ + } while (0) + +enum _ec_xattrop_flags { + EC_FLAG_XATTROP, + EC_FLAG_DATA_DIRTY, + EC_FLAG_METADATA_DIRTY, + + /* Add any new flag here, before EC_FLAG_MAX. The maximum number of + * flags that can be defined is 16. */ + + EC_FLAG_MAX +}; + +/* We keep two sets of flags. One to determine what's really providing the + * current xattrop and the other to know what the parent fop of the xattrop + * needs to proceed. It might happen that a fop needs some information that + * is being already requested by a previous fop. The two sets are stored + * contiguously. */ + +#define EC_FLAG_NEEDS(_flag) (1 << (_flag)) +#define EC_FLAG_PROVIDES(_flag) (1 << ((_flag) + EC_FLAG_MAX)) + +#define EC_NEEDED_FLAGS(_flags) ((_flags) & ((1 << EC_FLAG_MAX) - 1)) + +#define EC_PROVIDED_FLAGS(_flags) EC_NEEDED_FLAGS((_flags) >> EC_FLAG_MAX) + +#define EC_FLAGS_HAVE(_flags, _flag) (((_flags) & (1 << (_flag))) != 0) + +#define EC_SELFHEAL_BIT 62 + +#define EC_MINIMUM_ONE (1 << 6) +#define EC_MINIMUM_MIN (2 << 6) +#define EC_MINIMUM_ALL (3 << 6) +#define EC_FOP_NO_PROPAGATE_ERROR (1 << 8) +#define EC_FOP_MINIMUM(_flags) ((_flags)&255) +#define EC_FOP_FLAGS(_flags) ((_flags) & ~255) + +#define EC_UPDATE_DATA 1 +#define EC_UPDATE_META 2 +#define EC_QUERY_INFO 4 +#define EC_INODE_SIZE 8 + +#define EC_STATE_START 0 +#define EC_STATE_END 0 +#define EC_STATE_INIT 1 +#define EC_STATE_LOCK 2 +#define EC_STATE_DISPATCH 3 +#define EC_STATE_PREPARE_ANSWER 4 +#define EC_STATE_REPORT 5 +#define EC_STATE_LOCK_REUSE 6 +#define EC_STATE_UNLOCK 7 + +#define EC_STATE_DELAYED_START 100 + +#define EC_STATE_HEAL_ENTRY_LOOKUP 200 +#define EC_STATE_HEAL_ENTRY_PREPARE 201 +#define EC_STATE_HEAL_PRE_INODELK_LOCK 202 +#define EC_STATE_HEAL_PRE_INODE_LOOKUP 203 +#define EC_STATE_HEAL_XATTRIBUTES_REMOVE 204 +#define EC_STATE_HEAL_XATTRIBUTES_SET 205 +#define EC_STATE_HEAL_ATTRIBUTES 206 +#define EC_STATE_HEAL_OPEN 207 +#define EC_STATE_HEAL_REOPEN_FD 208 +#define EC_STATE_HEAL_UNLOCK 209 +#define EC_STATE_HEAL_UNLOCK_ENTRY 210 +#define EC_STATE_HEAL_DATA_LOCK 211 +#define EC_STATE_HEAL_DATA_COPY 212 +#define EC_STATE_HEAL_DATA_UNLOCK 213 +#define EC_STATE_HEAL_POST_INODELK_LOCK 214 +#define EC_STATE_HEAL_POST_INODE_LOOKUP 215 +#define EC_STATE_HEAL_SETATTR 216 +#define EC_STATE_HEAL_POST_INODELK_UNLOCK 217 +#define EC_STATE_HEAL_DISPATCH 218 + +/* Value to cover the full range of a file */ +#define EC_RANGE_FULL ((uint64_t)LLONG_MAX + 1) + +gf_boolean_t +ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk); +void +ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx); + +void +ec_complete(ec_fop_data_t *fop); + +void +ec_update_good(ec_fop_data_t *fop, uintptr_t good); + +void +ec_fop_set_error(ec_fop_data_t *fop, int32_t error); + +void +__ec_fop_set_error(ec_fop_data_t *fop, int32_t error); + +ec_cbk_data_t * +ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro); + +gf_boolean_t +ec_cbk_set_error(ec_cbk_data_t *cbk, int32_t error, gf_boolean_t ro); + +void +ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags, + off_t fl_start, uint64_t fl_size); +void +ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base, + uint32_t flags); +void +ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start, + uint64_t fl_size); +void +ec_lock(ec_fop_data_t *fop); +void +ec_lock_reuse(ec_fop_data_t *fop); +void +ec_unlock(ec_fop_data_t *fop); +void +ec_lock_release(ec_t *ec, inode_t *inode); + +gf_boolean_t +ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size); +gf_boolean_t +__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size); +gf_boolean_t +ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size); +gf_boolean_t +__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size); +void +ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode); + +void +ec_flush_size_version(ec_fop_data_t *fop); + +void +ec_dispatch_all(ec_fop_data_t *fop); +void +ec_dispatch_inc(ec_fop_data_t *fop); +void +ec_dispatch_min(ec_fop_data_t *fop); +void +ec_dispatch_one(ec_fop_data_t *fop); + +void +ec_succeed_all(ec_fop_data_t *fop); + +void +ec_sleep(ec_fop_data_t *fop); +void +ec_resume(ec_fop_data_t *fop, int32_t error); +void +ec_resume_parent(ec_fop_data_t *fop); + +void +ec_manager(ec_fop_data_t *fop, int32_t error); +gf_boolean_t +ec_is_recoverable_error(int32_t op_errno); +void +ec_handle_healers_done(ec_fop_data_t *fop); + +int32_t +ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal); +int32_t +ec_get_heal_info(xlator_t *this, loc_t *loc, dict_t **dict); + +int32_t +ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata); + +void +ec_update_fd_status(fd_t *fd, xlator_t *xl, int child_index, + int32_t ret_status); +gf_boolean_t +ec_is_entry_healing(ec_fop_data_t *fop); +void +ec_set_entry_healing(ec_fop_data_t *fop); +void +ec_reset_entry_healing(ec_fop_data_t *fop); +char * +ec_msg_str(ec_fop_data_t *fop); +gf_boolean_t +__ec_is_last_fop(ec_t *ec); +void +ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop); +#endif /* __EC_COMMON_H__ */ diff --git a/xlators/cluster/ec/src/ec-data.c b/xlators/cluster/ec/src/ec-data.c new file mode 100644 index 00000000000..06388833546 --- /dev/null +++ b/xlators/cluster/ec/src/ec-data.c @@ -0,0 +1,288 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-data.h" +#include "ec-messages.h" + +ec_cbk_data_t * +ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop, + int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno) +{ + ec_cbk_data_t *cbk; + ec_t *ec = this->private; + + if (fop->xl != this) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_MISMATCH, + "Mismatching xlators between request " + "and answer (req=%s, ans=%s).", + fop->xl->name, this->name); + + return NULL; + } + if (fop->frame != frame) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FRAME_MISMATCH, + "Mismatching frames between request " + "and answer (req=%p, ans=%p).", + fop->frame, frame); + + return NULL; + } + if (fop->id != id) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FOP_MISMATCH, + "Mismatching fops between request " + "and answer (req=%d, ans=%d).", + fop->id, id); + + return NULL; + } + + cbk = mem_get0(ec->cbk_pool); + if (cbk == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to allocate memory for an " + "answer."); + return NULL; + } + + cbk->fop = fop; + cbk->idx = idx; + cbk->mask = 1ULL << idx; + cbk->count = 1; + cbk->op_ret = op_ret; + cbk->op_errno = op_errno; + INIT_LIST_HEAD(&cbk->entries.list); + + LOCK(&fop->lock); + + list_add_tail(&cbk->answer_list, &fop->answer_list); + + UNLOCK(&fop->lock); + + return cbk; +} + +void +ec_cbk_data_destroy(ec_cbk_data_t *cbk) +{ + if (cbk->xdata != NULL) { + dict_unref(cbk->xdata); + } + if (cbk->dict != NULL) { + dict_unref(cbk->dict); + } + if (cbk->inode != NULL) { + inode_unref(cbk->inode); + } + if (cbk->fd != NULL) { + fd_unref(cbk->fd); + } + if (cbk->buffers != NULL) { + iobref_unref(cbk->buffers); + } + GF_FREE(cbk->vector); + gf_dirent_free(&cbk->entries); + GF_FREE(cbk->str); + + mem_put(cbk); +} + +ec_fop_data_t * +ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id, + uint32_t flags, uintptr_t target, uint32_t fop_flags, + ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks, + void *data) +{ + ec_fop_data_t *fop, *parent; + ec_t *ec = this->private; + + fop = mem_get0(ec->fop_pool); + if (fop == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to allocate memory for a " + "request."); + + return NULL; + } + + INIT_LIST_HEAD(&fop->cbk_list); + INIT_LIST_HEAD(&fop->healer); + INIT_LIST_HEAD(&fop->answer_list); + INIT_LIST_HEAD(&fop->pending_list); + INIT_LIST_HEAD(&fop->locks[0].owner_list); + INIT_LIST_HEAD(&fop->locks[0].wait_list); + INIT_LIST_HEAD(&fop->locks[1].owner_list); + INIT_LIST_HEAD(&fop->locks[1].wait_list); + + fop->xl = this; + fop->req_frame = frame; + + /* fops need a private frame to be able to execute some postop operations + * even if the original fop has completed and reported back to the upper + * xlator and it has destroyed the base frame. + * + * TODO: minimize usage of private frames. Reuse req_frame as much as + * possible. + */ + if (frame != NULL) { + fop->frame = copy_frame(frame); + } else { + fop->frame = create_frame(this, this->ctx->pool); + } + if (fop->frame == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to create a private frame " + "for a request"); + + mem_put(fop); + + return NULL; + } + fop->id = id; + fop->refs = 1; + + fop->flags = flags; + fop->minimum = EC_FOP_MINIMUM(fop_flags); + fop->fop_flags = EC_FOP_FLAGS(fop_flags); + fop->mask = target; + + fop->wind = wind; + fop->handler = handler; + fop->cbks = cbks; + fop->data = data; + + fop->uid = fop->frame->root->uid; + fop->gid = fop->frame->root->gid; + + LOCK_INIT(&fop->lock); + + fop->frame->local = fop; + + if (frame != NULL) { + parent = frame->local; + if (parent != NULL) { + ec_sleep(parent); + } + + fop->parent = parent; + } + + LOCK(&ec->lock); + + list_add_tail(&fop->pending_list, &ec->pending_fops); + + UNLOCK(&ec->lock); + + return fop; +} + +void +ec_fop_data_acquire(ec_fop_data_t *fop) +{ + LOCK(&fop->lock); + + ec_trace("ACQUIRE", fop, ""); + + fop->refs++; + + UNLOCK(&fop->lock); +} + +static void +ec_handle_last_pending_fop_completion(ec_fop_data_t *fop, gf_boolean_t *notify) +{ + ec_t *ec = fop->xl->private; + + *notify = _gf_false; + + if (!list_empty(&fop->pending_list)) { + LOCK(&ec->lock); + { + list_del_init(&fop->pending_list); + *notify = __ec_is_last_fop(ec); + } + UNLOCK(&ec->lock); + } +} + +void +ec_fop_cleanup(ec_fop_data_t *fop) +{ + ec_cbk_data_t *cbk, *tmp; + + list_for_each_entry_safe(cbk, tmp, &fop->answer_list, answer_list) + { + list_del_init(&cbk->answer_list); + + ec_cbk_data_destroy(cbk); + } + INIT_LIST_HEAD(&fop->cbk_list); + + fop->answer = NULL; +} + +void +ec_fop_data_release(ec_fop_data_t *fop) +{ + ec_t *ec = NULL; + int32_t refs; + gf_boolean_t notify = _gf_false; + + LOCK(&fop->lock); + + ec_trace("RELEASE", fop, ""); + + GF_ASSERT(fop->refs > 0); + refs = --fop->refs; + + UNLOCK(&fop->lock); + + if (refs == 0) { + fop->frame->local = NULL; + STACK_DESTROY(fop->frame->root); + + LOCK_DESTROY(&fop->lock); + + if (fop->xdata != NULL) { + dict_unref(fop->xdata); + } + if (fop->dict != NULL) { + dict_unref(fop->dict); + } + if (fop->inode != NULL) { + inode_unref(fop->inode); + } + if (fop->fd != NULL) { + fd_unref(fop->fd); + } + if (fop->buffers != NULL) { + iobref_unref(fop->buffers); + } + GF_FREE(fop->vector); + GF_FREE(fop->str[0]); + GF_FREE(fop->str[1]); + loc_wipe(&fop->loc[0]); + loc_wipe(&fop->loc[1]); + GF_FREE(fop->errstr); + + ec_resume_parent(fop); + + ec_fop_cleanup(fop); + + ec = fop->xl->private; + ec_handle_last_pending_fop_completion(fop, ¬ify); + ec_handle_healers_done(fop); + mem_put(fop); + if (notify) { + ec_pending_fops_completed(ec); + } + } +} diff --git a/xlators/cluster/ec/src/ec-data.h b/xlators/cluster/ec/src/ec-data.h new file mode 100644 index 00000000000..c8a74ffe1ed --- /dev/null +++ b/xlators/cluster/ec/src/ec-data.h @@ -0,0 +1,35 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_DATA_H__ +#define __EC_DATA_H__ + +#include "ec-types.h" + +ec_cbk_data_t * +ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop, + int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno); +ec_fop_data_t * +ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id, + uint32_t flags, uintptr_t target, uint32_t fop_flags, + ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks, + void *data); +void +ec_fop_data_acquire(ec_fop_data_t *fop); +void +ec_fop_data_release(ec_fop_data_t *fop); + +void +ec_fop_cleanup(ec_fop_data_t *fop); + +void +ec_pending_fops_completed(ec_t *ec); + +#endif /* __EC_DATA_H__ */ diff --git a/xlators/cluster/ec/src/ec-dir-read.c b/xlators/cluster/ec/src/ec-dir-read.c new file mode 100644 index 00000000000..f71dcfac293 --- /dev/null +++ b/xlators/cluster/ec/src/ec-dir-read.c @@ -0,0 +1,647 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "ec.h" +#include "ec-messages.h" +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-combine.h" +#include "ec-fops.h" + +/**************************************************************** + * + * File Operation: opendir + * + ***************************************************************/ + +int32_t +ec_combine_opendir(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (dst->fd != src->fd) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH, + "Mismatching fd in answers " + "of 'GF_FOP_OPENDIR': %p <-> %p", + dst->fd, src->fd); + + return 0; + } + + return 1; +} + +int32_t +ec_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPENDIR, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (fd != NULL) { + cbk->fd = fd_ref(fd); + if (cbk->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, + EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, ec_combine_opendir); + + ec_update_fd_status(fd, this, idx, op_ret); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_opendir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_opendir_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->opendir, + &fop->loc[0], fop->fd, fop->xdata); +} + +int32_t +ec_manager_opendir(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + ec_fd_t *ctx; + int32_t err; + + switch (state) { + case EC_STATE_INIT: + LOCK(&fop->fd->lock); + + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx == NULL) { + UNLOCK(&fop->fd->lock); + + fop->error = ENOMEM; + + return EC_STATE_REPORT; + } + if (!ctx->loc.inode) { + err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]); + if (err != 0) { + UNLOCK(&fop->fd->lock); + + fop->error = -err; + + return EC_STATE_REPORT; + } + } + + UNLOCK(&fop->fd->lock); + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + /* Save which subvolumes successfully opened the directory. + * If ctx->open is 0, it means that readdir cannot be + * processed in this directory. + */ + LOCK(&fop->fd->lock); + + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + ctx->open |= cbk->mask; + } + + UNLOCK(&fop->fd->lock); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.opendir != NULL) { + fop->cbks.opendir(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->fd, cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.opendir != NULL) { + fop->cbks.opendir(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc, + fd_t *fd, dict_t *xdata) +{ + ec_cbk_t callback = {.opendir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(OPENDIR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_OPENDIR, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_opendir, + ec_manager_opendir, callback, data); + if (fop == NULL) { + goto out; + } + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} + +/* Returns -1 if client_id is invalid else index of child subvol in xl_list */ +int +ec_deitransform(xlator_t *this, off_t offset) +{ + int idx = -1; + int client_id = -1; + ec_t *ec = this->private; + char id[32] = {0}; + int err; + + client_id = gf_deitransform(this, offset); + sprintf(id, "%d", client_id); + err = dict_get_int32(ec->leaf_to_subvolid, id, &idx); + if (err < 0) { + idx = err; + goto out; + } + +out: + if (idx < 0) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REQUEST, + "Invalid index %d in readdirp request", client_id); + idx = -EINVAL; + } + return idx; +} + +/* FOP: readdir */ + +void +ec_adjust_readdirp(ec_t *ec, int32_t idx, gf_dirent_t *entries) +{ + gf_dirent_t *entry; + + list_for_each_entry(entry, &entries->list, list) + { + if (!entry->inode) + continue; + + if (entry->d_stat.ia_type == IA_IFREG) { + if ((entry->dict == NULL) || + (ec_dict_del_number(entry->dict, EC_XATTR_SIZE, + &entry->d_stat.ia_size) != 0)) { + inode_unref(entry->inode); + entry->inode = NULL; + } else { + ec_iatt_rebuild(ec, &entry->d_stat, 1, 1); + } + } + } +} + +int32_t +ec_common_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, + dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, + op_errno); + if (cbk) { + if (xdata) + cbk->xdata = dict_ref(xdata); + if (cbk->op_ret >= 0) + list_splice_init(&entries->list, &cbk->entries.list); + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_readdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_common_readdir_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->readdir, + fop->fd, fop->size, fop->offset, fop->xdata); +} + +int32_t +ec_manager_readdir(ec_fop_data_t *fop, int32_t state) +{ + ec_fd_t *ctx = NULL; + ec_cbk_data_t *cbk = NULL; + + switch (state) { + case EC_STATE_INIT: + /* Return error if opendir has not been successfully called on + * any subvolume. */ + ctx = ec_fd_get(fop->fd, fop->xl); + if (ctx == NULL) { + fop->error = ENOMEM; + } else if (ctx->open == 0) { + fop->error = EBADFD; + } + + if (fop->error) { + gf_msg(fop->xl->name, GF_LOG_ERROR, fop->error, + EC_MSG_INVALID_REQUEST, "EC is not winding readdir: %s", + ec_msg_str(fop)); + return EC_STATE_REPORT; + } + + if (fop->id == GF_FOP_READDIRP) { + int32_t err; + + if (fop->xdata == NULL) { + fop->xdata = dict_new(); + if (fop->xdata == NULL) { + fop->error = ENOMEM; + + return EC_STATE_REPORT; + } + } + + err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0); + if (err != 0) { + fop->error = -err; + + return EC_STATE_REPORT; + } + } + + if (fop->offset != 0) { + /* Non-zero offset is irrecoverable error as the offset may not + * be valid on other bricks*/ + int32_t idx = -1; + + idx = ec_deitransform(fop->xl, fop->offset); + + if (idx < 0) { + fop->error = -idx; + return EC_STATE_REPORT; + } + fop->mask &= 1ULL << idx; + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, + EC_RANGE_FULL); + ec_lock(fop); + } + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_one(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + if (ec_dispatch_one_retry(fop, &cbk)) { + return EC_STATE_DISPATCH; + } + + if ((cbk != NULL) && (cbk->op_ret > 0) && + (fop->id == GF_FOP_READDIRP)) { + ec_adjust_readdirp(fop->xl->private, cbk->idx, &cbk->entries); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + GF_ASSERT(cbk); + if (fop->id == GF_FOP_READDIR) { + if (fop->cbks.readdir != NULL) { + fop->cbks.readdir(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, &cbk->entries, cbk->xdata); + } + } else { + if (fop->cbks.readdirp != NULL) { + fop->cbks.readdirp(fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, + &cbk->entries, cbk->xdata); + } + } + if (fop->offset == 0) + return EC_STATE_LOCK_REUSE; + else + return EC_STATE_END; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + if (fop->id == GF_FOP_READDIR) { + if (fop->cbks.readdir != NULL) { + fop->cbks.readdir(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL); + } + } else { + if (fop->cbks.readdirp != NULL) { + fop->cbks.readdirp(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL); + } + } + if (fop->offset == 0) + return EC_STATE_LOCK_REUSE; + else + return EC_STATE_END; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + GF_ASSERT(fop->offset == 0); + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + GF_ASSERT(fop->offset == 0); + ec_unlock(fop); + + return EC_STATE_END; + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, dict_t *xdata) +{ + ec_cbk_t callback = {.readdir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(READDIR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIR, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_readdir, + ec_manager_readdir, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->size = size; + fop->offset = offset; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} + +/* FOP: readdirp */ + +void +ec_wind_readdirp(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_common_readdir_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->readdirp, + fop->fd, fop->size, fop->offset, fop->xdata); +} + +void +ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, dict_t *xdata) +{ + ec_cbk_t callback = {.readdirp = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(READDIRP) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate( + frame, this, GF_FOP_READDIRP, EC_FLAG_LOCK_SHARED, target, fop_flags, + ec_wind_readdirp, ec_manager_readdir, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->size = size; + fop->offset = offset; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} diff --git a/xlators/cluster/ec/src/ec-dir-write.c b/xlators/cluster/ec/src/ec-dir-write.c new file mode 100644 index 00000000000..53d27d895c3 --- /dev/null +++ b/xlators/cluster/ec/src/ec-dir-write.c @@ -0,0 +1,1487 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "ec.h" +#include "ec-messages.h" +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-combine.h" +#include "ec-method.h" +#include "ec-fops.h" + +int +ec_dir_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie, int op_ret, + int op_errno, struct iatt *poststat, struct iatt *preparent, + struct iatt *postparent, struct iatt *preparent2, + struct iatt *postparent2, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int i = 0; + int idx = 0; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + idx = (long)cookie; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, + op_errno); + if (!cbk) + goto out; + + if (xdata) + cbk->xdata = dict_ref(xdata); + + if (op_ret < 0) + goto out; + + if (poststat) + cbk->iatt[i++] = *poststat; + + if (preparent) + cbk->iatt[i++] = *preparent; + + if (postparent) + cbk->iatt[i++] = *postparent; + + if (preparent2) + cbk->iatt[i++] = *preparent2; + + if (postparent2) + cbk->iatt[i++] = *postparent2; + +out: + if (cbk) + ec_combine(cbk, ec_combine_write); + + if (fop) + ec_complete(fop); + return 0; +} + +/* FOP: create */ + +int32_t +ec_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, fd_t *fd, inode_t *inode, struct iatt *buf, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata) +{ + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); +} + +void +ec_wind_create(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_create_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->create, + &fop->loc[0], fop->int32, fop->mode[0], fop->mode[1], + fop->fd, fop->xdata); +} + +int32_t +ec_manager_create(ec_fop_data_t *fop, int32_t state) +{ + ec_config_t config; + ec_t *ec; + ec_cbk_data_t *cbk; + ec_fd_t *ctx; + uint64_t version[2] = {0, 0}; + int32_t err; + + switch (state) { + case EC_STATE_INIT: + LOCK(&fop->fd->lock); + + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx == NULL) { + UNLOCK(&fop->fd->lock); + + fop->error = ENOMEM; + + return EC_STATE_REPORT; + } + err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]); + if (err != 0) { + UNLOCK(&fop->fd->lock); + + fop->error = -err; + + return EC_STATE_REPORT; + } + + ctx->flags = fop->int32; + + UNLOCK(&fop->fd->lock); + + if (fop->xdata == NULL) { + fop->xdata = dict_new(); + if (fop->xdata == NULL) { + fop->error = ENOMEM; + + return EC_STATE_REPORT; + } + } + + ec = fop->xl->private; + + config.version = EC_CONFIG_VERSION; + config.algorithm = EC_CONFIG_ALGORITHM; + config.gf_word_size = EC_GF_BITS; + config.bricks = ec->nodes; + config.redundancy = ec->redundancy; + config.chunk_size = EC_METHOD_CHUNK_SIZE; + + err = ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG, &config); + if (err != 0) { + fop->error = -err; + + return EC_STATE_REPORT; + } + err = ec_dict_set_array(fop->xdata, EC_XATTR_VERSION, version, + EC_VERSION_SIZE); + if (err != 0) { + fop->error = -err; + + return EC_STATE_REPORT; + } + err = ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0); + if (err != 0) { + fop->error = -err; + + return EC_STATE_REPORT; + } + + /* We need to write to specific offsets on the bricks, so we + * need to remove O_APPEND from flags (if present) */ + fop->int32 &= ~O_APPEND; + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + if (!ec_cbk_set_error(cbk, -err, _gf_false)) { + LOCK(&fop->fd->lock); + + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + ctx->open |= cbk->mask; + } + + UNLOCK(&fop->fd->lock); + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.create != NULL) { + QUORUM_CBK(fop->cbks.create, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->fd, + fop->loc[0].inode, &cbk->iatt[0], &cbk->iatt[1], + &cbk->iatt[2], cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.create != NULL) { + fop->cbks.create(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL, NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc, + int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata) +{ + ec_cbk_t callback = {.create = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(CREATE) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_CREATE, 0, target, fop_flags, + ec_wind_create, ec_manager_create, callback, + data); + if (fop == NULL) { + goto out; + } + + fop->int32 = flags; + fop->mode[0] = mode; + fop->mode[1] = umask; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL, NULL); + } +} + +/* FOP: link */ + +int32_t +ec_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, inode_t *inode, struct iatt *buf, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata) +{ + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); +} + +void +ec_wind_link(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_link_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->link, + &fop->loc[0], &fop->loc[1], fop->xdata); +} + +int32_t +ec_manager_link(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_parent_inode( + fop, &fop->loc[1], &fop->loc[0], + EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + if (cbk->iatt[0].ia_type == IA_IFREG) { + cbk->iatt[0].ia_size = fop->locks[0].size; + } + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + ec_cbk_set_error(cbk, -err, _gf_false); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.link != NULL) { + QUORUM_CBK(fop->cbks.link, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->loc[0].inode, + &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2], + cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.link != NULL) { + fop->cbks.link(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc, + loc_t *newloc, dict_t *xdata) +{ + ec_cbk_t callback = {.link = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(LINK) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_LINK, 0, target, fop_flags, + ec_wind_link, ec_manager_link, callback, data); + if (fop == NULL) { + goto out; + } + + if (oldloc != NULL) { + if (loc_copy(&fop->loc[0], oldloc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (newloc != NULL) { + if (loc_copy(&fop->loc[1], newloc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL); + } +} + +/* FOP: mkdir */ + +int32_t +ec_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, inode_t *inode, struct iatt *buf, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata) +{ + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); +} + +void +ec_wind_mkdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_mkdir_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->mkdir, + &fop->loc[0], fop->mode[0], fop->mode[1], fop->xdata); +} + +int32_t +ec_manager_mkdir(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + uint64_t version[2] = {0, 0}; + int32_t err; + + switch (state) { + case EC_STATE_INIT: + if (fop->xdata == NULL) { + fop->xdata = dict_new(); + if (fop->xdata == NULL) { + fop->error = ENOMEM; + + return EC_STATE_REPORT; + } + } + + err = ec_dict_set_array(fop->xdata, EC_XATTR_VERSION, version, + EC_VERSION_SIZE); + if (err != 0) { + fop->error = -err; + return EC_STATE_REPORT; + } + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + ec_cbk_set_error(cbk, -err, _gf_false); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.mkdir != NULL) { + QUORUM_CBK(fop->cbks.mkdir, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->loc[0].inode, + &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2], + cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + cbk = fop->answer; + GF_ASSERT(fop->error != 0); + + if (fop->cbks.mkdir != NULL) { + fop->cbks.mkdir(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL, NULL, + ((cbk) ? cbk->xdata : NULL)); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc, + mode_t mode, mode_t umask, dict_t *xdata) +{ + ec_cbk_t callback = {.mkdir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(MKDIR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_MKDIR, 0, target, fop_flags, + ec_wind_mkdir, ec_manager_mkdir, callback, data); + if (fop == NULL) { + goto out; + } + + fop->mode[0] = mode; + fop->mode[1] = umask; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL); + } +} + +/* FOP: mknod */ + +int32_t +ec_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, inode_t *inode, struct iatt *buf, + struct iatt *preparent, struct iatt *postparent, dict_t *xdata) +{ + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); +} + +void +ec_wind_mknod(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_mknod_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->mknod, + &fop->loc[0], fop->mode[0], fop->dev, fop->mode[1], + fop->xdata); +} + +int32_t +ec_manager_mknod(ec_fop_data_t *fop, int32_t state) +{ + ec_config_t config; + ec_t *ec; + ec_cbk_data_t *cbk; + uint64_t version[2] = {0, 0}; + + switch (state) { + case EC_STATE_INIT: + if (S_ISREG(fop->mode[0])) { + int32_t err; + + if (fop->xdata == NULL) { + fop->xdata = dict_new(); + if (fop->xdata == NULL) { + fop->error = ENOMEM; + + return EC_STATE_REPORT; + } + } + + ec = fop->xl->private; + + config.version = EC_CONFIG_VERSION; + config.algorithm = EC_CONFIG_ALGORITHM; + config.gf_word_size = EC_GF_BITS; + config.bricks = ec->nodes; + config.redundancy = ec->redundancy; + config.chunk_size = EC_METHOD_CHUNK_SIZE; + + err = ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG, &config); + if (err != 0) { + fop->error = -err; + + return EC_STATE_REPORT; + } + err = ec_dict_set_array(fop->xdata, EC_XATTR_VERSION, version, + EC_VERSION_SIZE); + if (err != 0) { + fop->error = -err; + + return EC_STATE_REPORT; + } + err = ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0); + if (err != 0) { + fop->error = -err; + + return EC_STATE_REPORT; + } + } + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + ec_cbk_set_error(cbk, -err, _gf_false); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.mknod != NULL) { + QUORUM_CBK(fop->cbks.mknod, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->loc[0].inode, + &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2], + cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.mknod != NULL) { + fop->cbks.mknod(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc, + mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata) +{ + ec_cbk_t callback = {.mknod = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(MKNOD) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_MKNOD, 0, target, fop_flags, + ec_wind_mknod, ec_manager_mknod, callback, data); + if (fop == NULL) { + goto out; + } + + fop->mode[0] = mode; + fop->dev = rdev; + fop->mode[1] = umask; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL); + } +} + +/* FOP: rename */ + +int32_t +ec_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *buf, struct iatt *preoldparent, + struct iatt *postoldparent, struct iatt *prenewparent, + struct iatt *postnewparent, dict_t *xdata) +{ + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preoldparent, postoldparent, prenewparent, + postnewparent, xdata); +} + +void +ec_wind_rename(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_rename_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->rename, + &fop->loc[0], &fop->loc[1], fop->xdata); +} + +int32_t +ec_manager_rename(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_parent_inode( + fop, &fop->loc[0], &fop->loc[0], + EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE); + ec_lock_prepare_parent_inode(fop, &fop->loc[1], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 5, cbk->count); + + if (cbk->iatt[0].ia_type == IA_IFREG) { + cbk->iatt[0].ia_size = fop->locks[0].size; + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.rename != NULL) { + QUORUM_CBK(fop->cbks.rename, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], &cbk->iatt[2], &cbk->iatt[3], + &cbk->iatt[4], cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.rename != NULL) { + fop->cbks.rename(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL, NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc, + loc_t *newloc, dict_t *xdata) +{ + ec_cbk_t callback = {.rename = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(RENAME) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_RENAME, 0, target, fop_flags, + ec_wind_rename, ec_manager_rename, callback, + data); + if (fop == NULL) { + goto out; + } + + if (oldloc != NULL) { + if (loc_copy(&fop->loc[0], oldloc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (newloc != NULL) { + if (loc_copy(&fop->loc[1], newloc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL, NULL); + } +} + +/* FOP: rmdir */ + +int32_t +ec_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *preparent, struct iatt *postparent, + dict_t *xdata) +{ + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, + preparent, postparent, NULL, NULL, xdata); +} + +void +ec_wind_rmdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_rmdir_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->rmdir, + &fop->loc[0], fop->int32, fop->xdata); +} + +int32_t +ec_manager_rmdir(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + ec_fop_prepare_answer(fop, _gf_false); + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.rmdir != NULL) { + QUORUM_CBK(fop->cbks.rmdir, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.rmdir != NULL) { + fop->cbks.rmdir(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc, + int xflags, dict_t *xdata) +{ + ec_cbk_t callback = {.rmdir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(RMDIR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_RMDIR, 0, target, fop_flags, + ec_wind_rmdir, ec_manager_rmdir, callback, data); + if (fop == NULL) { + goto out; + } + + fop->int32 = xflags; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} + +/* FOP: symlink */ + +int32_t +ec_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, inode_t *inode, + struct iatt *buf, struct iatt *preparent, + struct iatt *postparent, dict_t *xdata) +{ + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf, + preparent, postparent, NULL, NULL, xdata); +} + +void +ec_wind_symlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_symlink_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->symlink, + fop->str[0], &fop->loc[0], fop->mode[0], fop->xdata); +} + +int32_t +ec_manager_symlink(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, + &cbk->iatt[0]); + ec_cbk_set_error(cbk, -err, _gf_false); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.symlink != NULL) { + QUORUM_CBK(fop->cbks.symlink, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, fop->loc[0].inode, + &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2], + cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.symlink != NULL) { + fop->cbks.symlink(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_symlink_cbk_t func, void *data, + const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata) +{ + ec_cbk_t callback = {.symlink = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(SYMLINK) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_SYMLINK, 0, target, + fop_flags, ec_wind_symlink, ec_manager_symlink, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->mode[0] = umask; + + if (linkname != NULL) { + fop->str[0] = gf_strdup(linkname); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL); + } +} + +/* FOP: unlink */ + +int32_t +ec_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *preparent, struct iatt *postparent, + dict_t *xdata) +{ + return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, + preparent, postparent, NULL, NULL, xdata); +} + +void +ec_wind_unlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_unlink_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->unlink, + &fop->loc[0], fop->int32, fop->xdata); +} + +int32_t +ec_manager_unlink(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL, + EC_UPDATE_DATA | EC_UPDATE_META); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + ec_fop_prepare_answer(fop, _gf_false); + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.unlink != NULL) { + QUORUM_CBK(fop->cbks.unlink, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.unlink != NULL) { + fop->cbks.unlink(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc, + int xflags, dict_t *xdata) +{ + ec_cbk_t callback = {.unlink = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(UNLINK) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_UNLINK, 0, target, fop_flags, + ec_wind_unlink, ec_manager_unlink, callback, + data); + if (fop == NULL) { + goto out; + } + + fop->int32 = xflags; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} diff --git a/xlators/cluster/ec/src/ec-fops.h b/xlators/cluster/ec/src/ec-fops.h new file mode 100644 index 00000000000..07edf8a7fec --- /dev/null +++ b/xlators/cluster/ec/src/ec-fops.h @@ -0,0 +1,254 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_FOPS_H__ +#define __EC_FOPS_H__ + +#include <glusterfs/xlator.h> + +#include "ec-types.h" +#include "ec-common.h" + +void +ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc, + int32_t mask, dict_t *xdata); + +void +ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc, + int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata); + +void +ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_entrylk_cbk_t func, void *data, + const char *volume, loc_t *loc, const char *basename, + entrylk_cmd cmd, entrylk_type type, dict_t *xdata); + +void +ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data, + const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd, + entrylk_type type, dict_t *xdata); + +void +ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd, + dict_t *xdata); + +void +ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd, + int32_t datasync, dict_t *xdata); + +void +ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd, + int32_t datasync, dict_t *xdata); + +void +ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc, + const char *name, dict_t *xdata); + +void +ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd, + const char *name, dict_t *xdata); + +void +ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc, + int32_t partial, dict_t *xdata); + +void +ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd, + int32_t partial, dict_t *xdata); + +void +ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner, + uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func, + void *data, const char *volume, loc_t *loc, int32_t cmd, + struct gf_flock *flock, dict_t *xdata); + +void +ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner, + uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func, + void *data, const char *volume, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata); + +void +ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc, + loc_t *newloc, dict_t *xdata); + +void +ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags, + fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata); + +void +ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc, + dict_t *xdata); + +void +ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc, + mode_t mode, mode_t umask, dict_t *xdata); + +void +ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc, + mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata); + +void +ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc, + int32_t flags, fd_t *fd, dict_t *xdata); + +void +ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc, + fd_t *fd, dict_t *xdata); + +void +ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, dict_t *xdata); + +void +ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, dict_t *xdata); + +void +ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc, + size_t size, dict_t *xdata); + +void +ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, uint32_t flags, dict_t *xdata); + +void +ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_removexattr_cbk_t func, void *data, + loc_t *loc, const char *name, dict_t *xdata); + +void +ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data, + fd_t *fd, const char *name, dict_t *xdata); + +void +ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc, + loc_t *newloc, dict_t *xdata); + +void +ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc, + int xflags, dict_t *xdata); + +void +ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc, + struct iatt *stbuf, int32_t valid, dict_t *xdata); + +void +ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd, + struct iatt *stbuf, int32_t valid, dict_t *xdata); + +void +ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc, + dict_t *dict, int32_t flags, dict_t *xdata); + +void +ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd, + dict_t *dict, int32_t flags, dict_t *xdata); + +void +ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc, + dict_t *xdata); + +void +ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd, + dict_t *xdata); + +void +ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc, + dict_t *xdata); + +void +ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_symlink_cbk_t func, void *data, + const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata); + +void +ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd, + int32_t mode, off_t offset, size_t len, dict_t *xdata); + +void +ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd, + off_t offset, size_t len, dict_t *xdata); + +void +ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc, + off_t offset, dict_t *xdata); + +void +ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd, + off_t offset, dict_t *xdata); + +void +ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc, + int xflags, dict_t *xdata); + +void +ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd, + struct iovec *vector, int32_t count, off_t offset, uint32_t flags, + struct iobref *iobref, dict_t *xdata); + +void +ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata); + +void +ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata); + +void +ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd, + off_t offset, gf_seek_what_t what, dict_t *xdata); + +void +ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op, + dict_t *xdata); + +#endif /* __EC_FOPS_H__ */ diff --git a/xlators/cluster/ec/src/ec-galois.c b/xlators/cluster/ec/src/ec-galois.c new file mode 100644 index 00000000000..6e4990c71f5 --- /dev/null +++ b/xlators/cluster/ec/src/ec-galois.c @@ -0,0 +1,183 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <string.h> + +#include "ec-mem-types.h" +#include "ec-gf8.h" +#include "ec-helpers.h" + +static ec_gf_t * +ec_gf_alloc(uint32_t bits, uint32_t mod) +{ + ec_gf_t *gf; + + gf = GF_MALLOC(sizeof(ec_gf_t), ec_mt_ec_gf_t); + if (gf == NULL) { + goto failed; + } + + gf->bits = bits; + gf->size = 1 << bits; + gf->mod = mod; + + gf->log = GF_MALLOC(sizeof(uint32_t) * (gf->size * 2 - 1), + gf_common_mt_int); + if (gf->log == NULL) { + goto failed_gf; + } + gf->pow = GF_MALLOC(sizeof(uint32_t) * (gf->size * 2 - 1), + gf_common_mt_int); + if (gf->pow == NULL) { + goto failed_log; + } + + return gf; + +failed_log: + GF_FREE(gf->log); +failed_gf: + GF_FREE(gf); +failed: + return EC_ERR(ENOMEM); +} + +static void +ec_gf_init_tables(ec_gf_t *gf) +{ + uint32_t i, tmp; + + memset(gf->log, -1, sizeof(uint32_t) * gf->size); + + gf->pow[0] = 1; + gf->log[0] = gf->size; + gf->log[1] = 0; + for (i = 1; i < gf->size; i++) { + tmp = gf->pow[i - 1] << 1; + if (tmp >= gf->size) { + tmp ^= gf->mod; + } + gf->pow[i + gf->size - 1] = gf->pow[i] = tmp; + gf->log[tmp + gf->size - 1] = gf->log[tmp] = i; + } +} + +ec_gf_t * +ec_gf_prepare(uint32_t bits, uint32_t mod) +{ + ec_gf_mul_t **tbl; + ec_gf_t *gf; + uint32_t i, j; + + if (bits != 8) { + return EC_ERR(EINVAL); + } + + tbl = ec_gf8_mul; + if (mod == 0) { + mod = 0x11d; + } + + gf = ec_gf_alloc(bits, mod); + if (EC_IS_ERR(gf)) { + return gf; + } + ec_gf_init_tables(gf); + + gf->table = tbl; + gf->min_ops = bits * bits; + gf->max_ops = 0; + gf->avg_ops = 0; + for (i = 1; i < gf->size; i++) { + for (j = 0; tbl[i]->ops[j].op != EC_GF_OP_END; j++) { + } + if (gf->max_ops < j) { + gf->max_ops = j; + } + if (gf->min_ops > j) { + gf->min_ops = j; + } + gf->avg_ops += j; + } + gf->avg_ops /= gf->size; + + return gf; +} + +void +ec_gf_destroy(ec_gf_t *gf) +{ + GF_FREE(gf->pow); + GF_FREE(gf->log); + GF_FREE(gf); +} + +uint32_t +ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b) +{ + if ((a >= gf->size) || (b >= gf->size)) { + return gf->size; + } + + return a ^ b; +} + +uint32_t +ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b) +{ + if ((a >= gf->size) || (b >= gf->size)) { + return gf->size; + } + + if ((a != 0) && (b != 0)) { + return gf->pow[gf->log[a] + gf->log[b]]; + } + + return 0; +} + +uint32_t +ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b) +{ + if ((a >= gf->size) || (b >= gf->size)) { + return gf->size; + } + + if (b != 0) { + if (a != 0) { + return gf->pow[gf->size - 1 + gf->log[a] - gf->log[b]]; + } + + return 0; + } + + return gf->size; +} + +uint32_t +ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b) +{ + uint32_t r; + + if ((a >= gf->size) || ((a == 0) && (b == 0))) { + return gf->size; + } + + r = 1; + while (b != 0) { + if ((b & 1) != 0) { + r = ec_gf_mul(gf, r, a); + } + a = ec_gf_mul(gf, a, a); + b >>= 1; + } + + return r; +} diff --git a/xlators/cluster/ec/src/ec-galois.h b/xlators/cluster/ec/src/ec-galois.h new file mode 100644 index 00000000000..ed55d53e419 --- /dev/null +++ b/xlators/cluster/ec/src/ec-galois.h @@ -0,0 +1,32 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_GALOIS_H__ +#define __EC_GALOIS_H__ + +#include <inttypes.h> + +#include "ec-types.h" + +ec_gf_t * +ec_gf_prepare(uint32_t bits, uint32_t mod); +void +ec_gf_destroy(ec_gf_t *gf); + +uint32_t +ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b); +uint32_t +ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b); +uint32_t +ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b); +uint32_t +ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b); + +#endif /* __EC_GALOIS_H__ */ diff --git a/xlators/cluster/ec/src/ec-generic.c b/xlators/cluster/ec/src/ec-generic.c new file mode 100644 index 00000000000..884deb93669 --- /dev/null +++ b/xlators/cluster/ec/src/ec-generic.c @@ -0,0 +1,1591 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <glusterfs/byte-order.h> + +#include "ec.h" +#include "ec-messages.h" +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-combine.h" +#include "ec-fops.h" + +/* FOP: flush */ + +int32_t +ec_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FLUSH, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_flush(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_flush_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->flush, fop->fd, + fop->xdata); +} + +int32_t +ec_manager_flush(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_flush_size_version(fop); + + return EC_STATE_DELAYED_START; + + case EC_STATE_DELAYED_START: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + ec_fop_prepare_answer(fop, _gf_false); + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.flush != NULL) { + fop->cbks.flush(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DELAYED_START: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.flush != NULL) { + fop->cbks.flush(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +static int32_t +ec_validate_fd(fd_t *fd, xlator_t *xl) +{ + uint64_t iversion = 0; + uint64_t fversion = 0; + ec_inode_t *inode_ctx = NULL; + ec_fd_t *fd_ctx = NULL; + + LOCK(&fd->lock); + { + fd_ctx = __ec_fd_get(fd, xl); + if (fd_ctx) { + fversion = fd_ctx->bad_version; + } + } + UNLOCK(&fd->lock); + + LOCK(&fd->inode->lock); + { + inode_ctx = __ec_inode_get(fd->inode, xl); + if (inode_ctx) { + iversion = inode_ctx->bad_version; + } + } + UNLOCK(&fd->inode->lock); + if (fversion < iversion) { + return EBADF; + } + return 0; +} + +void +ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd, + dict_t *xdata) +{ + ec_cbk_t callback = {.flush = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FLUSH) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + if (fd) { + error = ec_validate_fd(fd, this); + if (error) { + gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD, + "Failing %s on %s", gf_fop_list[GF_FOP_FLUSH], + fd->inode ? uuid_utoa(fd->inode->gfid) : ""); + goto out; + } + } + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FLUSH, 0, target, fop_flags, + ec_wind_flush, ec_manager_flush, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: fsync */ + +int32_t +ec_combine_fsync(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_FSYNC'"); + + return 0; + } + + return 1; +} + +int32_t +ec_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf, + dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNC, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (prebuf != NULL) { + cbk->iatt[0] = *prebuf; + } + if (postbuf != NULL) { + cbk->iatt[1] = *postbuf; + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, ec_combine_fsync); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_fsync(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fsync_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fsync, fop->fd, + fop->int32, fop->xdata); +} + +int32_t +ec_manager_fsync(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, EC_RANGE_FULL); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_flush_size_version(fop); + + return EC_STATE_DELAYED_START; + + case EC_STATE_DELAYED_START: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, + &cbk->iatt[0].ia_size)); + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.fsync != NULL) { + fop->cbks.fsync(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1], + cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + case -EC_STATE_DELAYED_START: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.fsync != NULL) { + fop->cbks.fsync(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd, + int32_t datasync, dict_t *xdata) +{ + ec_cbk_t callback = {.fsync = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FSYNC) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + if (fd) { + error = ec_validate_fd(fd, this); + if (error) { + gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD, + "Failing %s on %s", gf_fop_list[GF_FOP_FSYNC], + fd->inode ? uuid_utoa(fd->inode->gfid) : ""); + goto out; + } + } + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNC, 0, target, fop_flags, + ec_wind_fsync, ec_manager_fsync, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->int32 = datasync; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} + +/* FOP: fsyncdir */ + +int32_t +ec_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNCDIR, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_fsyncdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fsyncdir_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fsyncdir, + fop->fd, fop->int32, fop->xdata); +} + +int32_t +ec_manager_fsyncdir(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_flush_size_version(fop); + + return EC_STATE_DELAYED_START; + + case EC_STATE_DELAYED_START: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + ec_fop_prepare_answer(fop, _gf_false); + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.fsyncdir != NULL) { + fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + case -EC_STATE_DELAYED_START: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.fsyncdir != NULL) { + fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd, + int32_t datasync, dict_t *xdata) +{ + ec_cbk_t callback = {.fsyncdir = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FSYNCDIR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNCDIR, 0, target, + fop_flags, ec_wind_fsyncdir, ec_manager_fsyncdir, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->int32 = datasync; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: lookup */ + +void +ec_lookup_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk) +{ + ec_inode_t *ctx = NULL; + uint64_t size = 0; + int32_t have_size = 0, err; + + if (cbk->op_ret < 0) { + return; + } + + ec_dict_del_array(cbk->xdata, EC_XATTR_VERSION, cbk->version, + EC_VERSION_SIZE); + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, &cbk->iatt[0]); + if (ec_cbk_set_error(cbk, -err, _gf_true)) { + return; + } + + LOCK(&cbk->inode->lock); + + ctx = __ec_inode_get(cbk->inode, fop->xl); + if (ctx != NULL) { + if (ctx->have_version) { + cbk->version[0] = ctx->post_version[0]; + cbk->version[1] = ctx->post_version[1]; + } + if (ctx->have_size) { + size = ctx->post_size; + have_size = 1; + } + } + + UNLOCK(&cbk->inode->lock); + + if (cbk->iatt[0].ia_type == IA_IFREG) { + cbk->size = cbk->iatt[0].ia_size; + ec_dict_del_number(cbk->xdata, EC_XATTR_SIZE, &cbk->iatt[0].ia_size); + if (have_size) { + cbk->iatt[0].ia_size = size; + } + } +} + +int32_t +ec_combine_lookup(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) { + gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_LOOKUP'"); + + return 0; + } + + return 1; +} + +int32_t +ec_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata, + struct iatt *postparent) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + uint64_t dirty[2] = {0}; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LOOKUP, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (inode != NULL) { + cbk->inode = inode_ref(inode); + if (cbk->inode == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_INODE_REF_FAIL, + "Failed to reference an inode."); + + goto out; + } + } + if (buf != NULL) { + cbk->iatt[0] = *buf; + } + if (postparent != NULL) { + cbk->iatt[1] = *postparent; + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + ec_dict_del_array(xdata, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); + } + + ec_combine(cbk, ec_combine_lookup); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_lookup(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_lookup_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->lookup, + &fop->loc[0], fop->xdata); +} + +int32_t +ec_manager_lookup(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + int32_t err; + + switch (state) { + case EC_STATE_INIT: + if (fop->xdata == NULL) { + fop->xdata = dict_new(); + if (fop->xdata == NULL) { + gf_msg(fop->xl->name, GF_LOG_ERROR, ENOMEM, + EC_MSG_LOOKUP_REQ_PREP_FAIL, + "Unable to prepare " + "lookup request"); + + fop->error = ENOMEM; + + return EC_STATE_REPORT; + } + } else { + /*TODO: To be handled once we have 'syndromes' */ + dict_del(fop->xdata, GF_CONTENT_KEY); + } + err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0); + if (err == 0) { + err = dict_set_uint64(fop->xdata, EC_XATTR_VERSION, 0); + } + if (err == 0) { + err = dict_set_uint64(fop->xdata, EC_XATTR_DIRTY, 0); + } + if (err != 0) { + gf_msg(fop->xl->name, GF_LOG_ERROR, -err, + EC_MSG_LOOKUP_REQ_PREP_FAIL, + "Unable to prepare lookup " + "request"); + + fop->error = -err; + + return EC_STATE_REPORT; + } + + /* Fall through */ + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + /* + * Lookup happens without any lock, so there is a chance that it + * will have answers before modification happened and after + * modification happened in the same response. So choose the next + * best answer when the answers don't match for EC_MINIMUM_MIN + */ + + if (!fop->answer && !list_empty(&fop->cbk_list)) { + fop->answer = list_entry(fop->cbk_list.next, ec_cbk_data_t, + list); + } + + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + ec_lookup_rebuild(fop->xl->private, fop, cbk); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.lookup != NULL) { + fop->cbks.lookup(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->inode, &cbk->iatt[0], + cbk->xdata, &cbk->iatt[1]); + } + + return EC_STATE_END; + + case -EC_STATE_INIT: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.lookup != NULL) { + fop->cbks.lookup(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL, NULL); + } + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc, + dict_t *xdata) +{ + ec_cbk_t callback = {.lookup = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(LOOKUP) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_LOOKUP, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_lookup, + ec_manager_lookup, callback, data); + if (fop == NULL) { + goto out; + } + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + /* Do not log failures here as a memory problem would have already + * been logged by the corresponding alloc functions */ + if (fop->xdata == NULL) + goto out; + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL); + } +} + +/* FOP: statfs */ + +int32_t +ec_combine_statfs(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + ec_statvfs_combine(&dst->statvfs, &src->statvfs); + + return 1; +} + +int32_t +ec_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct statvfs *buf, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STATFS, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (buf != NULL) { + cbk->statvfs = *buf; + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, ec_combine_statfs); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_statfs(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_statfs_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->statfs, + &fop->loc[0], fop->xdata); +} + +int32_t +ec_manager_statfs(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk = NULL; + gf_boolean_t deem_statfs_enabled = _gf_false; + int32_t err = 0; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + ec_t *ec = fop->xl->private; + + if (cbk->xdata) { + err = dict_get_int8(cbk->xdata, "quota-deem-statfs", + (int8_t *)&deem_statfs_enabled); + if (err != -ENOENT) { + ec_cbk_set_error(cbk, -err, _gf_true); + } + } + + if (err != 0 || deem_statfs_enabled == _gf_false) { + cbk->statvfs.f_blocks *= ec->fragments; + cbk->statvfs.f_bfree *= ec->fragments; + cbk->statvfs.f_bavail *= ec->fragments; + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.statfs != NULL) { + fop->cbks.statfs(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, &cbk->statvfs, cbk->xdata); + } + + return EC_STATE_END; + + case -EC_STATE_INIT: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.statfs != NULL) { + fop->cbks.statfs(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL); + } + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc, + dict_t *xdata) +{ + ec_cbk_t callback = {.statfs = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(STATFS) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_STATFS, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_statfs, + ec_manager_statfs, callback, data); + if (fop == NULL) { + goto out; + } + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} + +/* FOP: xattrop */ + +int32_t +ec_combine_xattrop(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (!ec_dict_compare(dst->dict, src->dict)) { + gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_DICT_MISMATCH, + "Mismatching dictionary in " + "answers of 'GF_FOP_XATTROP'"); + + return 0; + } + + return 1; +} + +int32_t +ec_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_lock_link_t *link = NULL; + ec_cbk_data_t *cbk = NULL; + uint64_t dirty[2] = {0}; + data_t *data; + uint64_t *version; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, + op_errno); + if (!cbk) + goto out; + + if (op_ret >= 0) { + cbk->dict = dict_ref(xattr); + + data = dict_get(cbk->dict, EC_XATTR_VERSION); + if ((data != NULL) && (data->len >= sizeof(uint64_t))) { + version = (uint64_t *)data->data; + + if (((ntoh64(version[0]) >> EC_SELFHEAL_BIT) & 1) != 0) { + LOCK(&fop->lock); + + fop->healing |= 1ULL << idx; + + UNLOCK(&fop->lock); + } + } + + ec_dict_del_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); + link = fop->data; + if (link) { + /*Keep a note of if the dirty is already set or not*/ + link->dirty[0] |= (dirty[0] != 0); + link->dirty[1] |= (dirty[1] != 0); + } + } + + if (xdata) + cbk->xdata = dict_ref(xdata); + + ec_combine(cbk, ec_combine_xattrop); + +out: + if (fop) + ec_complete(fop); + + return 0; +} + +void +ec_wind_xattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_xattrop_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->xattrop, + &fop->loc[0], fop->xattrop_flags, fop->dict, fop->xdata); +} + +int32_t +ec_manager_xattrop(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + if (fop->fd == NULL) { + ec_lock_prepare_inode(fop, &fop->loc[0], EC_UPDATE_META, 0, + EC_RANGE_FULL); + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META, 0, + EC_RANGE_FULL); + } + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + err = ec_dict_combine(cbk, EC_COMBINE_DICT); + ec_cbk_set_error(cbk, -err, _gf_false); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->id == GF_FOP_XATTROP) { + if (fop->cbks.xattrop != NULL) { + fop->cbks.xattrop(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->dict, cbk->xdata); + } + } else { + if (fop->cbks.fxattrop != NULL) { + fop->cbks.fxattrop(fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, cbk->dict, + cbk->xdata); + } + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->id == GF_FOP_XATTROP) { + if (fop->cbks.xattrop != NULL) { + fop->cbks.xattrop(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL); + } + } else { + if (fop->cbks.fxattrop != NULL) { + fop->cbks.fxattrop(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL); + } + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata) +{ + ec_cbk_t callback = {.xattrop = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(XATTROP) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_XATTROP, 0, target, + fop_flags, ec_wind_xattrop, ec_manager_xattrop, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->xattrop_flags = optype; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xattr != NULL) { + fop->dict = dict_ref(xattr); + if (fop->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} + +void +ec_wind_fxattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_xattrop_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fxattrop, + fop->fd, fop->xattrop_flags, fop->dict, fop->xdata); +} + +void +ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata) +{ + ec_cbk_t callback = {.fxattrop = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FXATTROP) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FXATTROP, 0, target, + fop_flags, ec_wind_fxattrop, ec_manager_xattrop, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->xattrop_flags = optype; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xattr != NULL) { + fop->dict = dict_ref(xattr); + if (fop->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} + +/* FOP: IPC */ + +int32_t +ec_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_IPC, idx, op_ret, + op_errno); + + if (cbk != NULL) { + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + } + + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_ipc(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_ipc_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->ipc, fop->int32, + fop->xdata); +} + +int32_t +ec_manager_ipc(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + ec_fop_prepare_answer(fop, _gf_true); + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + if (fop->cbks.ipc != NULL) { + fop->cbks.ipc(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); + } + + return EC_STATE_END; + + case -EC_STATE_INIT: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.ipc != NULL) { + fop->cbks.ipc(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL); + } + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op, + dict_t *xdata) +{ + ec_cbk_t callback = {.ipc = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(IPC) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_IPC, 0, target, fop_flags, + ec_wind_ipc, ec_manager_ipc, callback, data); + if (fop == NULL) { + goto out; + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + } + fop->int32 = op; + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} diff --git a/xlators/cluster/ec/src/ec-gf8.c b/xlators/cluster/ec/src/ec-gf8.c new file mode 100644 index 00000000000..039adae5929 --- /dev/null +++ b/xlators/cluster/ec/src/ec-gf8.c @@ -0,0 +1,5882 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "ec-gf8.h" + +static ec_gf_op_t ec_gf8_mul_00_ops[] = {{EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_00 = {0, + { + 0, + }, + ec_gf8_mul_00_ops}; + +static ec_gf_op_t ec_gf8_mul_01_ops[] = {{EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_01 = {8, + { + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_01_ops}; + +static ec_gf_op_t ec_gf8_mul_02_ops[] = {{EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_02 = {8, + { + 7, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_02_ops}; + +static ec_gf_op_t ec_gf8_mul_03_ops[] = { + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_COPY, 8, 3, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_03 = {9, + { + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + }, + ec_gf8_mul_03_ops}; + +static ec_gf_op_t ec_gf8_mul_04_ops[] = { + {EC_GF_OP_XOR3, 8, 6, 7}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_04 = {9, + { + 6, + 7, + 0, + 1, + 2, + 3, + 4, + 5, + 8, + }, + ec_gf8_mul_04_ops}; + +static ec_gf_op_t ec_gf8_mul_05_ops[] = { + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_05 = {8, + { + 0, + 1, + 2, + 6, + 7, + 3, + 4, + 5, + }, + ec_gf8_mul_05_ops}; + +static ec_gf_op_t ec_gf8_mul_06_ops[] = { + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_COPY, 8, 2, 0}, + {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_06 = {9, + { + 7, + 0, + 1, + 2, + 8, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_06_ops}; + +static ec_gf_op_t ec_gf8_mul_07_ops[] = { + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_07 = {8, + { + 6, + 0, + 1, + 3, + 2, + 4, + 5, + 7, + }, + ec_gf8_mul_07_ops}; + +static ec_gf_op_t ec_gf8_mul_08_ops[] = { + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 6, 7}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_08 = {9, + { + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 4, + 8, + }, + ec_gf8_mul_08_ops}; + +static ec_gf_op_t ec_gf8_mul_09_ops[] = { + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_09 = {8, + { + 0, + 1, + 2, + 3, + 5, + 6, + 7, + 4, + }, + ec_gf8_mul_09_ops}; + +static ec_gf_op_t ec_gf8_mul_0A_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0A = {8, + { + 5, + 0, + 1, + 2, + 6, + 7, + 3, + 4, + }, + ec_gf8_mul_0A_ops}; + +static ec_gf_op_t ec_gf8_mul_0B_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 5, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR3, 3, 8, 6}, {EC_GF_OP_XOR2, 1, 9, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0B = {10, + { + 7, + 1, + 5, + 2, + 4, + 3, + 0, + 6, + 8, + 9, + }, + ec_gf8_mul_0B_ops}; + +static ec_gf_op_t ec_gf8_mul_0C_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0}, + {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0C = {9, + { + 5, + 7, + 0, + 1, + 8, + 2, + 3, + 4, + 6, + }, + ec_gf8_mul_0C_ops}; + +static ec_gf_op_t ec_gf8_mul_0D_ops[] = { + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR3, 2, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0D = {9, + { + 5, + 6, + 7, + 3, + 1, + 0, + 2, + 4, + 8, + }, + ec_gf8_mul_0D_ops}; + +static ec_gf_op_t ec_gf8_mul_0E_ops[] = { + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0E = {8, + { + 7, + 0, + 6, + 1, + 3, + 2, + 4, + 5, + }, + ec_gf8_mul_0E_ops}; + +static ec_gf_op_t ec_gf8_mul_0F_ops[] = { + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_0F = {8, + { + 1, + 0, + 5, + 6, + 7, + 2, + 3, + 4, + }, + ec_gf8_mul_0F_ops}; + +static ec_gf_op_t ec_gf8_mul_10_ops[] = { + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_10 = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + }, + ec_gf8_mul_10_ops}; + +static ec_gf_op_t ec_gf8_mul_11_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_11 = {8, + { + 4, + 1, + 2, + 6, + 0, + 5, + 7, + 3, + }, + ec_gf8_mul_11_ops}; + +static ec_gf_op_t ec_gf8_mul_12_ops[] = { + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_12 = {8, + { + 7, + 0, + 1, + 2, + 3, + 5, + 6, + 4, + }, + ec_gf8_mul_12_ops}; + +static ec_gf_op_t ec_gf8_mul_13_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 3, 7}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_13 = {9, + { + 4, + 5, + 2, + 6, + 0, + 1, + 7, + 3, + 8, + }, + ec_gf8_mul_13_ops}; + +static ec_gf_op_t ec_gf8_mul_14_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_14 = {8, + { + 6, + 7, + 0, + 1, + 2, + 4, + 5, + 3, + }, + ec_gf8_mul_14_ops}; + +static ec_gf_op_t ec_gf8_mul_15_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR3, 5, 8, 7}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_15 = {9, + { + 0, + 1, + 2, + 4, + 7, + 6, + 5, + 3, + 8, + }, + ec_gf8_mul_15_ops}; + +static ec_gf_op_t ec_gf8_mul_16_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_16 = {8, + { + 6, + 7, + 4, + 1, + 2, + 3, + 5, + 0, + }, + ec_gf8_mul_16_ops}; + +static ec_gf_op_t ec_gf8_mul_17_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_17 = {8, + { + 5, + 7, + 0, + 1, + 3, + 2, + 4, + 6, + }, + ec_gf8_mul_17_ops}; + +static ec_gf_op_t ec_gf8_mul_18_ops[] = { + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_18 = {9, + { + 4, + 5, + 7, + 6, + 0, + 1, + 2, + 3, + 8, + }, + ec_gf8_mul_18_ops}; + +static ec_gf_op_t ec_gf8_mul_19_ops[] = { + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_19 = {8, + { + 0, + 5, + 2, + 6, + 7, + 1, + 3, + 4, + }, + ec_gf8_mul_19_ops}; + +static ec_gf_op_t ec_gf8_mul_1A_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1A = {8, + { + 7, + 0, + 4, + 5, + 3, + 1, + 2, + 6, + }, + ec_gf8_mul_1A_ops}; + +static ec_gf_op_t ec_gf8_mul_1B_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1B = {8, + { + 7, + 4, + 5, + 6, + 3, + 1, + 2, + 0, + }, + ec_gf8_mul_1B_ops}; + +static ec_gf_op_t ec_gf8_mul_1C_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1C = {8, + { + 5, + 4, + 3, + 0, + 1, + 7, + 2, + 6, + }, + ec_gf8_mul_1C_ops}; + +static ec_gf_op_t ec_gf8_mul_1D_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 4, 2}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1D = {9, + { + 0, + 7, + 5, + 8, + 2, + 3, + 4, + 1, + 6, + }, + ec_gf8_mul_1D_ops}; + +static ec_gf_op_t ec_gf8_mul_1E_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1E = {8, + { + 4, + 7, + 5, + 1, + 6, + 0, + 2, + 3, + }, + ec_gf8_mul_1E_ops}; + +static ec_gf_op_t ec_gf8_mul_1F_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR3, 8, 3, 7}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_1F = {9, + { + 1, + 4, + 5, + 6, + 7, + 0, + 3, + 2, + 8, + }, + ec_gf8_mul_1F_ops}; + +static ec_gf_op_t ec_gf8_mul_20_ops[] = { + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_20 = {8, + { + 7, + 4, + 5, + 6, + 3, + 0, + 1, + 2, + }, + ec_gf8_mul_20_ops}; + +static ec_gf_op_t ec_gf8_mul_21_ops[] = { + {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR3, 8, 7, 5}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_21 = {10, + { + 0, + 1, + 2, + 7, + 5, + 4, + 3, + 6, + 8, + 9, + }, + ec_gf8_mul_21_ops}; + +static ec_gf_op_t ec_gf8_mul_22_ops[] = { + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_22 = {8, + { + 3, + 0, + 5, + 2, + 6, + 4, + 1, + 7, + }, + ec_gf8_mul_22_ops}; + +static ec_gf_op_t ec_gf8_mul_23_ops[] = { + {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_23 = {9, + { + 0, + 4, + 3, + 2, + 5, + 6, + 1, + 8, + 7, + }, + ec_gf8_mul_23_ops}; + +static ec_gf_op_t ec_gf8_mul_24_ops[] = { + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_24 = {8, + { + 6, + 7, + 0, + 1, + 2, + 4, + 5, + 3, + }, + ec_gf8_mul_24_ops}; + +static ec_gf_op_t ec_gf8_mul_25_ops[] = { + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_25 = {8, + { + 2, + 7, + 0, + 1, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_25_ops}; + +static ec_gf_op_t ec_gf8_mul_26_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_26 = {8, + { + 3, + 4, + 1, + 2, + 0, + 5, + 6, + 7, + }, + ec_gf8_mul_26_ops}; + +static ec_gf_op_t ec_gf8_mul_27_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_27 = {8, + { + 3, + 0, + 1, + 2, + 6, + 7, + 4, + 5, + }, + ec_gf8_mul_27_ops}; + +static ec_gf_op_t ec_gf8_mul_28_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_28 = {8, + { + 5, + 6, + 3, + 0, + 1, + 2, + 4, + 7, + }, + ec_gf8_mul_28_ops}; + +static ec_gf_op_t ec_gf8_mul_29_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_29 = {8, + { + 4, + 6, + 3, + 5, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_29_ops}; + +static ec_gf_op_t ec_gf8_mul_2A_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR3, 6, 8, 4}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2A = {9, + { + 3, + 4, + 7, + 2, + 6, + 5, + 1, + 0, + 8, + }, + ec_gf8_mul_2A_ops}; + +static ec_gf_op_t ec_gf8_mul_2B_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2B = {8, + { + 3, + 4, + 7, + 5, + 6, + 0, + 1, + 2, + }, + ec_gf8_mul_2B_ops}; + +static ec_gf_op_t ec_gf8_mul_2C_ops[] = { + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2C = {8, + { + 5, + 6, + 7, + 0, + 2, + 3, + 4, + 1, + }, + ec_gf8_mul_2C_ops}; + +static ec_gf_op_t ec_gf8_mul_2D_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 6}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2D = {9, + { + 7, + 0, + 3, + 5, + 1, + 4, + 2, + 6, + 8, + }, + ec_gf8_mul_2D_ops}; + +static ec_gf_op_t ec_gf8_mul_2E_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2E = {9, + { + 5, + 0, + 7, + 3, + 2, + 6, + 4, + 1, + 8, + }, + ec_gf8_mul_2E_ops}; + +static ec_gf_op_t ec_gf8_mul_2F_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR3, 8, 7, 6}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_2F = {9, + { + 6, + 3, + 2, + 5, + 7, + 0, + 1, + 4, + 8, + }, + ec_gf8_mul_2F_ops}; + +static ec_gf_op_t ec_gf8_mul_30_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 7}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_30 = {9, + { + 3, + 4, + 7, + 5, + 0, + 6, + 1, + 2, + 8, + }, + ec_gf8_mul_30_ops}; + +static ec_gf_op_t ec_gf8_mul_31_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_31 = {8, + { + 7, + 1, + 4, + 5, + 6, + 0, + 2, + 3, + }, + ec_gf8_mul_31_ops}; + +static ec_gf_op_t ec_gf8_mul_32_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_32 = {8, + { + 3, + 4, + 6, + 7, + 5, + 0, + 1, + 2, + }, + ec_gf8_mul_32_ops}; + +static ec_gf_op_t ec_gf8_mul_33_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_33 = {8, + { + 5, + 4, + 3, + 0, + 2, + 1, + 6, + 7, + }, + ec_gf8_mul_33_ops}; + +static ec_gf_op_t ec_gf8_mul_34_ops[] = { + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_34 = {8, + { + 7, + 5, + 3, + 0, + 2, + 4, + 1, + 6, + }, + ec_gf8_mul_34_ops}; + +static ec_gf_op_t ec_gf8_mul_35_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_35 = {8, + { + 6, + 7, + 5, + 4, + 2, + 0, + 1, + 3, + }, + ec_gf8_mul_35_ops}; + +static ec_gf_op_t ec_gf8_mul_36_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_36 = {8, + { + 6, + 7, + 4, + 1, + 2, + 3, + 0, + 5, + }, + ec_gf8_mul_36_ops}; + +static ec_gf_op_t ec_gf8_mul_37_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 1}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_37 = {9, + { + 6, + 7, + 2, + 1, + 0, + 3, + 4, + 5, + 8, + }, + ec_gf8_mul_37_ops}; + +static ec_gf_op_t ec_gf8_mul_38_ops[] = { + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 7}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_38 = {9, + { + 4, + 5, + 6, + 3, + 0, + 1, + 7, + 2, + 8, + }, + ec_gf8_mul_38_ops}; + +static ec_gf_op_t ec_gf8_mul_39_ops[] = { + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_39 = {8, + { + 1, + 6, + 3, + 0, + 5, + 2, + 4, + 7, + }, + ec_gf8_mul_39_ops}; + +static ec_gf_op_t ec_gf8_mul_3A_ops[] = { + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3A = {8, + { + 3, + 4, + 7, + 0, + 5, + 6, + 1, + 2, + }, + ec_gf8_mul_3A_ops}; + +static ec_gf_op_t ec_gf8_mul_3B_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR3, 8, 7, 3}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3B = {9, + { + 3, + 0, + 1, + 7, + 6, + 2, + 4, + 8, + 5, + }, + ec_gf8_mul_3B_ops}; + +static ec_gf_op_t ec_gf8_mul_3C_ops[] = { + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3C = {8, + { + 3, + 6, + 4, + 1, + 7, + 2, + 0, + 5, + }, + ec_gf8_mul_3C_ops}; + +static ec_gf_op_t ec_gf8_mul_3D_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3D = {8, + { + 2, + 3, + 4, + 5, + 6, + 7, + 0, + 1, + }, + ec_gf8_mul_3D_ops}; + +static ec_gf_op_t ec_gf8_mul_3E_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3E = {8, + { + 6, + 1, + 2, + 7, + 0, + 3, + 5, + 4, + }, + ec_gf8_mul_3E_ops}; + +static ec_gf_op_t ec_gf8_mul_3F_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_COPY, 10, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_COPY, 9, 2, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR3, 4, 9, 7}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 3, 10, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_3F = {11, + { + 1, + 7, + 6, + 2, + 4, + 3, + 5, + 0, + 8, + 9, + 10, + }, + ec_gf8_mul_3F_ops}; + +static ec_gf_op_t ec_gf8_mul_40_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 7, 6}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_40 = {9, + { + 5, + 7, + 4, + 6, + 2, + 3, + 0, + 1, + 8, + }, + ec_gf8_mul_40_ops}; + +static ec_gf_op_t ec_gf8_mul_41_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 4, 0}, + {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_41 = {9, + { + 0, + 7, + 6, + 5, + 3, + 4, + 8, + 1, + 2, + }, + ec_gf8_mul_41_ops}; + +static ec_gf_op_t ec_gf8_mul_42_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_42 = {9, + { + 2, + 7, + 1, + 6, + 4, + 3, + 0, + 5, + 8, + }, + ec_gf8_mul_42_ops}; + +static ec_gf_op_t ec_gf8_mul_43_ops[] = { + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_43 = {8, + { + 2, + 6, + 4, + 1, + 7, + 3, + 0, + 5, + }, + ec_gf8_mul_43_ops}; + +static ec_gf_op_t ec_gf8_mul_44_ops[] = { + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_44 = {8, + { + 2, + 3, + 4, + 1, + 6, + 5, + 0, + 7, + }, + ec_gf8_mul_44_ops}; + +static ec_gf_op_t ec_gf8_mul_45_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_45 = {8, + { + 2, + 3, + 0, + 1, + 7, + 4, + 5, + 6, + }, + ec_gf8_mul_45_ops}; + +static ec_gf_op_t ec_gf8_mul_46_ops[] = { + {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_46 = {9, + { + 2, + 0, + 1, + 3, + 4, + 5, + 6, + 7, + 8, + }, + ec_gf8_mul_46_ops}; + +static ec_gf_op_t ec_gf8_mul_47_ops[] = { + {EC_GF_OP_XOR3, 8, 0, 1}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_47 = {9, + { + 2, + 3, + 4, + 5, + 6, + 7, + 0, + 1, + 8, + }, + ec_gf8_mul_47_ops}; + +static ec_gf_op_t ec_gf8_mul_48_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_48 = {8, + { + 4, + 5, + 6, + 0, + 1, + 3, + 7, + 2, + }, + ec_gf8_mul_48_ops}; + +static ec_gf_op_t ec_gf8_mul_49_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 6}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR3, 1, 8, 5}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_49 = {9, + { + 7, + 2, + 4, + 0, + 3, + 5, + 1, + 6, + 8, + }, + ec_gf8_mul_49_ops}; + +static ec_gf_op_t ec_gf8_mul_4A_ops[] = { + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4A = {8, + { + 5, + 6, + 7, + 0, + 1, + 3, + 4, + 2, + }, + ec_gf8_mul_4A_ops}; + +static ec_gf_op_t ec_gf8_mul_4B_ops[] = { + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR3, 8, 3, 7}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4B = {9, + { + 5, + 3, + 6, + 7, + 0, + 2, + 4, + 1, + 8, + }, + ec_gf8_mul_4B_ops}; + +static ec_gf_op_t ec_gf8_mul_4C_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4C = {8, + { + 5, + 3, + 4, + 7, + 0, + 6, + 2, + 1, + }, + ec_gf8_mul_4C_ops}; + +static ec_gf_op_t ec_gf8_mul_4D_ops[] = { + {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR3, 9, 3, 1}, + {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 0, 8, 2}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4D = {10, + { + 0, + 9, + 3, + 5, + 6, + 4, + 7, + 1, + 2, + 8, + }, + ec_gf8_mul_4D_ops}; + +static ec_gf_op_t ec_gf8_mul_4E_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4E = {8, + { + 2, + 3, + 0, + 1, + 5, + 6, + 7, + 4, + }, + ec_gf8_mul_4E_ops}; + +static ec_gf_op_t ec_gf8_mul_4F_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_4F = {8, + { + 0, + 3, + 5, + 6, + 1, + 2, + 7, + 4, + }, + ec_gf8_mul_4F_ops}; + +static ec_gf_op_t ec_gf8_mul_50_ops[] = { + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_50 = {8, + { + 4, + 5, + 7, + 3, + 0, + 1, + 2, + 6, + }, + ec_gf8_mul_50_ops}; + +static ec_gf_op_t ec_gf8_mul_51_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_51 = {8, + { + 0, + 1, + 7, + 2, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_51_ops}; + +static ec_gf_op_t ec_gf8_mul_52_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR3, 3, 5, 8}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 9, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_52 = {10, + { + 2, + 3, + 1, + 4, + 6, + 7, + 0, + 5, + 8, + 9, + }, + ec_gf8_mul_52_ops}; + +static ec_gf_op_t ec_gf8_mul_53_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_53 = {8, + { + 2, + 0, + 1, + 4, + 5, + 6, + 7, + 3, + }, + ec_gf8_mul_53_ops}; + +static ec_gf_op_t ec_gf8_mul_54_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_54 = {8, + { + 7, + 3, + 0, + 4, + 2, + 6, + 5, + 1, + }, + ec_gf8_mul_54_ops}; + +static ec_gf_op_t ec_gf8_mul_55_ops[] = { + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_55 = {8, + { + 1, + 5, + 6, + 4, + 3, + 7, + 2, + 0, + }, + ec_gf8_mul_55_ops}; + +static ec_gf_op_t ec_gf8_mul_56_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_56 = {8, + { + 2, + 3, + 0, + 4, + 5, + 6, + 7, + 1, + }, + ec_gf8_mul_56_ops}; + +static ec_gf_op_t ec_gf8_mul_57_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_57 = {8, + { + 2, + 3, + 0, + 1, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_57_ops}; + +static ec_gf_op_t ec_gf8_mul_58_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_58 = {8, + { + 4, + 3, + 2, + 7, + 0, + 1, + 5, + 6, + }, + ec_gf8_mul_58_ops}; + +static ec_gf_op_t ec_gf8_mul_59_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_59 = {8, + { + 7, + 3, + 5, + 6, + 1, + 2, + 0, + 4, + }, + ec_gf8_mul_59_ops}; + +static ec_gf_op_t ec_gf8_mul_5A_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5A = {8, + { + 6, + 7, + 0, + 1, + 2, + 3, + 5, + 4, + }, + ec_gf8_mul_5A_ops}; + +static ec_gf_op_t ec_gf8_mul_5B_ops[] = { + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5B = {8, + { + 6, + 0, + 7, + 5, + 2, + 1, + 3, + 4, + }, + ec_gf8_mul_5B_ops}; + +static ec_gf_op_t ec_gf8_mul_5C_ops[] = { + {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5C = {9, + { + 7, + 5, + 2, + 4, + 1, + 0, + 6, + 3, + 8, + }, + ec_gf8_mul_5C_ops}; + +static ec_gf_op_t ec_gf8_mul_5D_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5D = {8, + { + 1, + 3, + 5, + 4, + 6, + 7, + 2, + 0, + }, + ec_gf8_mul_5D_ops}; + +static ec_gf_op_t ec_gf8_mul_5E_ops[] = { + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5E = {8, + { + 4, + 3, + 6, + 2, + 5, + 7, + 0, + 1, + }, + ec_gf8_mul_5E_ops}; + +static ec_gf_op_t ec_gf8_mul_5F_ops[] = { + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_5F = {8, + { + 6, + 1, + 3, + 4, + 5, + 7, + 2, + 0, + }, + ec_gf8_mul_5F_ops}; + +static ec_gf_op_t ec_gf8_mul_60_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_60 = {8, + { + 2, + 3, + 4, + 7, + 5, + 6, + 0, + 1, + }, + ec_gf8_mul_60_ops}; + +static ec_gf_op_t ec_gf8_mul_61_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_61 = {8, + { + 0, + 5, + 6, + 7, + 4, + 2, + 1, + 3, + }, + ec_gf8_mul_61_ops}; + +static ec_gf_op_t ec_gf8_mul_62_ops[] = { + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_62 = {8, + { + 2, + 0, + 3, + 4, + 5, + 6, + 7, + 1, + }, + ec_gf8_mul_62_ops}; + +static ec_gf_op_t ec_gf8_mul_63_ops[] = { + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_63 = {8, + { + 3, + 4, + 6, + 5, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_63_ops}; + +static ec_gf_op_t ec_gf8_mul_64_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 8, 7, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_64 = {9, + { + 2, + 3, + 4, + 6, + 5, + 7, + 8, + 1, + 0, + }, + ec_gf8_mul_64_ops}; + +static ec_gf_op_t ec_gf8_mul_65_ops[] = { + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_65 = {8, + { + 2, + 5, + 1, + 3, + 4, + 0, + 6, + 7, + }, + ec_gf8_mul_65_ops}; + +static ec_gf_op_t ec_gf8_mul_66_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_66 = {8, + { + 2, + 3, + 1, + 4, + 5, + 7, + 0, + 6, + }, + ec_gf8_mul_66_ops}; + +static ec_gf_op_t ec_gf8_mul_67_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_67 = {8, + { + 2, + 4, + 5, + 6, + 7, + 3, + 1, + 0, + }, + ec_gf8_mul_67_ops}; + +static ec_gf_op_t ec_gf8_mul_68_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_68 = {8, + { + 5, + 7, + 2, + 3, + 0, + 6, + 4, + 1, + }, + ec_gf8_mul_68_ops}; + +static ec_gf_op_t ec_gf8_mul_69_ops[] = { + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_69 = {8, + { + 0, + 1, + 3, + 2, + 4, + 5, + 7, + 6, + }, + ec_gf8_mul_69_ops}; + +static ec_gf_op_t ec_gf8_mul_6A_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6A = {8, + { + 5, + 7, + 4, + 6, + 1, + 2, + 0, + 3, + }, + ec_gf8_mul_6A_ops}; + +static ec_gf_op_t ec_gf8_mul_6B_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6B = {9, + { + 6, + 7, + 2, + 0, + 3, + 1, + 5, + 4, + 8, + }, + ec_gf8_mul_6B_ops}; + +static ec_gf_op_t ec_gf8_mul_6C_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6C = {8, + { + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 4, + }, + ec_gf8_mul_6C_ops}; + +static ec_gf_op_t ec_gf8_mul_6D_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR3, 8, 3, 4}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6D = {9, + { + 3, + 6, + 7, + 0, + 4, + 5, + 1, + 2, + 8, + }, + ec_gf8_mul_6D_ops}; + +static ec_gf_op_t ec_gf8_mul_6E_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6E = {8, + { + 5, + 6, + 3, + 1, + 7, + 2, + 0, + 4, + }, + ec_gf8_mul_6E_ops}; + +static ec_gf_op_t ec_gf8_mul_6F_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR3, 0, 8, 7}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_6F = {9, + { + 2, + 6, + 3, + 7, + 0, + 1, + 4, + 5, + 8, + }, + ec_gf8_mul_6F_ops}; + +static ec_gf_op_t ec_gf8_mul_70_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_70 = {8, + { + 3, + 4, + 5, + 2, + 6, + 0, + 1, + 7, + }, + ec_gf8_mul_70_ops}; + +static ec_gf_op_t ec_gf8_mul_71_ops[] = { + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_71 = {8, + { + 4, + 7, + 5, + 3, + 6, + 0, + 2, + 1, + }, + ec_gf8_mul_71_ops}; + +static ec_gf_op_t ec_gf8_mul_72_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_72 = {8, + { + 0, + 5, + 2, + 7, + 4, + 1, + 3, + 6, + }, + ec_gf8_mul_72_ops}; + +static ec_gf_op_t ec_gf8_mul_73_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_73 = {8, + { + 6, + 0, + 1, + 7, + 4, + 5, + 2, + 3, + }, + ec_gf8_mul_73_ops}; + +static ec_gf_op_t ec_gf8_mul_74_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_74 = {8, + { + 3, + 2, + 1, + 0, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_74_ops}; + +static ec_gf_op_t ec_gf8_mul_75_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_75 = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + }, + ec_gf8_mul_75_ops}; + +static ec_gf_op_t ec_gf8_mul_76_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 8, 6, 2}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 8, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_76 = {9, + { + 2, + 3, + 0, + 6, + 5, + 1, + 7, + 8, + 4, + }, + ec_gf8_mul_76_ops}; + +static ec_gf_op_t ec_gf8_mul_77_ops[] = { + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_77 = {8, + { + 7, + 4, + 3, + 6, + 0, + 1, + 5, + 2, + }, + ec_gf8_mul_77_ops}; + +static ec_gf_op_t ec_gf8_mul_78_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 0, 2}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_78 = {9, + { + 4, + 7, + 3, + 2, + 5, + 1, + 6, + 0, + 8, + }, + ec_gf8_mul_78_ops}; + +static ec_gf_op_t ec_gf8_mul_79_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 7}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_79 = {9, + { + 4, + 5, + 7, + 3, + 1, + 6, + 2, + 0, + 8, + }, + ec_gf8_mul_79_ops}; + +static ec_gf_op_t ec_gf8_mul_7A_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7A = {8, + { + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 0, + }, + ec_gf8_mul_7A_ops}; + +static ec_gf_op_t ec_gf8_mul_7B_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR3, 8, 5, 3}, + {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_COPY, 9, 4, 0}, + {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 4, 1, 9}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7B = {10, + { + 1, + 2, + 3, + 4, + 8, + 5, + 6, + 0, + 7, + 9, + }, + ec_gf8_mul_7B_ops}; + +static ec_gf_op_t ec_gf8_mul_7C_ops[] = { + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7C = {8, + { + 2, + 4, + 1, + 6, + 3, + 5, + 7, + 0, + }, + ec_gf8_mul_7C_ops}; + +static ec_gf_op_t ec_gf8_mul_7D_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7D = {8, + { + 1, + 0, + 3, + 5, + 6, + 7, + 2, + 4, + }, + ec_gf8_mul_7D_ops}; + +static ec_gf_op_t ec_gf8_mul_7E_ops[] = { + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_COPY, 8, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 6, 2, 7}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7E = {9, + { + 5, + 1, + 2, + 0, + 7, + 3, + 4, + 6, + 8, + }, + ec_gf8_mul_7E_ops}; + +static ec_gf_op_t ec_gf8_mul_7F_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR3, 9, 7, 5}, {EC_GF_OP_XOR2, 2, 9, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 9, 0}, + {EC_GF_OP_XOR3, 9, 6, 4}, {EC_GF_OP_XOR2, 7, 9, 0}, + {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_7F = {10, + { + 4, + 1, + 0, + 5, + 6, + 7, + 2, + 3, + 8, + 9, + }, + ec_gf8_mul_7F_ops}; + +static ec_gf_op_t ec_gf8_mul_80_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_80 = {8, + { + 7, + 5, + 6, + 4, + 1, + 2, + 3, + 0, + }, + ec_gf8_mul_80_ops}; + +static ec_gf_op_t ec_gf8_mul_81_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_81 = {8, + { + 2, + 7, + 4, + 1, + 5, + 6, + 3, + 0, + }, + ec_gf8_mul_81_ops}; + +static ec_gf_op_t ec_gf8_mul_82_ops[] = { + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR3, 5, 8, 7}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_82 = {9, + { + 6, + 2, + 7, + 5, + 1, + 3, + 4, + 0, + 8, + }, + ec_gf8_mul_82_ops}; + +static ec_gf_op_t ec_gf8_mul_83_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_83 = {8, + { + 3, + 5, + 6, + 7, + 1, + 2, + 4, + 0, + }, + ec_gf8_mul_83_ops}; + +static ec_gf_op_t ec_gf8_mul_84_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_84 = {8, + { + 7, + 6, + 0, + 4, + 1, + 5, + 3, + 2, + }, + ec_gf8_mul_84_ops}; + +static ec_gf_op_t ec_gf8_mul_85_ops[] = { + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_85 = {8, + { + 7, + 6, + 0, + 3, + 2, + 4, + 5, + 1, + }, + ec_gf8_mul_85_ops}; + +static ec_gf_op_t ec_gf8_mul_86_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_86 = {8, + { + 1, + 2, + 6, + 4, + 5, + 7, + 3, + 0, + }, + ec_gf8_mul_86_ops}; + +static ec_gf_op_t ec_gf8_mul_87_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0}, + {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR3, 5, 8, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_87 = {9, + { + 1, + 2, + 3, + 4, + 5, + 7, + 6, + 0, + 8, + }, + ec_gf8_mul_87_ops}; + +static ec_gf_op_t ec_gf8_mul_88_ops[] = { + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_88 = {8, + { + 6, + 7, + 3, + 1, + 2, + 4, + 5, + 0, + }, + ec_gf8_mul_88_ops}; + +static ec_gf_op_t ec_gf8_mul_89_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR3, 8, 5, 2}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_89 = {9, + { + 2, + 1, + 6, + 5, + 7, + 3, + 4, + 0, + 8, + }, + ec_gf8_mul_89_ops}; + +static ec_gf_op_t ec_gf8_mul_8A_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8A = {8, + { + 1, + 2, + 3, + 0, + 6, + 7, + 4, + 5, + }, + ec_gf8_mul_8A_ops}; + +static ec_gf_op_t ec_gf8_mul_8B_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8B = {8, + { + 6, + 1, + 2, + 3, + 5, + 7, + 4, + 0, + }, + ec_gf8_mul_8B_ops}; + +static ec_gf_op_t ec_gf8_mul_8C_ops[] = { + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8C = {8, + { + 1, + 2, + 0, + 7, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_8C_ops}; + +static ec_gf_op_t ec_gf8_mul_8D_ops[] = { + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8D = {8, + { + 7, + 1, + 3, + 2, + 4, + 5, + 0, + 6, + }, + ec_gf8_mul_8D_ops}; + +static ec_gf_op_t ec_gf8_mul_8E_ops[] = {{EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8E = {8, + { + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 0, + }, + ec_gf8_mul_8E_ops}; + +static ec_gf_op_t ec_gf8_mul_8F_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_8F = {8, + { + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 0, + }, + ec_gf8_mul_8F_ops}; + +static ec_gf_op_t ec_gf8_mul_90_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_90 = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 3, + 2, + }, + ec_gf8_mul_90_ops}; + +static ec_gf_op_t ec_gf8_mul_91_ops[] = { + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_COPY, 9, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 9, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR3, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_91 = {10, + { + 2, + 3, + 1, + 4, + 0, + 6, + 7, + 5, + 8, + 9, + }, + ec_gf8_mul_91_ops}; + +static ec_gf_op_t ec_gf8_mul_92_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_92 = {8, + { + 6, + 7, + 0, + 1, + 2, + 3, + 5, + 4, + }, + ec_gf8_mul_92_ops}; + +static ec_gf_op_t ec_gf8_mul_93_ops[] = { + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_93 = {8, + { + 6, + 4, + 5, + 1, + 7, + 2, + 3, + 0, + }, + ec_gf8_mul_93_ops}; + +static ec_gf_op_t ec_gf8_mul_94_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_94 = {8, + { + 7, + 5, + 0, + 2, + 6, + 1, + 3, + 4, + }, + ec_gf8_mul_94_ops}; + +static ec_gf_op_t ec_gf8_mul_95_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_95 = {8, + { + 7, + 6, + 1, + 3, + 0, + 4, + 5, + 2, + }, + ec_gf8_mul_95_ops}; + +static ec_gf_op_t ec_gf8_mul_96_ops[] = { + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR3, 8, 0, 4}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_96 = {9, + { + 4, + 0, + 1, + 6, + 7, + 2, + 3, + 5, + 8, + }, + ec_gf8_mul_96_ops}; + +static ec_gf_op_t ec_gf8_mul_97_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 8, 6, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_97 = {9, + { + 4, + 5, + 3, + 6, + 7, + 1, + 2, + 0, + 8, + }, + ec_gf8_mul_97_ops}; + +static ec_gf_op_t ec_gf8_mul_98_ops[] = { + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_98 = {8, + { + 4, + 2, + 3, + 6, + 7, + 5, + 1, + 0, + }, + ec_gf8_mul_98_ops}; + +static ec_gf_op_t ec_gf8_mul_99_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_99 = {8, + { + 6, + 5, + 3, + 7, + 0, + 1, + 4, + 2, + }, + ec_gf8_mul_99_ops}; + +static ec_gf_op_t ec_gf8_mul_9A_ops[] = { + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9A = {9, + { + 6, + 3, + 4, + 0, + 5, + 1, + 2, + 7, + 8, + }, + ec_gf8_mul_9A_ops}; + +static ec_gf_op_t ec_gf8_mul_9B_ops[] = { + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_COPY, 9, 5, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR3, 8, 3, 2}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 9, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9B = {10, + { + 4, + 5, + 8, + 6, + 7, + 1, + 2, + 0, + 3, + 9, + }, + ec_gf8_mul_9B_ops}; + +static ec_gf_op_t ec_gf8_mul_9C_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9C = {8, + { + 3, + 2, + 1, + 0, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_9C_ops}; + +static ec_gf_op_t ec_gf8_mul_9D_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9D = {8, + { + 0, + 1, + 2, + 3, + 7, + 4, + 5, + 6, + }, + ec_gf8_mul_9D_ops}; + +static ec_gf_op_t ec_gf8_mul_9E_ops[] = { + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_COPY, 8, 7, 0}, + {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9E = {9, + { + 4, + 5, + 3, + 8, + 6, + 0, + 2, + 7, + 1, + }, + ec_gf8_mul_9E_ops}; + +static ec_gf_op_t ec_gf8_mul_9F_ops[] = { + {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_9F = {9, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 8, + }, + ec_gf8_mul_9F_ops}; + +static ec_gf_op_t ec_gf8_mul_A0_ops[] = { + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A0 = {8, + { + 3, + 1, + 6, + 7, + 5, + 2, + 4, + 0, + }, + ec_gf8_mul_A0_ops}; + +static ec_gf_op_t ec_gf8_mul_A1_ops[] = { + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 0, 6}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A1 = {9, + { + 7, + 4, + 1, + 5, + 6, + 0, + 2, + 3, + 8, + }, + ec_gf8_mul_A1_ops}; + +static ec_gf_op_t ec_gf8_mul_A2_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A2 = {8, + { + 7, + 0, + 6, + 3, + 2, + 1, + 4, + 5, + }, + ec_gf8_mul_A2_ops}; + +static ec_gf_op_t ec_gf8_mul_A3_ops[] = { + {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A3 = {9, + { + 3, + 7, + 2, + 6, + 1, + 4, + 0, + 5, + 8, + }, + ec_gf8_mul_A3_ops}; + +static ec_gf_op_t ec_gf8_mul_A4_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A4 = {8, + { + 5, + 6, + 7, + 2, + 4, + 3, + 0, + 1, + }, + ec_gf8_mul_A4_ops}; + +static ec_gf_op_t ec_gf8_mul_A5_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR3, 8, 5, 6}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A5 = {9, + { + 1, + 4, + 2, + 5, + 6, + 7, + 3, + 0, + 8, + }, + ec_gf8_mul_A5_ops}; + +static ec_gf_op_t ec_gf8_mul_A6_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A6 = {8, + { + 1, + 2, + 0, + 3, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_A6_ops}; + +static ec_gf_op_t ec_gf8_mul_A7_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A7 = {8, + { + 0, + 1, + 2, + 5, + 6, + 7, + 3, + 4, + }, + ec_gf8_mul_A7_ops}; + +static ec_gf_op_t ec_gf8_mul_A8_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 8, 3, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 9, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A8 = {10, + { + 1, + 7, + 5, + 8, + 6, + 3, + 4, + 0, + 2, + 9, + }, + ec_gf8_mul_A8_ops}; + +static ec_gf_op_t ec_gf8_mul_A9_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_A9 = {8, + { + 3, + 7, + 6, + 1, + 2, + 0, + 4, + 5, + }, + ec_gf8_mul_A9_ops}; + +static ec_gf_op_t ec_gf8_mul_AA_ops[] = { + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AA = {8, + { + 0, + 4, + 5, + 3, + 6, + 7, + 1, + 2, + }, + ec_gf8_mul_AA_ops}; + +static ec_gf_op_t ec_gf8_mul_AB_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_COPY, 9, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR3, 3, 9, 7}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AB = {10, + { + 2, + 3, + 8, + 0, + 5, + 6, + 1, + 4, + 7, + 9, + }, + ec_gf8_mul_AB_ops}; + +static ec_gf_op_t ec_gf8_mul_AC_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AC = {8, + { + 3, + 2, + 1, + 0, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_AC_ops}; + +static ec_gf_op_t ec_gf8_mul_AD_ops[] = { + {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AD = {9, + { + 3, + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 8, + }, + ec_gf8_mul_AD_ops}; + +static ec_gf_op_t ec_gf8_mul_AE_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_COPY, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AE = {9, + { + 7, + 0, + 5, + 6, + 3, + 4, + 1, + 2, + 8, + }, + ec_gf8_mul_AE_ops}; + +static ec_gf_op_t ec_gf8_mul_AF_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_AF = {8, + { + 0, + 1, + 2, + 7, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_AF_ops}; + +static ec_gf_op_t ec_gf8_mul_B0_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B0 = {8, + { + 4, + 0, + 7, + 2, + 3, + 1, + 6, + 5, + }, + ec_gf8_mul_B0_ops}; + +static ec_gf_op_t ec_gf8_mul_B1_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_COPY, 8, 4, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR3, 5, 8, 1}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B1 = {9, + { + 2, + 6, + 4, + 7, + 0, + 1, + 3, + 5, + 8, + }, + ec_gf8_mul_B1_ops}; + +static ec_gf_op_t ec_gf8_mul_B2_ops[] = { + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 4, 5}, + {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 8, 1, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B2 = {9, + { + 0, + 7, + 4, + 5, + 6, + 1, + 2, + 3, + 8, + }, + ec_gf8_mul_B2_ops}; + +static ec_gf_op_t ec_gf8_mul_B3_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 9, 5, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 4}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR3, 1, 9, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B3 = {10, + { + 2, + 3, + 4, + 5, + 1, + 6, + 0, + 7, + 8, + 9, + }, + ec_gf8_mul_B3_ops}; + +static ec_gf_op_t ec_gf8_mul_B4_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B4 = {8, + { + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 4, + }, + ec_gf8_mul_B4_ops}; + +static ec_gf_op_t ec_gf8_mul_B5_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR3, 4, 8, 3}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B5 = {9, + { + 3, + 4, + 0, + 7, + 1, + 5, + 6, + 2, + 8, + }, + ec_gf8_mul_B5_ops}; + +static ec_gf_op_t ec_gf8_mul_B6_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B6 = {8, + { + 5, + 3, + 6, + 4, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_B6_ops}; + +static ec_gf_op_t ec_gf8_mul_B7_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B7 = {8, + { + 5, + 0, + 1, + 4, + 2, + 6, + 7, + 3, + }, + ec_gf8_mul_B7_ops}; + +static ec_gf_op_t ec_gf8_mul_B8_ops[] = { + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B8 = {8, + { + 6, + 4, + 5, + 1, + 2, + 0, + 7, + 3, + }, + ec_gf8_mul_B8_ops}; + +static ec_gf_op_t ec_gf8_mul_B9_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR3, 0, 8, 2}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_B9 = {9, + { + 6, + 7, + 0, + 2, + 1, + 4, + 5, + 3, + 8, + }, + ec_gf8_mul_B9_ops}; + +static ec_gf_op_t ec_gf8_mul_BA_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BA = {8, + { + 1, + 2, + 4, + 3, + 5, + 6, + 0, + 7, + }, + ec_gf8_mul_BA_ops}; + +static ec_gf_op_t ec_gf8_mul_BB_ops[] = { + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 3, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BB = {9, + { + 7, + 2, + 1, + 8, + 3, + 5, + 6, + 4, + 0, + }, + ec_gf8_mul_BB_ops}; + +static ec_gf_op_t ec_gf8_mul_BC_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 2, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR3, 2, 8, 4}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BC = {9, + { + 2, + 6, + 3, + 4, + 5, + 1, + 7, + 0, + 8, + }, + ec_gf8_mul_BC_ops}; + +static ec_gf_op_t ec_gf8_mul_BD_ops[] = { + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BD = {8, + { + 4, + 5, + 0, + 2, + 7, + 1, + 6, + 3, + }, + ec_gf8_mul_BD_ops}; + +static ec_gf_op_t ec_gf8_mul_BE_ops[] = { + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BE = {8, + { + 0, + 6, + 7, + 4, + 5, + 1, + 3, + 2, + }, + ec_gf8_mul_BE_ops}; + +static ec_gf_op_t ec_gf8_mul_BF_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_BF = {8, + { + 5, + 6, + 1, + 7, + 3, + 0, + 2, + 4, + }, + ec_gf8_mul_BF_ops}; + +static ec_gf_op_t ec_gf8_mul_C0_ops[] = { + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C0 = {8, + { + 1, + 2, + 3, + 4, + 7, + 5, + 6, + 0, + }, + ec_gf8_mul_C0_ops}; + +static ec_gf_op_t ec_gf8_mul_C1_ops[] = { + {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C1 = {9, + { + 5, + 6, + 7, + 4, + 1, + 2, + 3, + 0, + 8, + }, + ec_gf8_mul_C1_ops}; + +static ec_gf_op_t ec_gf8_mul_C2_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C2 = {8, + { + 7, + 6, + 3, + 0, + 1, + 4, + 5, + 2, + }, + ec_gf8_mul_C2_ops}; + +static ec_gf_op_t ec_gf8_mul_C3_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR3, 0, 2, 6}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR3, 9, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 9, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C3 = {10, + { + 5, + 6, + 4, + 7, + 1, + 2, + 3, + 0, + 8, + 9, + }, + ec_gf8_mul_C3_ops}; + +static ec_gf_op_t ec_gf8_mul_C4_ops[] = { + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C4 = {8, + { + 0, + 2, + 1, + 3, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_C4_ops}; + +static ec_gf_op_t ec_gf8_mul_C5_ops[] = { + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C5 = {8, + { + 4, + 3, + 5, + 7, + 6, + 2, + 0, + 1, + }, + ec_gf8_mul_C5_ops}; + +static ec_gf_op_t ec_gf8_mul_C6_ops[] = { + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_COPY, 8, 4, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 9, 5, 4}, + {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 8, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C6 = {10, + { + 6, + 3, + 0, + 4, + 5, + 7, + 2, + 1, + 8, + 9, + }, + ec_gf8_mul_C6_ops}; + +static ec_gf_op_t ec_gf8_mul_C7_ops[] = { + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C7 = {8, + { + 7, + 0, + 6, + 2, + 5, + 3, + 4, + 1, + }, + ec_gf8_mul_C7_ops}; + +static ec_gf_op_t ec_gf8_mul_C8_ops[] = { + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C8 = {8, + { + 1, + 3, + 2, + 4, + 6, + 7, + 5, + 0, + }, + ec_gf8_mul_C8_ops}; + +static ec_gf_op_t ec_gf8_mul_C9_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_C9 = {8, + { + 2, + 3, + 4, + 5, + 6, + 7, + 0, + 1, + }, + ec_gf8_mul_C9_ops}; + +static ec_gf_op_t ec_gf8_mul_CA_ops[] = { + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CA = {8, + { + 1, + 2, + 5, + 7, + 3, + 4, + 0, + 6, + }, + ec_gf8_mul_CA_ops}; + +static ec_gf_op_t ec_gf8_mul_CB_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CB = {8, + { + 2, + 3, + 4, + 5, + 7, + 6, + 0, + 1, + }, + ec_gf8_mul_CB_ops}; + +static ec_gf_op_t ec_gf8_mul_CC_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CC = {8, + { + 2, + 7, + 1, + 0, + 5, + 6, + 3, + 4, + }, + ec_gf8_mul_CC_ops}; + +static ec_gf_op_t ec_gf8_mul_CD_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CD = {8, + { + 0, + 6, + 1, + 2, + 7, + 3, + 4, + 5, + }, + ec_gf8_mul_CD_ops}; + +static ec_gf_op_t ec_gf8_mul_CE_ops[] = { + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_COPY, 8, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 3, 6, 8}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 2, 3}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CE = {9, + { + 5, + 7, + 3, + 0, + 2, + 6, + 4, + 1, + 8, + }, + ec_gf8_mul_CE_ops}; + +static ec_gf_op_t ec_gf8_mul_CF_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_CF = {8, + { + 3, + 6, + 7, + 0, + 2, + 4, + 5, + 1, + }, + ec_gf8_mul_CF_ops}; + +static ec_gf_op_t ec_gf8_mul_D0_ops[] = { + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D0 = {8, + { + 5, + 6, + 7, + 2, + 0, + 3, + 1, + 4, + }, + ec_gf8_mul_D0_ops}; + +static ec_gf_op_t ec_gf8_mul_D1_ops[] = { + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D1 = {9, + { + 5, + 6, + 3, + 2, + 0, + 7, + 4, + 1, + 8, + }, + ec_gf8_mul_D1_ops}; + +static ec_gf_op_t ec_gf8_mul_D2_ops[] = { + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D2 = {8, + { + 7, + 0, + 2, + 1, + 3, + 4, + 6, + 5, + }, + ec_gf8_mul_D2_ops}; + +static ec_gf_op_t ec_gf8_mul_D3_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 8, 4, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D3 = {9, + { + 0, + 3, + 2, + 8, + 4, + 6, + 7, + 1, + 5, + }, + ec_gf8_mul_D3_ops}; + +static ec_gf_op_t ec_gf8_mul_D4_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 1, 7, 8}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D4 = {9, + { + 4, + 1, + 7, + 5, + 0, + 6, + 3, + 2, + 8, + }, + ec_gf8_mul_D4_ops}; + +static ec_gf_op_t ec_gf8_mul_D5_ops[] = { + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D5 = {8, + { + 6, + 7, + 4, + 5, + 2, + 3, + 1, + 0, + }, + ec_gf8_mul_D5_ops}; + +static ec_gf_op_t ec_gf8_mul_D6_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D6 = {9, + { + 0, + 6, + 2, + 7, + 1, + 3, + 4, + 5, + 8, + }, + ec_gf8_mul_D6_ops}; + +static ec_gf_op_t ec_gf8_mul_D7_ops[] = { + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR3, 8, 3, 5}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR3, 6, 7, 8}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D7 = {9, + { + 3, + 4, + 6, + 5, + 0, + 7, + 1, + 2, + 8, + }, + ec_gf8_mul_D7_ops}; + +static ec_gf_op_t ec_gf8_mul_D8_ops[] = { + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D8 = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + }, + ec_gf8_mul_D8_ops}; + +static ec_gf_op_t ec_gf8_mul_D9_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_D9 = {8, + { + 1, + 2, + 6, + 7, + 4, + 5, + 0, + 3, + }, + ec_gf8_mul_D9_ops}; + +static ec_gf_op_t ec_gf8_mul_DA_ops[] = { + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR3, 8, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 4, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DA = {9, + { + 2, + 5, + 7, + 1, + 0, + 4, + 3, + 6, + 8, + }, + ec_gf8_mul_DA_ops}; + +static ec_gf_op_t ec_gf8_mul_DB_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 8, 4, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 8, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DB = {9, + { + 7, + 5, + 6, + 2, + 3, + 4, + 1, + 0, + 8, + }, + ec_gf8_mul_DB_ops}; + +static ec_gf_op_t ec_gf8_mul_DC_ops[] = { + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DC = {8, + { + 4, + 5, + 2, + 6, + 7, + 1, + 0, + 3, + }, + ec_gf8_mul_DC_ops}; + +static ec_gf_op_t ec_gf8_mul_DD_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DD = {8, + { + 1, + 2, + 3, + 6, + 7, + 0, + 4, + 5, + }, + ec_gf8_mul_DD_ops}; + +static ec_gf_op_t ec_gf8_mul_DE_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DE = {8, + { + 0, + 5, + 2, + 6, + 7, + 1, + 3, + 4, + }, + ec_gf8_mul_DE_ops}; + +static ec_gf_op_t ec_gf8_mul_DF_ops[] = { + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 8, 3, 0}, + {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR3, 1, 9, 2}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_DF = {10, + { + 7, + 2, + 8, + 4, + 3, + 1, + 0, + 6, + 5, + 9, + }, + ec_gf8_mul_DF_ops}; + +static ec_gf_op_t ec_gf8_mul_E0_ops[] = { + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0}, + {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E0 = {8, + { + 2, + 3, + 4, + 7, + 5, + 6, + 0, + 1, + }, + ec_gf8_mul_E0_ops}; + +static ec_gf_op_t ec_gf8_mul_E1_ops[] = { + {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 7, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR3, 9, 5, 3}, + {EC_GF_OP_XOR2, 0, 9, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_XOR2, 0, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E1 = {10, + { + 0, + 7, + 1, + 3, + 4, + 5, + 6, + 2, + 8, + 9, + }, + ec_gf8_mul_E1_ops}; + +static ec_gf_op_t ec_gf8_mul_E2_ops[] = { + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E2 = {8, + { + 2, + 3, + 7, + 1, + 5, + 6, + 0, + 4, + }, + ec_gf8_mul_E2_ops}; + +static ec_gf_op_t ec_gf8_mul_E3_ops[] = { + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0}, + {EC_GF_OP_XOR3, 8, 2, 7}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 4}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E3 = {9, + { + 5, + 4, + 7, + 2, + 1, + 3, + 6, + 0, + 8, + }, + ec_gf8_mul_E3_ops}; + +static ec_gf_op_t ec_gf8_mul_E4_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0}, + {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E4 = {8, + { + 7, + 0, + 1, + 6, + 3, + 4, + 2, + 5, + }, + ec_gf8_mul_E4_ops}; + +static ec_gf_op_t ec_gf8_mul_E5_ops[] = { + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E5 = {9, + { + 4, + 5, + 3, + 6, + 7, + 1, + 0, + 2, + 8, + }, + ec_gf8_mul_E5_ops}; + +static ec_gf_op_t ec_gf8_mul_E6_ops[] = { + {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E6 = {8, + { + 5, + 4, + 3, + 6, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_E6_ops}; + +static ec_gf_op_t ec_gf8_mul_E7_ops[] = { + {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR3, 9, 0, 6}, {EC_GF_OP_XOR2, 4, 9, 0}, + {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E7 = {10, + { + 1, + 4, + 3, + 6, + 7, + 5, + 2, + 0, + 8, + 9, + }, + ec_gf8_mul_E7_ops}; + +static ec_gf_op_t ec_gf8_mul_E8_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E8 = {8, + { + 1, + 4, + 2, + 7, + 3, + 0, + 5, + 6, + }, + ec_gf8_mul_E8_ops}; + +static ec_gf_op_t ec_gf8_mul_E9_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0}, + {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_E9 = {9, + { + 6, + 2, + 0, + 3, + 4, + 1, + 5, + 7, + 8, + }, + ec_gf8_mul_E9_ops}; + +static ec_gf_op_t ec_gf8_mul_EA_ops[] = { + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EA = {8, + { + 3, + 4, + 5, + 6, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_EA_ops}; + +static ec_gf_op_t ec_gf8_mul_EB_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EB = {8, + { + 3, + 4, + 5, + 6, + 7, + 0, + 1, + 2, + }, + ec_gf8_mul_EB_ops}; + +static ec_gf_op_t ec_gf8_mul_EC_ops[] = { + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EC = {9, + { + 7, + 4, + 3, + 0, + 2, + 5, + 1, + 6, + 8, + }, + ec_gf8_mul_EC_ops}; + +static ec_gf_op_t ec_gf8_mul_ED_ops[] = { + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0}, + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_ED = {8, + { + 5, + 6, + 7, + 0, + 1, + 4, + 3, + 2, + }, + ec_gf8_mul_ED_ops}; + +static ec_gf_op_t ec_gf8_mul_EE_ops[] = { + {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 2, 3}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 8, 5, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EE = {9, + { + 6, + 4, + 5, + 7, + 2, + 3, + 0, + 1, + 8, + }, + ec_gf8_mul_EE_ops}; + +static ec_gf_op_t ec_gf8_mul_EF_ops[] = { + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_COPY, 8, 0, 0}, + {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_EF = {9, + { + 6, + 4, + 5, + 7, + 2, + 0, + 3, + 1, + 8, + }, + ec_gf8_mul_EF_ops}; + +static ec_gf_op_t ec_gf8_mul_F0_ops[] = { + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 8, 3, 6}, + {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 8, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 8, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 8, 0}, + {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F0 = {9, + { + 3, + 4, + 6, + 1, + 2, + 0, + 5, + 7, + 8, + }, + ec_gf8_mul_F0_ops}; + +static ec_gf_op_t ec_gf8_mul_F1_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_COPY, 9, 2, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 9, 0, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0}, + {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 4, 9, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR3, 9, 8, 7}, + {EC_GF_OP_XOR2, 1, 9, 0}, {EC_GF_OP_XOR2, 5, 9, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F1 = {10, + { + 7, + 2, + 6, + 3, + 5, + 1, + 4, + 0, + 8, + 9, + }, + ec_gf8_mul_F1_ops}; + +static ec_gf_op_t ec_gf8_mul_F2_ops[] = { + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0}, + {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0}, + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 4}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F2 = {9, + { + 1, + 0, + 6, + 7, + 4, + 5, + 2, + 3, + 8, + }, + ec_gf8_mul_F2_ops}; + +static ec_gf_op_t ec_gf8_mul_F3_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F3 = {8, + { + 5, + 6, + 7, + 0, + 1, + 2, + 3, + 4, + }, + ec_gf8_mul_F3_ops}; + +static ec_gf_op_t ec_gf8_mul_F4_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 3, 7, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F4 = {8, + { + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + }, + ec_gf8_mul_F4_ops}; + +static ec_gf_op_t ec_gf8_mul_F5_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F5 = {8, + { + 7, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + }, + ec_gf8_mul_F5_ops}; + +static ec_gf_op_t ec_gf8_mul_F6_ops[] = { + {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0}, + {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 9, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0}, + {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR3, 7, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F6 = {10, + { + 0, + 6, + 2, + 7, + 4, + 3, + 5, + 9, + 1, + 8, + }, + ec_gf8_mul_F6_ops}; + +static ec_gf_op_t ec_gf8_mul_F7_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0}, + {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F7 = {8, + { + 6, + 7, + 0, + 1, + 2, + 3, + 4, + 5, + }, + ec_gf8_mul_F7_ops}; + +static ec_gf_op_t ec_gf8_mul_F8_ops[] = { + {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 5, 0}, + {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 1, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0}, + {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F8 = {8, + { + 6, + 2, + 0, + 1, + 4, + 5, + 3, + 7, + }, + ec_gf8_mul_F8_ops}; + +static ec_gf_op_t ec_gf8_mul_F9_ops[] = { + {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR3, 8, 7, 1}, {EC_GF_OP_XOR2, 1, 3, 0}, + {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_F9 = {9, + { + 4, + 1, + 7, + 6, + 0, + 3, + 5, + 2, + 8, + }, + ec_gf8_mul_F9_ops}; + +static ec_gf_op_t ec_gf8_mul_FA_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0}, + {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0}, + {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0}, + {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FA = {8, + { + 0, + 1, + 2, + 4, + 5, + 6, + 7, + 3, + }, + ec_gf8_mul_FA_ops}; + +static ec_gf_op_t ec_gf8_mul_FB_ops[] = { + {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0}, + {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0}, + {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0}, + {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0}, + {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FB = {8, + { + 4, + 5, + 6, + 7, + 0, + 1, + 2, + 3, + }, + ec_gf8_mul_FB_ops}; + +static ec_gf_op_t ec_gf8_mul_FC_ops[] = { + {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 7, 4, 0}, + {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_COPY, 9, 3, 0}, + {EC_GF_OP_XOR3, 8, 5, 7}, {EC_GF_OP_XOR2, 3, 6, 0}, + {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 8, 0}, + {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0}, + {EC_GF_OP_XOR3, 0, 9, 2}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FC = {10, + { + 5, + 6, + 3, + 7, + 1, + 8, + 0, + 4, + 2, + 9, + }, + ec_gf8_mul_FC_ops}; + +static ec_gf_op_t ec_gf8_mul_FD_ops[] = { + {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_COPY, 8, 7, 0}, + {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0}, + {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0}, + {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0}, + {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0}, + {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FD = {9, + { + 5, + 3, + 7, + 6, + 1, + 2, + 4, + 0, + 8, + }, + ec_gf8_mul_FD_ops}; + +static ec_gf_op_t ec_gf8_mul_FE_ops[] = { + {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0}, + {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0}, + {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0}, + {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0}, + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FE = {9, + { + 3, + 4, + 8, + 2, + 5, + 0, + 6, + 1, + 7, + }, + ec_gf8_mul_FE_ops}; + +static ec_gf_op_t ec_gf8_mul_FF_ops[] = { + {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 9, 0, 0}, + {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 9, 1, 0}, + {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 9, 4, 0}, + {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0}, + {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 7, 3, 0}, + {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0}, + {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0}, + {EC_GF_OP_XOR3, 3, 8, 5}, {EC_GF_OP_XOR2, 4, 6, 0}, + {EC_GF_OP_END, 0, 0, 0}}; + +static ec_gf_mul_t ec_gf8_mul_FF = {10, + { + 6, + 5, + 0, + 1, + 2, + 4, + 9, + 3, + 7, + 8, + }, + ec_gf8_mul_FF_ops}; + +ec_gf_mul_t *ec_gf8_mul[] = { + &ec_gf8_mul_00, &ec_gf8_mul_01, &ec_gf8_mul_02, &ec_gf8_mul_03, + &ec_gf8_mul_04, &ec_gf8_mul_05, &ec_gf8_mul_06, &ec_gf8_mul_07, + &ec_gf8_mul_08, &ec_gf8_mul_09, &ec_gf8_mul_0A, &ec_gf8_mul_0B, + &ec_gf8_mul_0C, &ec_gf8_mul_0D, &ec_gf8_mul_0E, &ec_gf8_mul_0F, + &ec_gf8_mul_10, &ec_gf8_mul_11, &ec_gf8_mul_12, &ec_gf8_mul_13, + &ec_gf8_mul_14, &ec_gf8_mul_15, &ec_gf8_mul_16, &ec_gf8_mul_17, + &ec_gf8_mul_18, &ec_gf8_mul_19, &ec_gf8_mul_1A, &ec_gf8_mul_1B, + &ec_gf8_mul_1C, &ec_gf8_mul_1D, &ec_gf8_mul_1E, &ec_gf8_mul_1F, + &ec_gf8_mul_20, &ec_gf8_mul_21, &ec_gf8_mul_22, &ec_gf8_mul_23, + &ec_gf8_mul_24, &ec_gf8_mul_25, &ec_gf8_mul_26, &ec_gf8_mul_27, + &ec_gf8_mul_28, &ec_gf8_mul_29, &ec_gf8_mul_2A, &ec_gf8_mul_2B, + &ec_gf8_mul_2C, &ec_gf8_mul_2D, &ec_gf8_mul_2E, &ec_gf8_mul_2F, + &ec_gf8_mul_30, &ec_gf8_mul_31, &ec_gf8_mul_32, &ec_gf8_mul_33, + &ec_gf8_mul_34, &ec_gf8_mul_35, &ec_gf8_mul_36, &ec_gf8_mul_37, + &ec_gf8_mul_38, &ec_gf8_mul_39, &ec_gf8_mul_3A, &ec_gf8_mul_3B, + &ec_gf8_mul_3C, &ec_gf8_mul_3D, &ec_gf8_mul_3E, &ec_gf8_mul_3F, + &ec_gf8_mul_40, &ec_gf8_mul_41, &ec_gf8_mul_42, &ec_gf8_mul_43, + &ec_gf8_mul_44, &ec_gf8_mul_45, &ec_gf8_mul_46, &ec_gf8_mul_47, + &ec_gf8_mul_48, &ec_gf8_mul_49, &ec_gf8_mul_4A, &ec_gf8_mul_4B, + &ec_gf8_mul_4C, &ec_gf8_mul_4D, &ec_gf8_mul_4E, &ec_gf8_mul_4F, + &ec_gf8_mul_50, &ec_gf8_mul_51, &ec_gf8_mul_52, &ec_gf8_mul_53, + &ec_gf8_mul_54, &ec_gf8_mul_55, &ec_gf8_mul_56, &ec_gf8_mul_57, + &ec_gf8_mul_58, &ec_gf8_mul_59, &ec_gf8_mul_5A, &ec_gf8_mul_5B, + &ec_gf8_mul_5C, &ec_gf8_mul_5D, &ec_gf8_mul_5E, &ec_gf8_mul_5F, + &ec_gf8_mul_60, &ec_gf8_mul_61, &ec_gf8_mul_62, &ec_gf8_mul_63, + &ec_gf8_mul_64, &ec_gf8_mul_65, &ec_gf8_mul_66, &ec_gf8_mul_67, + &ec_gf8_mul_68, &ec_gf8_mul_69, &ec_gf8_mul_6A, &ec_gf8_mul_6B, + &ec_gf8_mul_6C, &ec_gf8_mul_6D, &ec_gf8_mul_6E, &ec_gf8_mul_6F, + &ec_gf8_mul_70, &ec_gf8_mul_71, &ec_gf8_mul_72, &ec_gf8_mul_73, + &ec_gf8_mul_74, &ec_gf8_mul_75, &ec_gf8_mul_76, &ec_gf8_mul_77, + &ec_gf8_mul_78, &ec_gf8_mul_79, &ec_gf8_mul_7A, &ec_gf8_mul_7B, + &ec_gf8_mul_7C, &ec_gf8_mul_7D, &ec_gf8_mul_7E, &ec_gf8_mul_7F, + &ec_gf8_mul_80, &ec_gf8_mul_81, &ec_gf8_mul_82, &ec_gf8_mul_83, + &ec_gf8_mul_84, &ec_gf8_mul_85, &ec_gf8_mul_86, &ec_gf8_mul_87, + &ec_gf8_mul_88, &ec_gf8_mul_89, &ec_gf8_mul_8A, &ec_gf8_mul_8B, + &ec_gf8_mul_8C, &ec_gf8_mul_8D, &ec_gf8_mul_8E, &ec_gf8_mul_8F, + &ec_gf8_mul_90, &ec_gf8_mul_91, &ec_gf8_mul_92, &ec_gf8_mul_93, + &ec_gf8_mul_94, &ec_gf8_mul_95, &ec_gf8_mul_96, &ec_gf8_mul_97, + &ec_gf8_mul_98, &ec_gf8_mul_99, &ec_gf8_mul_9A, &ec_gf8_mul_9B, + &ec_gf8_mul_9C, &ec_gf8_mul_9D, &ec_gf8_mul_9E, &ec_gf8_mul_9F, + &ec_gf8_mul_A0, &ec_gf8_mul_A1, &ec_gf8_mul_A2, &ec_gf8_mul_A3, + &ec_gf8_mul_A4, &ec_gf8_mul_A5, &ec_gf8_mul_A6, &ec_gf8_mul_A7, + &ec_gf8_mul_A8, &ec_gf8_mul_A9, &ec_gf8_mul_AA, &ec_gf8_mul_AB, + &ec_gf8_mul_AC, &ec_gf8_mul_AD, &ec_gf8_mul_AE, &ec_gf8_mul_AF, + &ec_gf8_mul_B0, &ec_gf8_mul_B1, &ec_gf8_mul_B2, &ec_gf8_mul_B3, + &ec_gf8_mul_B4, &ec_gf8_mul_B5, &ec_gf8_mul_B6, &ec_gf8_mul_B7, + &ec_gf8_mul_B8, &ec_gf8_mul_B9, &ec_gf8_mul_BA, &ec_gf8_mul_BB, + &ec_gf8_mul_BC, &ec_gf8_mul_BD, &ec_gf8_mul_BE, &ec_gf8_mul_BF, + &ec_gf8_mul_C0, &ec_gf8_mul_C1, &ec_gf8_mul_C2, &ec_gf8_mul_C3, + &ec_gf8_mul_C4, &ec_gf8_mul_C5, &ec_gf8_mul_C6, &ec_gf8_mul_C7, + &ec_gf8_mul_C8, &ec_gf8_mul_C9, &ec_gf8_mul_CA, &ec_gf8_mul_CB, + &ec_gf8_mul_CC, &ec_gf8_mul_CD, &ec_gf8_mul_CE, &ec_gf8_mul_CF, + &ec_gf8_mul_D0, &ec_gf8_mul_D1, &ec_gf8_mul_D2, &ec_gf8_mul_D3, + &ec_gf8_mul_D4, &ec_gf8_mul_D5, &ec_gf8_mul_D6, &ec_gf8_mul_D7, + &ec_gf8_mul_D8, &ec_gf8_mul_D9, &ec_gf8_mul_DA, &ec_gf8_mul_DB, + &ec_gf8_mul_DC, &ec_gf8_mul_DD, &ec_gf8_mul_DE, &ec_gf8_mul_DF, + &ec_gf8_mul_E0, &ec_gf8_mul_E1, &ec_gf8_mul_E2, &ec_gf8_mul_E3, + &ec_gf8_mul_E4, &ec_gf8_mul_E5, &ec_gf8_mul_E6, &ec_gf8_mul_E7, + &ec_gf8_mul_E8, &ec_gf8_mul_E9, &ec_gf8_mul_EA, &ec_gf8_mul_EB, + &ec_gf8_mul_EC, &ec_gf8_mul_ED, &ec_gf8_mul_EE, &ec_gf8_mul_EF, + &ec_gf8_mul_F0, &ec_gf8_mul_F1, &ec_gf8_mul_F2, &ec_gf8_mul_F3, + &ec_gf8_mul_F4, &ec_gf8_mul_F5, &ec_gf8_mul_F6, &ec_gf8_mul_F7, + &ec_gf8_mul_F8, &ec_gf8_mul_F9, &ec_gf8_mul_FA, &ec_gf8_mul_FB, + &ec_gf8_mul_FC, &ec_gf8_mul_FD, &ec_gf8_mul_FE, &ec_gf8_mul_FF}; diff --git a/xlators/cluster/ec/src/ec-gf8.h b/xlators/cluster/ec/src/ec-gf8.h new file mode 100644 index 00000000000..4aca91127fc --- /dev/null +++ b/xlators/cluster/ec/src/ec-gf8.h @@ -0,0 +1,18 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_GF8_H__ +#define __EC_GF8_H__ + +#include "ec-galois.h" + +extern ec_gf_mul_t *ec_gf8_mul[]; + +#endif /* __EC_GF8_H__ */ diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c new file mode 100644 index 00000000000..7d991f04aac --- /dev/null +++ b/xlators/cluster/ec/src/ec-heal.c @@ -0,0 +1,3367 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <glusterfs/defaults.h> +#include <glusterfs/compat-errno.h> +#include <glusterfs/byte-order.h> +#include <glusterfs/syncop.h> +#include <glusterfs/syncop-utils.h> +#include <glusterfs/cluster-syncop.h> + +#include "ec.h" +#include "ec-types.h" +#include "ec-messages.h" +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-combine.h" +#include "ec-method.h" +#include "ec-fops.h" +#include "ec-heald.h" + +#define EC_COUNT(array, max) \ + ({ \ + int __i; \ + int __res = 0; \ + for (__i = 0; __i < max; __i++) \ + if (array[__i]) \ + __res++; \ + __res; \ + }) +#define EC_INTERSECT(dst, src1, src2, max) \ + ({ \ + int __i; \ + for (__i = 0; __i < max; __i++) \ + dst[__i] = src1[__i] && src2[__i]; \ + }) +#define EC_ADJUST_SOURCE(source, sources, max) \ + ({ \ + int __i; \ + if (sources[source] == 0) { \ + source = -1; \ + for (__i = 0; __i < max; __i++) \ + if (sources[__i]) \ + source = __i; \ + } \ + }) +#define IA_EQUAL(f, s, field) \ + (memcmp(&(f.ia_##field), &(s.ia_##field), sizeof(s.ia_##field)) == 0) +#define EC_REPLIES_ALLOC(replies, numsubvols) \ + do { \ + int __i = 0; \ + replies = alloca0(numsubvols * sizeof(*replies)); \ + for (__i = 0; __i < numsubvols; __i++) \ + INIT_LIST_HEAD(&replies[__i].entries.list); \ + } while (0) + +struct ec_name_data { + call_frame_t *frame; + unsigned char *participants; + unsigned char *failed_on; + unsigned char *gfidless; + unsigned char *enoent; + unsigned char *same; + char *name; + inode_t *parent; + default_args_cbk_t *replies; + uint32_t heal_pending; +}; + +static char *ec_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL}; + +static gf_boolean_t +ec_ignorable_key_match(dict_t *dict, char *key, data_t *val, void *mdata) +{ + int i = 0; + + if (!key) + goto out; + + if (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0) + return _gf_true; + + for (i = 0; ec_ignore_xattrs[i]; i++) { + if (!strcmp(key, ec_ignore_xattrs[i])) + return _gf_true; + } + +out: + return _gf_false; +} + +static gf_boolean_t +ec_sh_key_match(dict_t *dict, char *key, data_t *val, void *mdata) +{ + return !ec_ignorable_key_match(dict, key, val, mdata); +} +/* FOP: heal */ + +void +ec_set_entry_healing(ec_fop_data_t *fop) +{ + ec_inode_t *ctx = NULL; + loc_t *loc = NULL; + + if (!fop) + return; + + loc = &fop->loc[0]; + LOCK(&loc->inode->lock); + { + ctx = __ec_inode_get(loc->inode, fop->xl); + if (ctx) { + ctx->heal_count += 1; + } + } + UNLOCK(&loc->inode->lock); +} + +void +ec_reset_entry_healing(ec_fop_data_t *fop) +{ + ec_inode_t *ctx = NULL; + loc_t *loc = NULL; + int32_t heal_count = 0; + if (!fop) + return; + + loc = &fop->loc[0]; + LOCK(&loc->inode->lock); + { + ctx = __ec_inode_get(loc->inode, fop->xl); + if (ctx) { + ctx->heal_count += -1; + heal_count = ctx->heal_count; + } + } + UNLOCK(&loc->inode->lock); + GF_ASSERT(heal_count >= 0); +} + +uintptr_t +ec_heal_check(ec_fop_data_t *fop, uintptr_t *pgood) +{ + ec_cbk_data_t *cbk; + uintptr_t mask[2] = {0, 0}; + + list_for_each_entry(cbk, &fop->cbk_list, list) + { + mask[cbk->op_ret >= 0] |= cbk->mask; + } + + if (pgood != NULL) { + *pgood = mask[1]; + } + + return mask[0]; +} + +void +ec_heal_update(ec_fop_data_t *fop, int32_t is_open) +{ + ec_heal_t *heal = fop->data; + uintptr_t good, bad; + + bad = ec_heal_check(fop, &good); + + LOCK(&heal->lock); + + heal->bad &= ~bad; + if (is_open) { + heal->open |= good; + } + + UNLOCK(&heal->lock); + + fop->error = 0; +} + +void +ec_heal_avoid(ec_fop_data_t *fop) +{ + ec_heal_t *heal = fop->data; + uintptr_t bad; + + bad = ec_heal_check(fop, NULL); + + LOCK(&heal->lock); + + heal->good &= ~bad; + + UNLOCK(&heal->lock); +} + +int32_t +ec_heal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_heal_t *heal = fop->data; + + if (op_ret >= 0) { + GF_ASSERT( + ec_set_inode_size(heal->fop, heal->fd->inode, heal->total_size)); + } + + return 0; +} + +void +ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc, off_t offset, + size_t size) +{ + struct gf_flock flock; + fop_inodelk_cbk_t cbk = NULL; + + flock.l_type = type; + flock.l_whence = SEEK_SET; + flock.l_start = offset; + flock.l_len = size; + flock.l_pid = 0; + flock.l_owner.len = 0; + + if (type == F_UNLCK) { + /* Remove inode size information before unlocking it. */ + if (fd == NULL) { + ec_clear_inode_info(heal->fop, heal->loc.inode); + } else { + ec_clear_inode_info(heal->fop, heal->fd->inode); + } + cbk = ec_lock_unlocked; + } else { + /* Otherwise use the callback to update size information. */ + cbk = ec_heal_lock_cbk; + } + + if (fd != NULL) { + ec_finodelk(heal->fop->frame, heal->xl, + &heal->fop->frame->root->lk_owner, heal->fop->mask, + EC_MINIMUM_ALL, cbk, heal, heal->xl->name, fd, F_SETLKW, + &flock, NULL); + } else { + ec_inodelk(heal->fop->frame, heal->xl, + &heal->fop->frame->root->lk_owner, heal->fop->mask, + EC_MINIMUM_ALL, cbk, heal, heal->xl->name, loc, F_SETLKW, + &flock, NULL); + } +} + +void +ec_heal_inodelk(ec_heal_t *heal, int32_t type, int32_t use_fd, off_t offset, + size_t size) +{ + ec_heal_lock(heal, type, use_fd ? heal->fd : NULL, &heal->loc, offset, + size); +} + +int32_t +ec_heal_xattr_clean(dict_t *dict, char *key, data_t *data, void *arg) +{ + dict_t *base = arg; + + if (ec_ignorable_key_match(NULL, key, NULL, NULL)) { + dict_del(dict, key); + return 0; + } + + if (dict_get(base, key) != NULL) + dict_del(dict, key); + + return 0; +} + +/******************************************************************** + * ec_wind_xattrop_parallel: + * Helper function to update the extended attributes + * in parallel. + * + *******************************************************************/ +void +ec_wind_xattrop_parallel(call_frame_t *frame, xlator_t *subvol, int child_index, + loc_t *loc, gf_xattrop_flags_t flags, dict_t **dict, + dict_t *xdata) +{ + gf_msg_debug("EC", 0, "WIND: on child %d ", child_index); + STACK_WIND_COOKIE( + frame, cluster_xattrop_cbk, (void *)(uintptr_t)child_index, subvol, + subvol->fops->xattrop, loc, flags, dict[child_index], xdata); +} + +int32_t +ec_heal_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_heal_t *heal = fop->data; + + ec_trace("WRITE_CBK", cookie, "ret=%d, errno=%d", op_ret, op_errno); + + gf_msg_debug(fop->xl->name, 0, + "%s: write op_ret %d, op_errno %s" + " at %" PRIu64, + uuid_utoa(heal->fd->inode->gfid), op_ret, strerror(op_errno), + heal->offset); + + ec_heal_update(cookie, 0); + + return 0; +} + +int32_t +ec_heal_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iovec *vector, + int32_t count, struct iatt *stbuf, struct iobref *iobref, + dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_heal_t *heal = fop->data; + + ec_trace("READ_CBK", fop, "ret=%d, errno=%d", op_ret, op_errno); + + ec_heal_avoid(fop); + + if (op_ret > 0) { + gf_msg_debug(fop->xl->name, 0, + "%s: read succeeded, proceeding " + "to write at %" PRIu64, + uuid_utoa(heal->fd->inode->gfid), heal->offset); + ec_writev(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE, + ec_heal_writev_cbk, heal, heal->fd, vector, count, + heal->offset, 0, iobref, NULL); + } else { + if (op_ret < 0) { + gf_msg_debug(fop->xl->name, 0, + "%s: read failed %s, failing " + "to heal block at %" PRIu64, + uuid_utoa(heal->fd->inode->gfid), strerror(op_errno), + heal->offset); + heal->bad = 0; + } + heal->done = 1; + } + + return 0; +} + +void +ec_heal_data_block(ec_heal_t *heal) +{ + ec_trace("DATA", heal->fop, "good=%lX, bad=%lX", heal->good, heal->bad); + + if ((heal->good != 0) && (heal->bad != 0) && + (heal->iatt.ia_type == IA_IFREG)) { + ec_readv(heal->fop->frame, heal->xl, heal->good, EC_MINIMUM_MIN, + ec_heal_readv_cbk, heal, heal->fd, heal->size, heal->offset, 0, + NULL); + } +} + +/* FOP: fheal */ + +void +ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd, + int32_t partial, dict_t *xdata) +{ + ec_fd_t *ctx = ec_fd_get(fd, this); + + if (ctx != NULL) { + gf_msg_trace("ec", 0, "FHEAL ctx: flags=%X, open=%" PRIXPTR, ctx->flags, + ctx->open); + ec_heal(frame, this, target, fop_flags, func, data, &ctx->loc, partial, + xdata); + } +} + +/* Common heal code */ +void +ec_mask_to_char_array(uintptr_t mask, unsigned char *array, int numsubvols) +{ + int i = 0; + + for (i = 0; i < numsubvols; i++) + array[i] = ((mask >> i) & 1); +} + +uintptr_t +ec_char_array_to_mask(unsigned char *array, int numsubvols) +{ + int i = 0; + uintptr_t mask = 0; + + if (array == NULL) + goto out; + + for (i = 0; i < numsubvols; i++) + if (array[i]) + mask |= (1ULL << i); +out: + return mask; +} + +int +ec_heal_entry_find_direction(ec_t *ec, default_args_cbk_t *replies, + uint64_t *versions, uint64_t *dirty, + unsigned char *sources, + unsigned char *healed_sinks) +{ + uint64_t xattr[EC_VERSION_SIZE] = {0}; + int source = -1; + uint64_t max_version = 0; + int ret = 0; + int i = 0; + + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; + + if (replies[i].op_ret == -1) + continue; + + if (source == -1) + source = i; + + ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr, + EC_VERSION_SIZE); + if (ret == 0) { + versions[i] = xattr[EC_DATA_TXN]; + if (max_version < versions[i]) { + max_version = versions[i]; + source = i; + } + } + + memset(xattr, 0, sizeof(xattr)); + ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr, + EC_VERSION_SIZE); + if (ret == 0) { + dirty[i] = xattr[EC_DATA_TXN]; + } + } + + if (source < 0) + goto out; + + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; + + if (replies[i].op_ret == -1) + continue; + + if (versions[i] == versions[source]) + sources[i] = 1; + else + healed_sinks[i] = 1; + } + +out: + return source; +} + +int +ec_adjust_versions(call_frame_t *frame, ec_t *ec, ec_txn_t type, inode_t *inode, + int source, unsigned char *sources, + unsigned char *healed_sinks, uint64_t *versions, + uint64_t *dirty) +{ + int i = 0; + int ret = 0; + int call_count = 0; + dict_t **xattr = NULL; + int op_ret = 0; + loc_t loc = {0}; + gf_boolean_t erase_dirty = _gf_false; + uint64_t *versions_xattr = NULL; + uint64_t *dirty_xattr = NULL; + uint64_t allzero[2] = {0}; + unsigned char *on = NULL; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + + /* Allocate the required memory */ + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + EC_REPLIES_ALLOC(replies, ec->nodes); + xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer); + if (!xattr) { + op_ret = -ENOMEM; + goto out; + } + for (i = 0; i < ec->nodes; i++) { + xattr[i] = dict_new(); + if (!xattr[i]) { + op_ret = -ENOMEM; + goto out; + } + } + + /* dirty xattr represents if the file/dir needs heal. Unless all the + * copies are healed, don't erase it */ + if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) == + ec->nodes) + erase_dirty = _gf_true; + else + op_ret = -ENOTCONN; + + /* Populate the xattr array */ + for (i = 0; i < ec->nodes; i++) { + if (!sources[i] && !healed_sinks[i]) + continue; + versions_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*versions_xattr), + gf_common_mt_pointer); + if (!versions_xattr) { + op_ret = -ENOMEM; + continue; + } + + versions_xattr[type] = hton64(versions[source] - versions[i]); + ret = dict_set_bin(xattr[i], EC_XATTR_VERSION, versions_xattr, + (sizeof(*versions_xattr) * EC_VERSION_SIZE)); + if (ret < 0) { + op_ret = -ENOMEM; + continue; + } + + if (erase_dirty) { + dirty_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*dirty_xattr), + gf_common_mt_pointer); + if (!dirty_xattr) { + op_ret = -ENOMEM; + continue; + } + + dirty_xattr[type] = hton64(-dirty[i]); + ret = dict_set_bin(xattr[i], EC_XATTR_DIRTY, dirty_xattr, + (sizeof(*dirty_xattr) * EC_VERSION_SIZE)); + if (ret < 0) { + op_ret = -ENOMEM; + continue; + } + } + + if (memcmp(versions_xattr, allzero, + (sizeof(*versions_xattr) * EC_VERSION_SIZE)) == 0) { + if (!erase_dirty) { + continue; + } + + if (memcmp(dirty_xattr, allzero, + (sizeof(*dirty_xattr) * EC_VERSION_SIZE)) == 0) { + continue; + } + } + + on[i] = 1; + call_count++; + } + + /* Update the bricks with xattr */ + if (call_count) { + PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame, + ec_wind_xattrop_parallel, &loc, + GF_XATTROP_ADD_ARRAY64, xattr, NULL); + ret = cluster_fop_success_fill(replies, ec->nodes, output); + } + + if (ret < call_count) { + op_ret = -ENOTCONN; + goto out; + } + +out: + /* Cleanup */ + if (xattr) { + for (i = 0; i < ec->nodes; i++) { + if (xattr[i]) + dict_unref(xattr[i]); + } + GF_FREE(xattr); + } + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return op_ret; +} + +int +ec_heal_metadata_find_direction(ec_t *ec, default_args_cbk_t *replies, + uint64_t *versions, uint64_t *dirty, + unsigned char *sources, + unsigned char *healed_sinks) +{ + uint64_t xattr[EC_VERSION_SIZE] = {0}; + uint64_t max_version = 0; + int same_count = 0; + int max_same_count = 0; + int same_source = -1; + int ret = 0; + int i = 0; + int j = 0; + int *groups = NULL; + struct iatt source_ia = {0}; + struct iatt child_ia = {0}; + + groups = alloca0(ec->nodes * sizeof(*groups)); + for (i = 0; i < ec->nodes; i++) + groups[i] = -1; + + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; + if (replies[i].op_ret < 0) + continue; + ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr, + EC_VERSION_SIZE); + if (ret == 0) { + versions[i] = xattr[EC_METADATA_TXN]; + } + + memset(xattr, 0, sizeof(xattr)); + ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr, + EC_VERSION_SIZE); + if (ret == 0) { + dirty[i] = xattr[EC_METADATA_TXN]; + } + if (groups[i] >= 0) /*Already part of group*/ + continue; + groups[i] = i; + same_count = 1; + source_ia = replies[i].stat; + for (j = i + 1; j < ec->nodes; j++) { + if (!replies[j].valid || replies[j].op_ret < 0) + continue; + child_ia = replies[j].stat; + if (!IA_EQUAL(source_ia, child_ia, gfid) || + !IA_EQUAL(source_ia, child_ia, type) || + !IA_EQUAL(source_ia, child_ia, prot) || + !IA_EQUAL(source_ia, child_ia, uid) || + !IA_EQUAL(source_ia, child_ia, gid)) + continue; + if (!are_dicts_equal(replies[i].xdata, replies[j].xdata, + ec_sh_key_match, NULL)) + continue; + groups[j] = i; + same_count++; + } + + if (max_same_count < same_count) { + max_same_count = same_count; + same_source = i; + } + } + + if (max_same_count < ec->fragments) { + ret = -EIO; + goto out; + } + + for (i = 0; i < ec->nodes; i++) { + if (groups[i] == groups[same_source]) + sources[i] = 1; + else if (replies[i].valid && replies[i].op_ret >= 0) + healed_sinks[i] = 1; + } + for (i = 0; i < ec->nodes; i++) { + if (sources[i] && (versions[i] > max_version)) { + same_source = i; + max_version = versions[i]; + } + } + ret = same_source; +out: + return ret; +} + +int +__ec_heal_metadata_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, + default_args_cbk_t *replies, uint64_t *versions, + uint64_t *dirty, unsigned char *sources, + unsigned char *healed_sinks) +{ + loc_t loc = {0}; + unsigned char *output = NULL; + unsigned char *lookup_on = NULL; + int ret = 0; + int source = 0; + default_args_cbk_t *greplies = NULL; + int i = 0; + EC_REPLIES_ALLOC(greplies, ec->nodes); + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + output = alloca0(ec->nodes); + lookup_on = alloca0(ec->nodes); + ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output, + frame, ec->xl, &loc, NULL); + if (ret <= ec->fragments) { + ret = -ENOTCONN; + goto out; + } + + memcpy(lookup_on, output, ec->nodes); + /*Use getxattr to get the filtered xattrs which filter internal xattrs*/ + ret = cluster_getxattr(ec->xl_list, lookup_on, ec->nodes, greplies, output, + frame, ec->xl, &loc, NULL, NULL); + for (i = 0; i < ec->nodes; i++) { + if (lookup_on[i] && !output[i]) { + replies[i].valid = 0; + continue; + } + if (replies[i].xdata) { + dict_unref(replies[i].xdata); + replies[i].xdata = NULL; + if (greplies[i].xattr) + replies[i].xdata = dict_ref(greplies[i].xattr); + } + } + + source = ec_heal_metadata_find_direction(ec, replies, versions, dirty, + sources, healed_sinks); + if (source < 0) { + ret = -EIO; + goto out; + } + ret = source; +out: + cluster_replies_wipe(greplies, ec->nodes); + loc_wipe(&loc); + return ret; +} + +/* Metadata heal */ +int +__ec_removexattr_sinks(call_frame_t *frame, ec_t *ec, inode_t *inode, + int source, unsigned char *sources, + unsigned char *healed_sinks, default_args_cbk_t *replies) +{ + int i = 0; + int ret = 0; + loc_t loc = {0}; + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + + for (i = 0; i < ec->nodes; i++) { + if (i == source) + continue; + if (!sources[i] && !healed_sinks[i]) + continue; + ret = dict_foreach(replies[i].xdata, ec_heal_xattr_clean, + replies[source].xdata); + if (ret < 0) { + sources[i] = 0; + healed_sinks[i] = 0; + continue; + } + + if (replies[i].xdata->count == 0) { + continue; + } else if (sources[i]) { + /* This can happen if setxattr/removexattr succeeds on + * the bricks but fails to update the version. This + * will make sure that the xattrs are made equal after + * heal*/ + sources[i] = 0; + healed_sinks[i] = 1; + } + + ret = syncop_removexattr(ec->xl_list[i], &loc, "", replies[i].xdata, + NULL); + if (ret < 0) + healed_sinks[i] = 0; + } + + loc_wipe(&loc); + if (EC_COUNT(healed_sinks, ec->nodes) == 0) + return -ENOTCONN; + return 0; +} + +int +__ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, unsigned char *sources, + unsigned char *healed_sinks) +{ + loc_t loc = {0}; + int ret = 0; + int source = 0; + default_args_cbk_t *replies = NULL; + default_args_cbk_t *sreplies = NULL; + uint64_t *versions = NULL; + uint64_t *dirty = NULL; + unsigned char *output = NULL; + dict_t *source_dict = NULL; + struct iatt source_buf = {0}; + + EC_REPLIES_ALLOC(replies, ec->nodes); + EC_REPLIES_ALLOC(sreplies, ec->nodes); + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + output = alloca0(ec->nodes); + versions = alloca0(ec->nodes * sizeof(*versions)); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + source = __ec_heal_metadata_prepare(frame, ec, inode, locked_on, replies, + versions, dirty, sources, healed_sinks); + if (source < 0) { + ret = -EIO; + goto out; + } + + if ((EC_COUNT(sources, ec->nodes) == ec->nodes) || + (EC_COUNT(healed_sinks, ec->nodes) == 0)) { + ret = 0; + goto erase_dirty; + } + + source_buf = replies[source].stat; + ret = cluster_setattr(ec->xl_list, healed_sinks, ec->nodes, sreplies, + output, frame, ec->xl, &loc, &source_buf, + GF_SET_ATTR_MODE | GF_SET_ATTR_UID | GF_SET_ATTR_GID, + NULL); + /*In case the operation fails on some of the subvols*/ + memcpy(healed_sinks, output, ec->nodes); + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; + } + + ret = __ec_removexattr_sinks(frame, ec, inode, source, sources, + healed_sinks, replies); + if (ret < 0) + goto out; + + source_dict = dict_ref(replies[source].xdata); + if (dict_foreach_match(source_dict, ec_ignorable_key_match, NULL, + dict_remove_foreach_fn, NULL) == -1) { + ret = -ENOMEM; + goto out; + } + + ret = cluster_setxattr(ec->xl_list, healed_sinks, ec->nodes, replies, + output, frame, ec->xl, &loc, source_dict, 0, NULL); + + EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes); + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; + } + +erase_dirty: + ret = ec_adjust_versions(frame, ec, EC_METADATA_TXN, inode, source, sources, + healed_sinks, versions, dirty); +out: + if (source_dict) + dict_unref(source_dict); + + loc_wipe(&loc); + cluster_replies_wipe(replies, ec->nodes); + cluster_replies_wipe(sreplies, ec->nodes); + return ret; +} + +int +ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *sources, unsigned char *healed_sinks) +{ + unsigned char *locked_on = NULL; + unsigned char *up_subvols = NULL; + unsigned char *output = NULL; + int ret = 0; + default_args_cbk_t *replies = NULL; + + EC_REPLIES_ALLOC(replies, ec->nodes); + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + up_subvols = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies, + locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0); + { + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + ret = __ec_heal_metadata(frame, ec, inode, locked_on, sources, + healed_sinks); + } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, inode, 0, 0); + cluster_replies_wipe(replies, ec->nodes); + return ret; +} + +/*entry heal*/ +int +__ec_heal_entry_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, uint64_t *versions, + uint64_t *dirty, unsigned char *sources, + unsigned char *healed_sinks) +{ + loc_t loc = {0}; + int source = 0; + int ret = 0; + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + dict_t *xdata = NULL; + + EC_REPLIES_ALLOC(replies, ec->nodes); + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + xdata = dict_new(); + if (!xdata) { + ret = -ENOMEM; + goto out; + } + + if (dict_set_uint64(xdata, EC_XATTR_VERSION, 0) || + dict_set_uint64(xdata, EC_XATTR_DIRTY, 0)) { + ret = -ENOMEM; + goto out; + } + + output = alloca0(ec->nodes); + ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output, + frame, ec->xl, &loc, xdata); + if (ret <= ec->fragments) { + ret = -ENOTCONN; + goto out; + } + + source = ec_heal_entry_find_direction(ec, replies, versions, dirty, sources, + healed_sinks); + if (source < 0) { + ret = -EIO; + goto out; + } + ret = source; +out: + if (xdata) + dict_unref(xdata); + loc_wipe(&loc); + cluster_replies_wipe(replies, ec->nodes); + return ret; +} +int32_t +ec_set_new_entry_dirty(ec_t *ec, loc_t *loc, struct iatt *ia, + call_frame_t *frame, xlator_t *this, unsigned char *on) +{ + dict_t *xattr = NULL; + int32_t ret = -1; + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + uint64_t dirty[EC_VERSION_SIZE] = {1, 1}; + loc_t newloc = {0}; + + /*Symlinks don't have any data to be healed*/ + if (ia->ia_type == IA_IFLNK) + dirty[EC_DATA_TXN] = 0; + + newloc.inode = inode_ref(loc->inode); + gf_uuid_copy(newloc.gfid, ia->ia_gfid); + EC_REPLIES_ALLOC(replies, ec->nodes); + output = alloca0(ec->nodes); + xattr = dict_new(); + if (!xattr) { + ret = -ENOMEM; + goto out; + } + + ret = ec_dict_set_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE); + if (ret) + goto out; + + ret = cluster_xattrop(ec->xl_list, on, ec->nodes, replies, output, frame, + ec->xl, &newloc, GF_XATTROP_ADD_ARRAY64, xattr, NULL); + + if (ret < ec->fragments) { + ret = -ENOTCONN; + goto out; + } + +out: + if (xattr) + dict_unref(xattr); + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&newloc); + return ret; +} + +/*Name heal*/ +int +ec_delete_stale_name(dict_t *gfid_db, char *key, data_t *d, void *data) +{ + struct ec_name_data *name_data = data; + struct iatt *ia = NULL; + ec_t *ec = NULL; + loc_t loc = {0}; + unsigned char *same = data_to_bin(d); + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + int ret = 0; + int estale_count = 0; + int i = 0; + call_frame_t *frame = name_data->frame; + uuid_t gfid; + + ec = name_data->frame->this->private; + EC_REPLIES_ALLOC(replies, ec->nodes); + if (EC_COUNT(same, ec->nodes) >= ec->fragments) { + ret = 0; + goto out; + } + + loc.parent = inode_ref(name_data->parent); + loc.inode = inode_new(name_data->parent->table); + if (!loc.inode) { + ret = -ENOMEM; + goto out; + } + + gf_uuid_parse(key, gfid); + gf_uuid_copy(loc.pargfid, name_data->parent->gfid); + loc.name = name_data->name; + output = alloca0(ec->nodes); + ret = cluster_lookup(ec->xl_list, name_data->participants, ec->nodes, + replies, output, name_data->frame, ec->xl, &loc, NULL); + + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; + if (replies[i].op_ret == -1) { + if (replies[i].op_errno == ESTALE || replies[i].op_errno == ENOENT) + estale_count++; + else + name_data->participants[i] = 0; + } else if (gf_uuid_compare(gfid, replies[i].stat.ia_gfid)) { + estale_count++; + gf_msg_debug(ec->xl->name, 0, "%s/%s: different gfid as %s", + uuid_utoa(name_data->parent->gfid), name_data->name, + key); + } + } + + if (estale_count <= ec->redundancy) { + /* We have at least ec->fragments number of fragments, so the + * file is recoverable, so don't delete it*/ + + /* Please note that the lookup call above could fail with + * ENOTCONN on all subvoumes and still this branch will be + * true, but in those cases conservatively we decide to not + * delete the file until we are sure*/ + ret = 0; + goto out; + } + + /*Noway to recover, delete the name*/ + loc_wipe(&loc); + loc.parent = inode_ref(name_data->parent); + gf_uuid_copy(loc.pargfid, loc.parent->gfid); + loc.name = name_data->name; + for (i = 0; i < ec->nodes; i++) { + if (same[i] && replies[i].valid && (replies[i].op_ret == 0)) { + ia = &replies[i].stat; + break; + } + } + + if (!ia) { + ret = -ENOTCONN; + goto out; + } + + if (IA_ISDIR(ia->ia_type)) { + ret = cluster_rmdir(ec->xl_list, same, ec->nodes, replies, output, + frame, ec->xl, &loc, 1, NULL); + gf_msg_debug(ec->xl->name, 0, + "cluster rmdir succeeded on %d " + "nodes", + ret); + } else { + ret = cluster_unlink(ec->xl_list, same, ec->nodes, replies, output, + frame, ec->xl, &loc, 0, NULL); + gf_msg_debug(ec->xl->name, 0, + "cluster unlink succeeded on %d " + "nodes", + ret); + } + + for (i = 0; i < ec->nodes; i++) { + if (output[i]) { + same[i] = 0; + name_data->enoent[i] = 1; + } else { + /*op failed*/ + if (same[i]) + name_data->participants[i] = 0; + } + } + ret = 0; + /*This will help in making decisions about creating names*/ + dict_del(gfid_db, key); +out: + if (ret < 0) { + gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s", + uuid_utoa(name_data->parent->gfid), name_data->name, + strerror(-ret)); + } + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return ret; +} + +int +ec_delete_stale_names(call_frame_t *frame, ec_t *ec, inode_t *parent, + char *name, default_args_cbk_t *replies, dict_t *gfid_db, + unsigned char *enoent, unsigned char *gfidless, + unsigned char *participants) +{ + struct ec_name_data name_data = {0}; + + name_data.enoent = enoent; + name_data.gfidless = gfidless; + name_data.participants = participants; + name_data.name = name; + name_data.parent = parent; + name_data.frame = frame; + name_data.replies = replies; + return dict_foreach(gfid_db, ec_delete_stale_name, &name_data); +} + +int +_assign_same(dict_t *dict, char *key, data_t *value, void *data) +{ + struct ec_name_data *name_data = data; + + name_data->same = data_to_bin(value); + return 0; +} + +int +ec_create_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name, + default_args_cbk_t *lookup_replies, dict_t *gfid_db, + unsigned char *enoent, unsigned char *participants) +{ + int ret = 0; + int i = 0; + struct ec_name_data name_data = {0}; + struct iatt *ia = NULL; + unsigned char *output = 0; + unsigned char *output1 = 0; + unsigned char *on = NULL; + default_args_cbk_t *replies = NULL; + loc_t loc = {0}; + loc_t srcloc = {0}; + unsigned char *link = NULL; + unsigned char *create = NULL; + dict_t *xdata = NULL; + char *linkname = NULL; + ec_config_t config; + + /* There should be just one gfid key */ + EC_REPLIES_ALLOC(replies, ec->nodes); + if (gfid_db->count != 1) { + ret = -EINVAL; + goto out; + } + + ret = dict_foreach(gfid_db, _assign_same, &name_data); + if (ret < 0) + goto out; + /*There should at least be one valid success reply with gfid*/ + for (i = 0; i < ec->nodes; i++) + if (name_data.same[i]) + break; + + if (i == ec->nodes) { + ret = -EINVAL; + goto out; + } + + ia = &lookup_replies[i].stat; + xdata = dict_new(); + loc.parent = inode_ref(parent); + gf_uuid_copy(loc.pargfid, parent->gfid); + loc.inode = inode_new(parent->table); + if (loc.inode) + srcloc.inode = inode_ref(loc.inode); + gf_uuid_copy(srcloc.gfid, ia->ia_gfid); + if (!loc.inode || !xdata || + dict_set_static_bin(xdata, "gfid-req", ia->ia_gfid, + sizeof(ia->ia_gfid))) { + ret = -ENOMEM; + goto out; + } + loc.name = name; + link = alloca0(ec->nodes); + create = alloca0(ec->nodes); + on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + output1 = alloca0(ec->nodes); + + for (i = 0; i < ec->nodes; i++) { + if (!lookup_replies[i].valid) + continue; + if (lookup_replies[i].op_ret) + continue; + on[i] = 1; + } + switch (ia->ia_type) { + case IA_IFDIR: + ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on); + (void)cluster_mkdir( + ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl, + &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type), 0, xdata); + break; + + case IA_IFLNK: + /*Check for hard links and create/link*/ + ret = cluster_lookup(ec->xl_list, enoent, ec->nodes, replies, + output, frame, ec->xl, &srcloc, NULL); + for (i = 0; i < ec->nodes; i++) { + if (output[i]) { + link[i] = 1; + } else { + if (replies[i].op_errno == ENOENT || + replies[i].op_errno == ESTALE) { + create[i] = 1; + } + } + } + + if (EC_COUNT(link, ec->nodes)) { + cluster_link(ec->xl_list, link, ec->nodes, replies, output1, + frame, ec->xl, &srcloc, &loc, NULL); + } + + if (EC_COUNT(create, ec->nodes)) { + cluster_readlink(ec->xl_list, name_data.same, ec->nodes, + replies, output, frame, ec->xl, &srcloc, 4096, + NULL); + if (EC_COUNT(output, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; + } + + for (i = 0; i < ec->nodes; i++) { + if (output[i]) + break; + } + linkname = alloca0(strlen(replies[i].buf) + 1); + strcpy(linkname, replies[i].buf); + ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on); + cluster_symlink(ec->xl_list, create, ec->nodes, replies, output, + frame, ec->xl, linkname, &loc, 0, xdata); + } + for (i = 0; i < ec->nodes; i++) + if (output1[i]) + output[i] = 1; + break; + case IA_IFREG: + ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on); + config.version = EC_CONFIG_VERSION; + config.algorithm = EC_CONFIG_ALGORITHM; + config.gf_word_size = EC_GF_BITS; + config.bricks = ec->nodes; + config.redundancy = ec->redundancy; + config.chunk_size = EC_METHOD_CHUNK_SIZE; + + ret = ec_dict_set_config(xdata, EC_XATTR_CONFIG, &config); + if (ret != 0) { + goto out; + } + + /* Fall through */ + + default: + ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1); + if (ret) + goto out; + ret = cluster_mknod( + ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl, + &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type), + makedev(ia_major(ia->ia_rdev), ia_minor(ia->ia_rdev)), 0, + xdata); + break; + } + + for (i = 0; i < ec->nodes; i++) { + if (enoent[i] && !output[i]) + participants[i] = 0; + } + + ret = 0; +out: + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s", + uuid_utoa(parent->gfid), name, strerror(-ret)); + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + loc_wipe(&srcloc); + if (xdata) + dict_unref(xdata); + return ret; +} + +int +__ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name, + unsigned char *participants) +{ + unsigned char *output = NULL; + unsigned char *enoent = NULL; + default_args_cbk_t *replies = NULL; + dict_t *xdata = NULL; + dict_t *gfid_db = NULL; + int ret = 0; + loc_t loc = {0}; + int i = 0; + struct iatt *ia = NULL; + char gfid[64] = {0}; + unsigned char *same = NULL; + unsigned char *gfidless = NULL; + + EC_REPLIES_ALLOC(replies, ec->nodes); + loc.parent = inode_ref(parent); + loc.inode = inode_new(parent->table); + gf_uuid_copy(loc.pargfid, parent->gfid); + loc.name = name; + xdata = dict_new(); + gfid_db = dict_new(); + if (!xdata || !gfid_db || !loc.inode) { + ret = -ENOMEM; + goto out; + } + + ret = dict_set_int32(xdata, GF_GFIDLESS_LOOKUP, 1); + if (ret) { + ret = -ENOMEM; + goto out; + } + + output = alloca0(ec->nodes); + gfidless = alloca0(ec->nodes); + enoent = alloca0(ec->nodes); + ret = cluster_lookup(ec->xl_list, participants, ec->nodes, replies, output, + frame, ec->xl, &loc, NULL); + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; + + if (replies[i].op_ret == -1) { + /*If ESTALE comes here, that means parent dir is not + * present, nothing to do there, so reset participants + * for that brick*/ + if (replies[i].op_errno == ENOENT) + enoent[i] = 1; + else + participants[i] = 0; + continue; + } + ia = &replies[i].stat; + if (gf_uuid_is_null(ia->ia_gfid)) { + if (IA_ISDIR(ia->ia_type) || ia->ia_size == 0) + gfidless[i] = 1; + else + participants[i] = 0; + } else { + uuid_utoa_r(ia->ia_gfid, gfid); + ret = dict_get_bin(gfid_db, gfid, (void **)&same); + if (ret < 0) { + same = alloca0(ec->nodes); + } + same[i] = 1; + if (ret < 0) { + ret = dict_set_static_bin(gfid_db, gfid, same, ec->nodes); + } + if (ret < 0) + goto out; + } + } + + ret = ec_delete_stale_names(frame, ec, parent, name, replies, gfid_db, + enoent, gfidless, participants); + + if (gfid_db->count == 0) { + /* All entries seem to be stale entries and deleted, + * nothing more to do.*/ + goto out; + } + + if (gfid_db->count > 1) { + gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL, + "%s/%s: Not able to heal", uuid_utoa(parent->gfid), name); + memset(participants, 0, ec->nodes); + goto out; + } + + EC_INTERSECT(enoent, enoent, participants, ec->nodes); + if (EC_COUNT(enoent, ec->nodes) == 0) { + ret = 0; + goto out; + } + + ret = ec_create_name(frame, ec, parent, name, replies, gfid_db, enoent, + participants); + if (ret >= 0) { + /* If ec_create_name() succeeded we return 1 to indicate that a new + * file has been created and it will need to be healed. */ + ret = 1; + } +out: + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + if (xdata) + dict_unref(xdata); + if (gfid_db) + dict_unref(gfid_db); + return ret; +} + +int +ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name, + unsigned char *participants) +{ + int ret = 0; + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + unsigned char *locked_on = NULL; + loc_t loc = {0}; + + loc.parent = inode_ref(parent); + loc.name = name; + loc.inode = inode_new(parent->table); + if (!loc.inode) { + ret = -ENOMEM; + goto out; + } + + EC_REPLIES_ALLOC(replies, ec->nodes); + output = alloca0(ec->nodes); + locked_on = alloca0(ec->nodes); + ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies, + locked_on, frame, ec->xl, ec->xl->name, parent, 0, 0); + { + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s/%s: Skipping " + "heal as only %d number of subvolumes could " + "be locked", + uuid_utoa(parent->gfid), name, ret); + ret = -ENOTCONN; + goto unlock; + } + EC_INTERSECT(participants, participants, locked_on, ec->nodes); + ret = __ec_heal_name(frame, ec, parent, name, participants); + } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, parent, 0, 0); +out: + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return ret; +} + +int +ec_name_heal_handler(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent, + void *data) +{ + struct ec_name_data *name_data = data; + xlator_t *this = THIS; + ec_t *ec = this->private; + unsigned char *name_on = alloca0(ec->nodes); + int i = 0; + int ret = 0; + + if (ec->shutdown) { + gf_msg_debug(this->name, 0, + "Cancelling directory heal " + "because EC is stopping."); + return -ENOTCONN; + } + + memcpy(name_on, name_data->participants, ec->nodes); + ret = ec_heal_name(name_data->frame, ec, parent->inode, entry->d_name, + name_on); + + if (ret < 0) { + memset(name_on, 0, ec->nodes); + } else { + name_data->heal_pending += ret; + } + + for (i = 0; i < ec->nodes; i++) + if (name_data->participants[i] && !name_on[i]) + name_data->failed_on[i] = 1; + + return 0; +} + +int +ec_heal_names(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *participants, uint32_t *pending) +{ + int i = 0; + int j = 0; + loc_t loc = {0}; + struct ec_name_data name_data = {0}; + int ret = 0; + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + name_data.frame = frame; + name_data.participants = participants; + name_data.failed_on = alloca0(ec->nodes); + name_data.heal_pending = 0; + + for (i = 0; i < ec->nodes; i++) { + if (!participants[i]) + continue; + ret = syncop_dir_scan(ec->xl_list[i], &loc, GF_CLIENT_PID_SELF_HEALD, + &name_data, ec_name_heal_handler); + if (ret < 0) { + break; + } + for (j = 0; j < ec->nodes; j++) + if (name_data.failed_on[j]) + participants[j] = 0; + + if (EC_COUNT(participants, ec->nodes) <= ec->fragments) { + ret = -ENOTCONN; + break; + } + } + *pending += name_data.heal_pending; + + loc_wipe(&loc); + return ret; +} + +int +__ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *heal_on, unsigned char *sources, + unsigned char *healed_sinks, uint32_t *pending) +{ + unsigned char *locked_on = NULL; + unsigned char *output = NULL; + uint64_t *versions = NULL; + uint64_t *dirty = NULL; + unsigned char *participants = NULL; + default_args_cbk_t *replies = NULL; + int ret = 0; + int source = 0; + int i = 0; + + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + versions = alloca0(ec->nodes * sizeof(*versions)); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + + EC_REPLIES_ALLOC(replies, ec->nodes); + ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on, + frame, ec->xl, ec->xl->name, inode, 0, 0); + { + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + ret = __ec_heal_entry_prepare(frame, ec, inode, locked_on, versions, + dirty, sources, healed_sinks); + source = ret; + } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, inode, 0, 0); + if (ret < 0) + goto out; + + participants = alloca0(ec->nodes); + for (i = 0; i < ec->nodes; i++) { + if (sources[i] || healed_sinks[i]) + participants[i] = 1; + } + ret = ec_heal_names(frame, ec, inode, participants, pending); + + if (EC_COUNT(participants, ec->nodes) <= ec->fragments) + goto out; + + for (i = 0; i < ec->nodes; i++) { + if (!participants[i]) { + sources[i] = 0; + healed_sinks[i] = 0; + } + } + + ec_adjust_versions(frame, ec, EC_DATA_TXN, inode, source, sources, + healed_sinks, versions, dirty); +out: + cluster_replies_wipe(replies, ec->nodes); + return ret; +} + +int +ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *sources, unsigned char *healed_sinks, + uint32_t *pending) +{ + unsigned char *locked_on = NULL; + unsigned char *up_subvols = NULL; + unsigned char *output = NULL; + char selfheal_domain[1024] = {0}; + int ret = 0; + default_args_cbk_t *replies = NULL; + + EC_REPLIES_ALLOC(replies, ec->nodes); + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + up_subvols = alloca0(ec->nodes); + + sprintf(selfheal_domain, "%s:self-heal", ec->xl->name); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + /*If other processes are already doing the heal, don't block*/ + ret = cluster_tiebreaker_inodelk(ec->xl_list, up_subvols, ec->nodes, + replies, locked_on, frame, ec->xl, + selfheal_domain, inode, 0, 0); + { + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + ret = __ec_heal_entry(frame, ec, inode, locked_on, sources, + healed_sinks, pending); + } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, selfheal_domain, inode, 0, 0); + cluster_replies_wipe(replies, ec->nodes); + return ret; +} + +/*Find direction for data heal and heal info*/ +int +ec_heal_data_find_direction(ec_t *ec, default_args_cbk_t *replies, + uint64_t *data_versions, uint64_t *dirty, + uint64_t *size, unsigned char *sources, + unsigned char *healed_sinks, + gf_boolean_t check_ondisksize, int which) +{ + uint64_t xattr[EC_VERSION_SIZE] = {0}; + char version_size[128] = {0}; + dict_t *version_size_db = NULL; + unsigned char *same = NULL; + int max_same_count = 0; + int source = 0; + int i = 0; + int ret = 0; + dict_t *dict = NULL; + uint64_t source_size = 0; + + version_size_db = dict_new(); + if (!version_size_db) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < ec->nodes; i++) { + if (!replies[i].valid) + continue; + if (replies[i].op_ret < 0) + continue; + dict = (which == EC_COMBINE_XDATA) ? replies[i].xdata + : replies[i].xattr; + + ret = ec_dict_get_array(dict, EC_XATTR_VERSION, xattr, EC_VERSION_SIZE); + if (ret == 0) { + data_versions[i] = xattr[EC_DATA_TXN]; + } + + memset(xattr, 0, sizeof(xattr)); + ret = ec_dict_get_array(dict, EC_XATTR_DIRTY, xattr, EC_VERSION_SIZE); + if (ret == 0) { + dirty[i] = xattr[EC_DATA_TXN]; + } + ret = ec_dict_del_number(dict, EC_XATTR_SIZE, &size[i]); + /*Build a db of same metadata and data version and size*/ + snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64, + data_versions[i], size[i]); + + ret = dict_get_bin(version_size_db, version_size, (void **)&same); + if (ret < 0) { + same = alloca0(ec->nodes); + } + + same[i] = 1; + if (max_same_count < EC_COUNT(same, ec->nodes)) { + max_same_count = EC_COUNT(same, ec->nodes); + source = i; + } + + if (ret < 0) { + ret = dict_set_static_bin(version_size_db, version_size, same, + ec->nodes); + } + + if (ret < 0) { + ret = -ENOMEM; + goto out; + } + } + /* If we don't have ec->fragments number of same version,size it is not + * recoverable*/ + if (max_same_count < ec->fragments) { + ret = -EIO; + goto out; + } else { + snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64, + data_versions[source], size[source]); + + ret = dict_get_bin(version_size_db, version_size, (void **)&same); + if (ret < 0) + goto out; + memcpy(sources, same, ec->nodes); + for (i = 0; i < ec->nodes; i++) { + if (replies[i].valid && (replies[i].op_ret == 0) && !sources[i]) + healed_sinks[i] = 1; + } + } + + /* There could be files with versions, size same but on disk ia_size + * could be different because of disk crashes, mark them as sinks as + * well*/ + + if (check_ondisksize) { + source_size = size[source]; + ec_adjust_size_up(ec, &source_size, _gf_true); + + for (i = 0; i < ec->nodes; i++) { + if (sources[i]) { + if (replies[i].stat.ia_size != source_size) { + sources[i] = 0; + healed_sinks[i] = 1; + max_same_count--; + } else { + source = i; + } + } + } + if (max_same_count < ec->fragments) { + ret = -EIO; + goto out; + } + } + + ret = source; +out: + if (version_size_db) + dict_unref(version_size_db); + return ret; +} + +int +__ec_heal_data_prepare(call_frame_t *frame, ec_t *ec, fd_t *fd, + unsigned char *locked_on, uint64_t *versions, + uint64_t *dirty, uint64_t *size, unsigned char *sources, + unsigned char *healed_sinks, unsigned char *trim, + struct iatt *stbuf) +{ + default_args_cbk_t *replies = NULL; + default_args_cbk_t *fstat_replies = NULL; + unsigned char *output = NULL; + unsigned char *fstat_output = NULL; + dict_t *xattrs = NULL; + uint64_t zero_array[2] = {0}; + int source = 0; + int ret = 0; + uint64_t zero_value = 0; + int i = 0; + + EC_REPLIES_ALLOC(replies, ec->nodes); + EC_REPLIES_ALLOC(fstat_replies, ec->nodes); + output = alloca0(ec->nodes); + fstat_output = alloca0(ec->nodes); + xattrs = dict_new(); + if (!xattrs || + dict_set_static_bin(xattrs, EC_XATTR_VERSION, zero_array, + sizeof(zero_array)) || + dict_set_static_bin(xattrs, EC_XATTR_DIRTY, zero_array, + sizeof(zero_array)) || + dict_set_static_bin(xattrs, EC_XATTR_SIZE, &zero_value, + sizeof(zero_value))) { + ret = -ENOMEM; + goto out; + } + + ret = cluster_fxattrop(ec->xl_list, locked_on, ec->nodes, replies, output, + frame, ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs, + NULL); + + ret = cluster_fstat(ec->xl_list, locked_on, ec->nodes, fstat_replies, + fstat_output, frame, ec->xl, fd, NULL); + + for (i = 0; i < ec->nodes; i++) { + output[i] = output[i] && fstat_output[i]; + replies[i].valid = output[i]; + if (output[i]) + replies[i].stat = fstat_replies[i].stat; + } + + if (EC_COUNT(output, ec->nodes) <= ec->fragments) { + ret = -ENOTCONN; + goto out; + } + + source = ec_heal_data_find_direction(ec, replies, versions, dirty, size, + sources, healed_sinks, _gf_true, + EC_COMBINE_DICT); + ret = source; + if (ret < 0) + goto out; + + if (stbuf) + *stbuf = replies[source].stat; + + for (i = 0; i < ec->nodes; i++) { + if (healed_sinks[i]) { + if (replies[i].stat.ia_size) + trim[i] = 1; + } + } + + if (EC_COUNT(sources, ec->nodes) < ec->fragments) { + ret = -ENOTCONN; + goto out; + } + + ret = source; +out: + if (xattrs) + dict_unref(xattrs); + cluster_replies_wipe(replies, ec->nodes); + cluster_replies_wipe(fstat_replies, ec->nodes); + if (ret < 0) { + gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s", + uuid_utoa(fd->inode->gfid), strerror(-ret)); + } else { + gf_msg_debug(ec->xl->name, 0, + "%s: sources: %d, sinks: " + "%d", + uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes), + EC_COUNT(healed_sinks, ec->nodes)); + } + return ret; +} + +int +__ec_heal_mark_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd, + uint64_t *versions, unsigned char *healed_sinks) +{ + int i = 0; + int ret = 0; + unsigned char *mark = NULL; + dict_t *xattrs = NULL; + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + uint64_t versions_xattr[2] = {0}; + + EC_REPLIES_ALLOC(replies, ec->nodes); + xattrs = dict_new(); + if (!xattrs) { + ret = -ENOMEM; + goto out; + } + + mark = alloca0(ec->nodes); + for (i = 0; i < ec->nodes; i++) { + if (!healed_sinks[i]) + continue; + if ((versions[i] >> EC_SELFHEAL_BIT) & 1) + continue; + mark[i] = 1; + } + + if (EC_COUNT(mark, ec->nodes) == 0) + return 0; + + versions_xattr[EC_DATA_TXN] = hton64(1ULL << EC_SELFHEAL_BIT); + if (dict_set_static_bin(xattrs, EC_XATTR_VERSION, versions_xattr, + sizeof(versions_xattr))) { + ret = -ENOMEM; + goto out; + } + + output = alloca0(ec->nodes); + ret = cluster_fxattrop(ec->xl_list, mark, ec->nodes, replies, output, frame, + ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs, NULL); + for (i = 0; i < ec->nodes; i++) { + if (!output[i]) { + if (mark[i]) + healed_sinks[i] = 0; + continue; + } + versions[i] |= (1ULL << EC_SELFHEAL_BIT); + } + + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; + } + ret = 0; + +out: + cluster_replies_wipe(replies, ec->nodes); + if (xattrs) + dict_unref(xattrs); + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s", + uuid_utoa(fd->inode->gfid), strerror(-ret)); + return ret; +} + +int32_t +ec_manager_heal_block(ec_fop_data_t *fop, int32_t state) +{ + ec_heal_t *heal = fop->data; + heal->fop = fop; + + switch (state) { + case EC_STATE_INIT: + ec_owner_set(fop->frame, fop->frame->root); + + ec_heal_inodelk(heal, F_WRLCK, 1, 0, 0); + + return EC_STATE_HEAL_DATA_COPY; + + case EC_STATE_HEAL_DATA_COPY: + gf_msg_debug(fop->xl->name, 0, "%s: read/write starting", + uuid_utoa(heal->fd->inode->gfid)); + ec_heal_data_block(heal); + + return EC_STATE_HEAL_DATA_UNLOCK; + + case -EC_STATE_HEAL_DATA_COPY: + case -EC_STATE_HEAL_DATA_UNLOCK: + case EC_STATE_HEAL_DATA_UNLOCK: + ec_heal_inodelk(heal, F_UNLCK, 1, 0, 0); + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + if (fop->cbks.heal) { + fop->cbks.heal(fop->req_frame, fop->data, fop->xl, 0, 0, + (heal->good | heal->bad), heal->good, heal->bad, + 0, NULL); + } + + return EC_STATE_END; + case -EC_STATE_REPORT: + if (fop->cbks.heal) { + fop->cbks.heal(fop->req_frame, fop->data, fop->xl, -1, + fop->error, 0, 0, 0, 0, NULL); + } + + return EC_STATE_END; + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +/*Takes lock */ +void +ec_heal_block(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_heal_cbk_t func, ec_heal_t *heal) +{ + ec_cbk_t callback = {.heal = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(HEAL) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags, + NULL, ec_manager_heal_block, callback, heal); + if (fop == NULL) + goto out; + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, heal, this, -1, error, 0, 0, 0, 0, NULL); + } +} + +int32_t +ec_heal_block_done(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, uintptr_t mask, + uintptr_t good, uintptr_t bad, uint32_t pending, + dict_t *xdata) +{ + ec_heal_t *heal = cookie; + + if (heal->fop) { + heal->fop->heal = NULL; + } + heal->fop = NULL; + heal->error = op_ret < 0 ? op_errno : 0; + syncbarrier_wake(heal->data); + return 0; +} + +int +ec_sync_heal_block(call_frame_t *frame, xlator_t *this, ec_heal_t *heal) +{ + ec_heal_block(frame, this, heal->bad | heal->good, EC_MINIMUM_ONE, + ec_heal_block_done, heal); + syncbarrier_wait(heal->data, 1); + if (heal->error != 0) { + return -heal->error; + } + if (heal->bad == 0) + return -ENOTCONN; + return 0; +} + +int +ec_rebuild_data(call_frame_t *frame, ec_t *ec, fd_t *fd, uint64_t size, + unsigned char *sources, unsigned char *healed_sinks) +{ + ec_heal_t *heal = NULL; + int ret = 0; + syncbarrier_t barrier; + + if (syncbarrier_init(&barrier)) + return -ENOMEM; + + heal = alloca0(sizeof(*heal)); + heal->fd = fd_ref(fd); + heal->xl = ec->xl; + heal->data = &barrier; + ec_adjust_size_up(ec, &size, _gf_false); + heal->total_size = size; + heal->size = (128 * GF_UNIT_KB * (ec->self_heal_window_size)); + /* We need to adjust the size to a multiple of the stripe size of the + * volume. Otherwise writes would need to fill gaps (head and/or tail) + * with existent data from the bad bricks. This could be garbage on a + * damaged file or it could fail if there aren't enough bricks. */ + heal->size -= heal->size % ec->stripe_size; + heal->bad = ec_char_array_to_mask(healed_sinks, ec->nodes); + heal->good = ec_char_array_to_mask(sources, ec->nodes); + heal->iatt.ia_type = IA_IFREG; + LOCK_INIT(&heal->lock); + + for (heal->offset = 0; (heal->offset < size) && !heal->done; + heal->offset += heal->size) { + /* We immediately abort any heal if a shutdown request has been + * received to avoid delays. The healing of this file will be + * restarted by another SHD or other client that accesses the + * file. */ + if (ec->shutdown) { + gf_msg_debug(ec->xl->name, 0, + "Cancelling heal because " + "EC is stopping."); + ret = -ENOTCONN; + break; + } + + gf_msg_debug(ec->xl->name, 0, + "%s: sources: %d, sinks: " + "%d, offset: %" PRIu64 " bsize: %" PRIu64, + uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes), + EC_COUNT(healed_sinks, ec->nodes), heal->offset, + heal->size); + ret = ec_sync_heal_block(frame, ec->xl, heal); + if (ret < 0) + break; + } + memset(healed_sinks, 0, ec->nodes); + ec_mask_to_char_array(heal->bad, healed_sinks, ec->nodes); + fd_unref(heal->fd); + LOCK_DESTROY(&heal->lock); + syncbarrier_destroy(heal->data); + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s", + uuid_utoa(fd->inode->gfid), strerror(-ret)); + return ret; +} + +int +__ec_heal_trim_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd, + unsigned char *healed_sinks, unsigned char *trim, + uint64_t size) +{ + default_args_cbk_t *replies = NULL; + unsigned char *output = NULL; + int ret = 0; + int i = 0; + off_t trim_offset = 0; + + EC_REPLIES_ALLOC(replies, ec->nodes); + output = alloca0(ec->nodes); + + if (EC_COUNT(trim, ec->nodes) == 0) { + ret = 0; + goto out; + } + trim_offset = size; + ec_adjust_offset_up(ec, &trim_offset, _gf_true); + ret = cluster_ftruncate(ec->xl_list, trim, ec->nodes, replies, output, + frame, ec->xl, fd, trim_offset, NULL); + for (i = 0; i < ec->nodes; i++) { + if (!output[i] && trim[i]) + healed_sinks[i] = 0; + } + + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto out; + } + +out: + cluster_replies_wipe(replies, ec->nodes); + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s", + uuid_utoa(fd->inode->gfid), strerror(-ret)); + return ret; +} + +int +ec_data_undo_pending(call_frame_t *frame, ec_t *ec, fd_t *fd, dict_t *xattr, + uint64_t *versions, uint64_t *dirty, uint64_t *size, + int source, gf_boolean_t erase_dirty, int idx) +{ + uint64_t versions_xattr[2] = {0}; + uint64_t dirty_xattr[2] = {0}; + uint64_t allzero[2] = {0}; + uint64_t size_xattr = 0; + int ret = 0; + + versions_xattr[EC_DATA_TXN] = hton64(versions[source] - versions[idx]); + ret = dict_set_static_bin(xattr, EC_XATTR_VERSION, versions_xattr, + sizeof(versions_xattr)); + if (ret < 0) + goto out; + + size_xattr = hton64(size[source] - size[idx]); + ret = dict_set_static_bin(xattr, EC_XATTR_SIZE, &size_xattr, + sizeof(size_xattr)); + if (ret < 0) + goto out; + + if (erase_dirty) { + dirty_xattr[EC_DATA_TXN] = hton64(-dirty[idx]); + ret = dict_set_static_bin(xattr, EC_XATTR_DIRTY, dirty_xattr, + sizeof(dirty_xattr)); + if (ret < 0) + goto out; + } + + if ((memcmp(versions_xattr, allzero, sizeof(allzero)) == 0) && + (memcmp(dirty_xattr, allzero, sizeof(allzero)) == 0) && + (size_xattr == 0)) { + ret = 0; + goto out; + } + + ret = syncop_fxattrop(ec->xl_list[idx], fd, GF_XATTROP_ADD_ARRAY64, xattr, + NULL, NULL, NULL); +out: + return ret; +} + +int +__ec_fd_data_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd, + unsigned char *sources, + unsigned char *healed_sinks, uint64_t *versions, + uint64_t *dirty, uint64_t *size) +{ + dict_t *xattr = NULL; + int i = 0; + int ret = 0; + int op_ret = 0; + int source = -1; + gf_boolean_t erase_dirty = _gf_false; + + xattr = dict_new(); + if (!xattr) { + op_ret = -ENOMEM; + goto out; + } + + /* dirty xattr represents if the file needs heal. Unless all the + * copies are healed, don't erase it */ + if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) == + ec->nodes) + erase_dirty = _gf_true; + + for (i = 0; i < ec->nodes; i++) { + if (sources[i]) { + source = i; + break; + } + } + + if (source == -1) { + op_ret = -ENOTCONN; + goto out; + } + + for (i = 0; i < ec->nodes; i++) { + if (healed_sinks[i]) { + ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty, + size, source, erase_dirty, i); + if (ret < 0) + goto out; + } + } + + if (!erase_dirty) + goto out; + + for (i = 0; i < ec->nodes; i++) { + if (sources[i]) { + ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty, + size, source, erase_dirty, i); + if (ret < 0) + continue; + } + } +out: + if (xattr) + dict_unref(xattr); + return op_ret; +} + +int +ec_restore_time_and_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd, + unsigned char *sources, + unsigned char *healed_sinks, + uint64_t *versions, uint64_t *dirty, + uint64_t *size) +{ + unsigned char *locked_on = NULL; + unsigned char *participants = NULL; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + unsigned char *postsh_sources = NULL; + unsigned char *postsh_healed_sinks = NULL; + unsigned char *postsh_trim = NULL; + uint64_t *postsh_versions = NULL; + uint64_t *postsh_dirty = NULL; + uint64_t *postsh_size = NULL; + int ret = 0; + int i = 0; + struct iatt source_buf = {0}; + loc_t loc = {0}; + + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + participants = alloca0(ec->nodes); + postsh_sources = alloca0(ec->nodes); + postsh_healed_sinks = alloca0(ec->nodes); + postsh_trim = alloca0(ec->nodes); + postsh_versions = alloca0(ec->nodes * sizeof(*postsh_versions)); + postsh_dirty = alloca0(ec->nodes * sizeof(*postsh_dirty)); + postsh_size = alloca0(ec->nodes * sizeof(*postsh_size)); + + for (i = 0; i < ec->nodes; i++) { + if (healed_sinks[i] || sources[i]) + participants[i] = 1; + } + + EC_REPLIES_ALLOC(replies, ec->nodes); + ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies, + locked_on, frame, ec->xl, ec->xl->name, fd->inode, 0, + 0); + { + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(fd->inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + + ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, postsh_versions, + postsh_dirty, postsh_size, postsh_sources, + postsh_healed_sinks, postsh_trim, + &source_buf); + if (ret < 0) + goto unlock; + + loc.inode = inode_ref(fd->inode); + gf_uuid_copy(loc.gfid, fd->inode->gfid); + ret = cluster_setattr( + ec->xl_list, healed_sinks, ec->nodes, replies, output, frame, + ec->xl, &loc, &source_buf, + GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME, NULL); + EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes); + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = -ENOTCONN; + goto unlock; + } + ret = __ec_fd_data_adjust_versions(frame, ec, fd, sources, healed_sinks, + versions, dirty, size); + } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, fd->inode, 0, 0); + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return ret; +} + +int +__ec_heal_data(call_frame_t *frame, ec_t *ec, fd_t *fd, unsigned char *heal_on, + unsigned char *sources, unsigned char *healed_sinks) +{ + unsigned char *locked_on = NULL; + unsigned char *output = NULL; + uint64_t *versions = NULL; + uint64_t *dirty = NULL; + uint64_t *size = NULL; + unsigned char *trim = NULL; + default_args_cbk_t *replies = NULL; + int ret = 0; + int source = 0; + + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + trim = alloca0(ec->nodes); + versions = alloca0(ec->nodes * sizeof(*versions)); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + size = alloca0(ec->nodes * sizeof(*size)); + + EC_REPLIES_ALLOC(replies, ec->nodes); + ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on, + frame, ec->xl, ec->xl->name, fd->inode, 0, 0); + { + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(fd->inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + + ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, versions, dirty, + size, sources, healed_sinks, trim, NULL); + if (ret < 0) + goto unlock; + + if (EC_COUNT(healed_sinks, ec->nodes) == 0) { + ret = __ec_fd_data_adjust_versions( + frame, ec, fd, sources, healed_sinks, versions, dirty, size); + goto unlock; + } + + source = ret; + ret = __ec_heal_mark_sinks(frame, ec, fd, versions, healed_sinks); + if (ret < 0) + goto unlock; + + ret = __ec_heal_trim_sinks(frame, ec, fd, healed_sinks, trim, + size[source]); + } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, fd->inode, 0, 0); + if (ret < 0) + goto out; + + if (EC_COUNT(healed_sinks, ec->nodes) == 0) + goto out; + + gf_msg_debug(ec->xl->name, 0, + "%s: sources: %d, sinks: " + "%d", + uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes), + EC_COUNT(healed_sinks, ec->nodes)); + + ret = ec_rebuild_data(frame, ec, fd, size[source], sources, healed_sinks); + if (ret < 0) + goto out; + + ret = ec_restore_time_and_adjust_versions( + frame, ec, fd, sources, healed_sinks, versions, dirty, size); +out: + cluster_replies_wipe(replies, ec->nodes); + return ret; +} + +int +ec_heal_data(call_frame_t *frame, ec_t *ec, gf_boolean_t block, inode_t *inode, + unsigned char *sources, unsigned char *healed_sinks) +{ + unsigned char *locked_on = NULL; + unsigned char *up_subvols = NULL; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + fd_t *fd = NULL; + loc_t loc = {0}; + char selfheal_domain[1024] = {0}; + int ret = 0; + + EC_REPLIES_ALLOC(replies, ec->nodes); + + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + up_subvols = alloca0(ec->nodes); + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + + fd = fd_create(inode, 0); + if (!fd) { + ret = -ENOMEM; + goto out; + } + + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + + ret = cluster_open(ec->xl_list, up_subvols, ec->nodes, replies, output, + frame, ec->xl, &loc, O_RDWR | O_LARGEFILE, fd, NULL); + if (ret <= ec->fragments) { + ret = -ENOTCONN; + goto out; + } + + fd_bind(fd); + sprintf(selfheal_domain, "%s:self-heal", ec->xl->name); + /*If other processes are already doing the heal, don't block*/ + if (block) { + ret = cluster_inodelk(ec->xl_list, output, ec->nodes, replies, + locked_on, frame, ec->xl, selfheal_domain, inode, + 0, 0); + } else { + ret = cluster_tiebreaker_inodelk(ec->xl_list, output, ec->nodes, + replies, locked_on, frame, ec->xl, + selfheal_domain, inode, 0, 0); + } + { + if (ret <= ec->fragments) { + gf_msg_debug(ec->xl->name, 0, + "%s: Skipping heal " + "as only %d number of subvolumes could " + "be locked", + uuid_utoa(inode->gfid), ret); + ret = -ENOTCONN; + goto unlock; + } + ret = __ec_heal_data(frame, ec, fd, locked_on, sources, healed_sinks); + } +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, selfheal_domain, inode, 0, 0); +out: + if (fd) + fd_unref(fd); + loc_wipe(&loc); + cluster_replies_wipe(replies, ec->nodes); + return ret; +} + +int +ec_heal_purge_stale_index(call_frame_t *frame, ec_t *ec, inode_t *inode) +{ + int i = 0; + int ret = 0; + dict_t **xattr = NULL; + loc_t loc = {0}; + uint64_t dirty_xattr[EC_VERSION_SIZE] = {0}; + unsigned char *on = NULL; + default_args_cbk_t *replies = NULL; + dict_t *dict = NULL; + + /* Allocate the required memory */ + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + on = alloca0(ec->nodes); + EC_REPLIES_ALLOC(replies, ec->nodes); + xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer); + if (!xattr) { + ret = -ENOMEM; + goto out; + } + dict = dict_new(); + if (!dict) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < ec->nodes; i++) { + xattr[i] = dict; + on[i] = 1; + } + ret = dict_set_static_bin(dict, EC_XATTR_DIRTY, dirty_xattr, + (sizeof(*dirty_xattr) * EC_VERSION_SIZE)); + if (ret < 0) { + ret = -ENOMEM; + goto out; + } + PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame, + ec_wind_xattrop_parallel, &loc, GF_XATTROP_ADD_ARRAY64, + xattr, NULL); +out: + if (dict) { + dict_unref(dict); + } + if (xattr) { + GF_FREE(xattr); + } + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + return ret; +} + +void +ec_heal_do(xlator_t *this, void *data, loc_t *loc, int32_t partial) +{ + call_frame_t *frame = NULL; + unsigned char *participants = NULL; + unsigned char *msources = NULL; + unsigned char *mhealed_sinks = NULL; + unsigned char *sources = NULL; + unsigned char *healed_sinks = NULL; + ec_t *ec = NULL; + int ret = 0; + int op_ret = 0; + int op_errno = 0; + intptr_t mgood = 0; + intptr_t mbad = 0; + intptr_t good = 0; + intptr_t bad = 0; + uint32_t pending = 0; + ec_fop_data_t *fop = data; + gf_boolean_t blocking = _gf_false; + ec_heal_need_t need_heal = EC_HEAL_NONEED; + unsigned char *up_subvols = NULL; + char up_bricks[32]; + + ec = this->private; + + /* If it is heal request from getxattr, complete the heal and then + * unwind, if it is ec_heal with NULL as frame then no need to block + * the heal as the caller doesn't care about its completion. In case + * of heald whichever gets tiebreaking inodelk will take care of the + * heal, so no need to block*/ + if (fop->req_frame && !ec->shd.iamshd) + blocking = _gf_true; + + frame = create_frame(this, this->ctx->pool); + if (!frame) + goto out; + + ec_owner_set(frame, frame->root); + /*Do heal as root*/ + frame->root->uid = 0; + frame->root->gid = 0; + /*Mark the fops as internal*/ + frame->root->pid = GF_CLIENT_PID_SELF_HEALD; + participants = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, participants, ec->nodes); + + up_subvols = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + + if (loc->name && strlen(loc->name)) { + ret = ec_heal_name(frame, ec, loc->parent, (char *)loc->name, + participants); + if (ret >= 0) { + gf_msg_debug(this->name, 0, + "%s: name heal " + "successful on %" PRIXPTR, + loc->path, + ec_char_array_to_mask(participants, ec->nodes)); + } else { + gf_msg_debug( + this->name, 0, + "%s: name heal " + "failed. ret = %d, subvolumes up = %s", + loc->path, ret, + ec_bin(up_bricks, sizeof(up_bricks), ec->xl_up, ec->nodes)); + } + } + + /* Mount triggers heal only when it detects that it must need heal, shd + * triggers heals periodically which need not be thorough*/ + if (ec->shd.iamshd && (ret <= 0)) { + ec_heal_inspect(frame, ec, loc->inode, up_subvols, _gf_false, _gf_false, + &need_heal); + + if (need_heal == EC_HEAL_PURGE_INDEX) { + gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL, + "Index entry needs to be purged for: %s ", + uuid_utoa(loc->gfid)); + /* We need to send zero-xattrop so that stale index entry could be + * removed. We need not take lock on this entry to do so as + * xattrop on a brick is atomic. */ + ec_heal_purge_stale_index(frame, ec, loc->inode); + goto out; + } else if (need_heal == EC_HEAL_NONEED) { + gf_msg(ec->xl->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL, + "Heal is not required for : %s ", uuid_utoa(loc->gfid)); + goto out; + } + } + + sources = alloca0(ec->nodes); + healed_sinks = alloca0(ec->nodes); + if (IA_ISREG(loc->inode->ia_type)) { + ret = ec_heal_data(frame, ec, blocking, loc->inode, sources, + healed_sinks); + } else if (IA_ISDIR(loc->inode->ia_type) && !partial) { + ret = ec_heal_entry(frame, ec, loc->inode, sources, healed_sinks, + &pending); + } else { + ret = 0; + memcpy(sources, participants, ec->nodes); + memcpy(healed_sinks, participants, ec->nodes); + } + + if (ret == 0) { + good = ec_char_array_to_mask(sources, ec->nodes); + bad = ec_char_array_to_mask(healed_sinks, ec->nodes); + } else { + op_ret = -1; + op_errno = -ret; + } + msources = alloca0(ec->nodes); + mhealed_sinks = alloca0(ec->nodes); + ret = ec_heal_metadata(frame, ec, loc->inode, msources, mhealed_sinks); + if (ret == 0) { + mgood = ec_char_array_to_mask(msources, ec->nodes); + mbad = ec_char_array_to_mask(mhealed_sinks, ec->nodes); + } else { + op_ret = -1; + op_errno = -ret; + } + +out: + ec_reset_entry_healing(fop); + if (fop->cbks.heal) { + fop->cbks.heal(fop->req_frame, fop->data, fop->xl, op_ret, op_errno, + ec_char_array_to_mask(participants, ec->nodes), + mgood & good, mbad & bad, pending, NULL); + } + if (frame) + STACK_DESTROY(frame->root); + + return; +} + +int +ec_synctask_heal_wrap(void *opaque) +{ + ec_fop_data_t *fop = opaque; + ec_heal_do(fop->xl, fop, &fop->loc[0], fop->int32); + return 0; +} + +int +ec_heal_done(int ret, call_frame_t *heal, void *opaque) +{ + if (opaque) + ec_fop_data_release(opaque); + return 0; +} + +ec_fop_data_t * +__ec_dequeue_heals(ec_t *ec) +{ + ec_fop_data_t *fop = NULL; + + if (list_empty(&ec->heal_waiting)) + goto none; + + if ((ec->background_heals > 0) && (ec->healers >= ec->background_heals)) + goto none; + + fop = list_entry(ec->heal_waiting.next, ec_fop_data_t, healer); + ec->heal_waiters--; + list_del_init(&fop->healer); + list_add(&fop->healer, &ec->healing); + ec->healers++; + return fop; +none: + gf_msg_debug(ec->xl->name, 0, "Num healers: %d, Num Waiters: %d", + ec->healers, ec->heal_waiters); + return NULL; +} + +void +ec_heal_fail(ec_t *ec, ec_fop_data_t *fop) +{ + if (fop->cbks.heal) { + fop->cbks.heal(fop->req_frame, fop->data, ec->xl, -1, fop->error, 0, 0, + 0, 0, NULL); + } + ec_fop_data_release(fop); +} + +void +ec_launch_heal(ec_t *ec, ec_fop_data_t *fop) +{ + int ret = 0; + call_frame_t *frame = NULL; + + frame = create_frame(ec->xl, ec->xl->ctx->pool); + if (!frame) { + ret = -1; + goto out; + } + + ec_owner_set(frame, frame->root); + /*Do heal as root*/ + frame->root->uid = 0; + frame->root->gid = 0; + /*Mark the fops as internal*/ + frame->root->pid = GF_CLIENT_PID_SELF_HEALD; + + ret = synctask_new(ec->xl->ctx->env, ec_synctask_heal_wrap, ec_heal_done, + frame, fop); +out: + if (ret < 0) { + ec_fop_set_error(fop, ENOMEM); + ec_heal_fail(ec, fop); + } + + if (frame) + STACK_DESTROY(frame->root); +} + +void +ec_handle_healers_done(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + ec_fop_data_t *heal_fop = NULL; + + if (list_empty(&fop->healer)) + return; + + LOCK(&ec->lock); + + list_del_init(&fop->healer); + + do { + ec->healers--; + heal_fop = __ec_dequeue_heals(ec); + + if ((heal_fop != NULL) && ec->shutdown) { + /* This will prevent ec_handle_healers_done() to be + * called recursively. That would be problematic if + * the queue is too big. */ + list_del_init(&heal_fop->healer); + + UNLOCK(&ec->lock); + + ec_fop_set_error(fop, ENOTCONN); + ec_heal_fail(ec, heal_fop); + + LOCK(&ec->lock); + } + } while ((heal_fop != NULL) && ec->shutdown); + + UNLOCK(&ec->lock); + + if (heal_fop) + ec_launch_heal(ec, heal_fop); +} + +gf_boolean_t +ec_is_entry_healing(ec_fop_data_t *fop) +{ + ec_inode_t *ctx = NULL; + int32_t heal_count = 0; + loc_t *loc = NULL; + + loc = &fop->loc[0]; + + LOCK(&loc->inode->lock); + { + ctx = __ec_inode_get(loc->inode, fop->xl); + if (ctx) { + heal_count = ctx->heal_count; + } + } + UNLOCK(&loc->inode->lock); + GF_ASSERT(heal_count >= 0); + return heal_count; +} + +void +ec_heal_throttle(xlator_t *this, ec_fop_data_t *fop) +{ + gf_boolean_t can_heal = _gf_true; + ec_t *ec = this->private; + ec_fop_data_t *fop_rel = NULL; + + if (fop->req_frame == NULL) { + LOCK(&ec->lock); + { + if ((ec->background_heals > 0) && + (ec->heal_wait_qlen + ec->background_heals) > + (ec->heal_waiters + ec->healers)) { + if (!ec_is_entry_healing(fop)) { + list_add_tail(&fop->healer, &ec->heal_waiting); + ec->heal_waiters++; + ec_set_entry_healing(fop); + } else { + fop_rel = fop; + } + fop = __ec_dequeue_heals(ec); + } else { + can_heal = _gf_false; + } + } + UNLOCK(&ec->lock); + } + + if (can_heal) { + if (fop) { + if (fop->req_frame != NULL) { + ec_set_entry_healing(fop); + } + ec_launch_heal(ec, fop); + } + } else { + gf_msg_debug(this->name, 0, + "Max number of heals are " + "pending, background self-heal rejected"); + ec_fop_set_error(fop, EBUSY); + ec_heal_fail(ec, fop); + } + if (fop_rel) { + ec_heal_done(0, NULL, fop_rel); + } +} + +void +ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc, + int32_t partial, dict_t *xdata) +{ + ec_cbk_t callback = {.heal = func}; + ec_fop_data_t *fop = NULL; + int32_t err = EINVAL; + + gf_msg_trace("ec", 0, "EC(HEAL) %p", frame); + + VALIDATE_OR_GOTO(this, fail); + GF_VALIDATE_OR_GOTO(this->name, this->private, fail); + + if (!loc || !loc->inode || gf_uuid_is_null(loc->inode->gfid)) + goto fail; + + if (frame && frame->local) + goto fail; + fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags, + NULL, NULL, callback, data); + + err = ENOMEM; + + if (fop == NULL) + goto fail; + + fop->int32 = partial; + + if (loc) { + if (loc_copy(&fop->loc[0], loc) != 0) + goto fail; + } + + if (xdata) + fop->xdata = dict_ref(xdata); + + ec_heal_throttle(this, fop); + + return; + +fail: + if (fop) + ec_fop_data_release(fop); + if (func) + func(frame, data, this, -1, err, 0, 0, 0, 0, NULL); +} + +int +ec_replace_heal_done(int ret, call_frame_t *heal, void *opaque) +{ + ec_t *ec = opaque; + gf_boolean_t last_fop = _gf_false; + + if (GF_ATOMIC_DEC(ec->async_fop_count) == 0) { + LOCK(&ec->lock); + { + last_fop = __ec_is_last_fop(ec); + } + UNLOCK(&ec->lock); + } + gf_msg_debug(ec->xl->name, 0, "getxattr on bricks is done ret %d", ret); + + if (last_fop) + ec_pending_fops_completed(ec); + + return 0; +} + +int32_t +ec_replace_heal(ec_t *ec, inode_t *inode) +{ + loc_t loc = {0}; + int ret = 0; + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + ret = syncop_getxattr(ec->xl, &loc, NULL, EC_XATTR_HEAL, NULL, NULL); + if (ret < 0) + gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d", + ret); + + /* Once the root inode has been checked, it might have triggered a + * self-heal on it after a replace brick command or for some other + * reason. It can also happen that the volume already had damaged + * files in the index, even if the heal on the root directory failed. + * In both cases we need to wake all index healers to continue + * healing remaining entries that are marked as dirty. */ + ec_shd_index_healer_wake(ec); + + loc_wipe(&loc); + return ret; +} + +int32_t +ec_replace_brick_heal_wrap(void *opaque) +{ + ec_t *ec = opaque; + inode_table_t *itable = NULL; + int32_t ret = -1; + + if (ec->xl->itable) + itable = ec->xl->itable; + else + goto out; + + if (xlator_is_cleanup_starting(ec->xl)) + goto out; + + ret = ec_replace_heal(ec, itable->root); +out: + return ret; +} + +int32_t +ec_launch_replace_heal(ec_t *ec) +{ + int ret = -1; + + ret = synctask_new(ec->xl->ctx->env, ec_replace_brick_heal_wrap, + ec_replace_heal_done, NULL, ec); + + if (ret < 0) { + gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d", + ret); + ec_replace_heal_done(-1, NULL, ec); + } + + return ret; +} + +int32_t +ec_set_heal_info(dict_t **dict_rsp, char *status) +{ + dict_t *dict = NULL; + int ret = 0; + + dict = dict_new(); + if (!dict) { + ret = -ENOMEM; + goto out; + } + ret = dict_set_str(dict, "heal-info", status); + if (ret) { + gf_msg(THIS->name, GF_LOG_WARNING, -ret, EC_MSG_HEAL_FAIL, + "Failed to set heal-info key to " + "%s", + status); + dict_unref(dict); + dict = NULL; + } + *dict_rsp = dict; +out: + return ret; +} + +static int32_t +_need_heal_calculate(ec_t *ec, uint64_t *dirty, unsigned char *sources, + gf_boolean_t self_locked, int32_t lock_count, + ec_heal_need_t *need_heal, uint64_t *versions) +{ + int i = 0; + int source_count = 0; + + source_count = EC_COUNT(sources, ec->nodes); + if (source_count == ec->nodes) { + *need_heal = EC_HEAL_NONEED; + if (self_locked || lock_count == 0) { + for (i = 0; i < ec->nodes; i++) { + if (dirty[i] || (versions[i] != versions[0])) { + *need_heal = EC_HEAL_MUST; + goto out; + } + } + /* If lock count is 0, all dirty flags are 0 and all the + * versions are macthing then why are we here. It looks + * like something went wrong while removing the index entries + * after completing a successful heal or fop. In this case + * we need to remove this index entry to avoid triggering heal + * in a loop and causing lookups again and again*/ + *need_heal = EC_HEAL_PURGE_INDEX; + } else { + for (i = 0; i < ec->nodes; i++) { + /* Since each lock can only increment the dirty + * count once, if dirty is > 1 it means that + * another operation has left the dirty count + * set and this indicates a problem in the + * inode.*/ + if (dirty[i] > 1) { + *need_heal = EC_HEAL_MUST; + goto out; + } + if (dirty[i] != dirty[0] || (versions[i] != versions[0])) { + *need_heal = EC_HEAL_MAYBE; + } + } + } + } else { + *need_heal = EC_HEAL_MUST; + } + +out: + return source_count; +} + +static int32_t +ec_need_metadata_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies, + int32_t lock_count, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) +{ + uint64_t *dirty = NULL; + unsigned char *sources = NULL; + unsigned char *healed_sinks = NULL; + uint64_t *meta_versions = NULL; + int ret = 0; + + sources = alloca0(ec->nodes); + healed_sinks = alloca0(ec->nodes); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + meta_versions = alloca0(ec->nodes * sizeof(*meta_versions)); + ret = ec_heal_metadata_find_direction(ec, replies, meta_versions, dirty, + sources, healed_sinks); + if (ret < 0 && ret != -EIO) { + goto out; + } + + ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count, + need_heal, meta_versions); +out: + return ret; +} + +static int32_t +ec_need_data_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies, + int32_t lock_count, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) +{ + uint64_t *dirty = NULL; + unsigned char *sources = NULL; + unsigned char *healed_sinks = NULL; + uint64_t *data_versions = NULL; + uint64_t *size = NULL; + int ret = 0; + + sources = alloca0(ec->nodes); + healed_sinks = alloca0(ec->nodes); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + data_versions = alloca0(ec->nodes * sizeof(*data_versions)); + size = alloca0(ec->nodes * sizeof(*size)); + + /* When dd is going on and heal info is called there is a very good + * chance for on disk sizes to mismatch even though nothing is wrong + * we don't need ondisk size check there. But if the file is either + * self-locked or the caller wants a thorough check then make sure to + * perform on disk check also. */ + ret = ec_heal_data_find_direction( + ec, replies, data_versions, dirty, size, sources, healed_sinks, + self_locked || thorough, EC_COMBINE_XDATA); + if (ret < 0 && ret != -EIO) { + goto out; + } + + ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count, + need_heal, data_versions); +out: + return ret; +} + +static int32_t +ec_need_entry_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies, + int32_t lock_count, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) +{ + uint64_t *dirty = NULL; + unsigned char *sources = NULL; + unsigned char *healed_sinks = NULL; + uint64_t *data_versions = NULL; + int ret = 0; + + sources = alloca0(ec->nodes); + healed_sinks = alloca0(ec->nodes); + dirty = alloca0(ec->nodes * sizeof(*dirty)); + data_versions = alloca0(ec->nodes * sizeof(*data_versions)); + + ret = ec_heal_entry_find_direction(ec, replies, data_versions, dirty, + sources, healed_sinks); + if (ret < 0 && ret != -EIO) { + goto out; + } + + ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count, + need_heal, data_versions); +out: + return ret; +} + +static int32_t +ec_need_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies, + int32_t lock_count, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) +{ + int ret = 0; + + ret = ec_need_metadata_heal(ec, inode, replies, lock_count, self_locked, + thorough, need_heal); + if (ret < 0) + goto out; + + if (*need_heal == EC_HEAL_MUST) + goto out; + + if (inode->ia_type == IA_IFREG) { + ret = ec_need_data_heal(ec, inode, replies, lock_count, self_locked, + thorough, need_heal); + } else if (inode->ia_type == IA_IFDIR) { + ret = ec_need_entry_heal(ec, inode, replies, lock_count, self_locked, + thorough, need_heal); + } + +out: + return ret; +} + +int32_t +ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode, + unsigned char *locked_on, gf_boolean_t self_locked, + gf_boolean_t thorough, ec_heal_need_t *need_heal) +{ + loc_t loc = {0}; + int i = 0; + int ret = 0; + dict_t *xdata = NULL; + uint64_t zero_array[2] = {0}; + uint64_t zero_value = 0; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + int32_t lock_count = 0; + + EC_REPLIES_ALLOC(replies, ec->nodes); + output = alloca0(ec->nodes); + + loc.inode = inode_ref(inode); + gf_uuid_copy(loc.gfid, inode->gfid); + + xdata = dict_new(); + if (!xdata || + dict_set_static_bin(xdata, EC_XATTR_VERSION, zero_array, + sizeof(zero_array)) || + dict_set_static_bin(xdata, EC_XATTR_DIRTY, zero_array, + sizeof(zero_array)) || + dict_set_static_bin(xdata, EC_XATTR_SIZE, &zero_value, + sizeof(zero_value))) { + ret = -ENOMEM; + goto out; + } + + if (!self_locked) { + ret = dict_set_str(xdata, GLUSTERFS_INODELK_DOM_COUNT, ec->xl->name); + if (ret) { + ret = -ENOMEM; + goto out; + } + } + + ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output, + frame, ec->xl, &loc, xdata); + + if (ret != ec->nodes) { + ret = ec->nodes; + *need_heal = EC_HEAL_MUST; + goto out; + } + + if (self_locked) + goto need_heal; + + for (i = 0; i < ec->nodes; i++) { + if (!output[i] || !replies[i].xdata) { + continue; + } + if ((dict_get_int32(replies[i].xdata, GLUSTERFS_INODELK_COUNT, + &lock_count) == 0) && + lock_count > 0) { + break; + } + } +need_heal: + ret = ec_need_heal(ec, inode, replies, lock_count, self_locked, thorough, + need_heal); +out: + cluster_replies_wipe(replies, ec->nodes); + loc_wipe(&loc); + if (xdata) { + dict_unref(xdata); + } + return ret; +} + +int32_t +ec_heal_locked_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode, + ec_heal_need_t *need_heal) +{ + unsigned char *locked_on = NULL; + unsigned char *up_subvols = NULL; + unsigned char *output = NULL; + default_args_cbk_t *replies = NULL; + int ret = 0; + + EC_REPLIES_ALLOC(replies, ec->nodes); + locked_on = alloca0(ec->nodes); + output = alloca0(ec->nodes); + up_subvols = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + + ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies, + locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0); + if (ret != ec->nodes) { + *need_heal = EC_HEAL_MUST; + goto unlock; + } + ret = ec_heal_inspect(frame, ec, inode, locked_on, _gf_true, _gf_true, + need_heal); +unlock: + cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame, + ec->xl, ec->xl->name, inode, 0, 0); + cluster_replies_wipe(replies, ec->nodes); + return ret; +} + +int32_t +ec_get_heal_info(xlator_t *this, loc_t *entry_loc, dict_t **dict_rsp) +{ + int ret = -ENOMEM; + ec_heal_need_t need_heal = EC_HEAL_NONEED; + call_frame_t *frame = NULL; + ec_t *ec = NULL; + unsigned char *up_subvols = NULL; + loc_t loc = { + 0, + }; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, entry_loc, out); + + ec = this->private; + up_subvols = alloca0(ec->nodes); + ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes); + + if (EC_COUNT(up_subvols, ec->nodes) != ec->nodes) { + need_heal = EC_HEAL_MUST; + goto set_heal; + } + frame = create_frame(this, this->ctx->pool); + if (!frame) { + goto out; + } + ec_owner_set(frame, frame->root); + frame->root->uid = 0; + frame->root->gid = 0; + frame->root->pid = GF_CLIENT_PID_SELF_HEALD; + + if (loc_copy(&loc, entry_loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + goto out; + } + if (!loc.inode) { + ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL); + if (ret < 0) + goto out; + } + + ret = ec_heal_inspect(frame, ec, loc.inode, up_subvols, _gf_false, + _gf_false, &need_heal); + if (ret == ec->nodes && need_heal != EC_HEAL_MAYBE) { + goto set_heal; + } + need_heal = EC_HEAL_NONEED; + ret = ec_heal_locked_inspect(frame, ec, loc.inode, &need_heal); + if (ret < 0) + goto out; +set_heal: + if (need_heal == EC_HEAL_MUST) { + ret = ec_set_heal_info(dict_rsp, "heal"); + } else { + ret = ec_set_heal_info(dict_rsp, "no-heal"); + } +out: + if (frame) { + STACK_DESTROY(frame->root); + } + loc_wipe(&loc); + return ret; +} diff --git a/xlators/cluster/ec/src/ec-heald.c b/xlators/cluster/ec/src/ec-heald.c new file mode 100644 index 00000000000..5c1586bc9c5 --- /dev/null +++ b/xlators/cluster/ec/src/ec-heald.c @@ -0,0 +1,681 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <glusterfs/defaults.h> +#include <glusterfs/compat-errno.h> +#include "ec.h" +#include "ec-messages.h" +#include "ec-heald.h" +#include "ec-mem-types.h" +#include <glusterfs/syncop.h> +#include <glusterfs/syncop-utils.h> +#include "protocol-common.h" + +#define NTH_INDEX_HEALER(this, n) \ + (&((((ec_t *)this->private))->shd.index_healers[n])) +#define NTH_FULL_HEALER(this, n) \ + (&((((ec_t *)this->private))->shd.full_healers[n])) + +gf_boolean_t +ec_shd_is_subvol_local(xlator_t *this, int subvol) +{ + ec_t *ec = NULL; + gf_boolean_t is_local = _gf_false; + loc_t loc = { + 0, + }; + + ec = this->private; + loc.inode = this->itable->root; + syncop_is_subvol_local(ec->xl_list[subvol], &loc, &is_local); + return is_local; +} + +char * +ec_subvol_name(xlator_t *this, int subvol) +{ + ec_t *ec = NULL; + + ec = this->private; + if (subvol < 0 || subvol > ec->nodes) + return NULL; + + return ec->xl_list[subvol]->name; +} + +int +__ec_shd_healer_wait(struct subvol_healer *healer) +{ + ec_t *ec = NULL; + struct timespec wait_till = { + 0, + }; + int ret = 0; + + ec = healer->this->private; + +disabled_loop: + wait_till.tv_sec = gf_time() + ec->shd.timeout; + + while (!healer->rerun) { + ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till); + if (ret == ETIMEDOUT) + break; + } + + if (ec->shutdown) { + healer->running = _gf_false; + return -1; + } + + ret = healer->rerun; + healer->rerun = 0; + + if (!ec->shd.enabled || !ec->up) + goto disabled_loop; + + return ret; +} + +int +ec_shd_healer_wait(struct subvol_healer *healer) +{ + int ret = 0; + + pthread_mutex_lock(&healer->mutex); + { + ret = __ec_shd_healer_wait(healer); + } + pthread_mutex_unlock(&healer->mutex); + + return ret; +} + +int +ec_shd_index_inode(xlator_t *this, xlator_t *subvol, inode_t **inode) +{ + loc_t rootloc = { + 0, + }; + int ret = 0; + dict_t *xattr = NULL; + void *index_gfid = NULL; + + *inode = NULL; + rootloc.inode = inode_ref(this->itable->root); + gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid); + + ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_GFID, NULL, + NULL); + if (ret < 0) + goto out; + if (!xattr) { + ret = -EINVAL; + goto out; + } + + ret = dict_get_ptr(xattr, GF_XATTROP_INDEX_GFID, &index_gfid); + if (ret) + goto out; + + gf_msg_debug(this->name, 0, "index-dir gfid for %s: %s", subvol->name, + uuid_utoa(index_gfid)); + + ret = syncop_inode_find(this, subvol, index_gfid, inode, NULL, NULL); + +out: + loc_wipe(&rootloc); + + if (xattr) + dict_unref(xattr); + + return ret; +} + +int +ec_shd_index_purge(xlator_t *subvol, inode_t *inode, char *name) +{ + loc_t loc = { + 0, + }; + int ret = 0; + + loc.parent = inode_ref(inode); + loc.name = name; + + ret = syncop_unlink(subvol, &loc, NULL, NULL); + + loc_wipe(&loc); + return ret; +} + +static gf_boolean_t +ec_is_heal_completed(char *status) +{ + char *bad_pos = NULL; + char *zero_pos = NULL; + + if (!status) { + return _gf_false; + } + + /*Logic: + * Status will be of the form Good: <binary>, Bad: <binary> + * If heal completes, if we do strchr for '0' it should be present after + * 'Bad:' i.e. strRchr for ':' + * */ + + zero_pos = strchr(status, '0'); + bad_pos = strrchr(status, ':'); + if (!zero_pos || !bad_pos) { + /*malformed status*/ + return _gf_false; + } + + if (zero_pos > bad_pos) { + return _gf_true; + } + + return _gf_false; +} + +int +ec_shd_selfheal(struct subvol_healer *healer, int child, loc_t *loc, + gf_boolean_t full) +{ + dict_t *xdata = NULL; + dict_t *dict = NULL; + uint32_t count; + int32_t ret; + char *heal_status = NULL; + ec_t *ec = healer->this->private; + + GF_ATOMIC_INC(ec->stats.shd.attempted); + ret = syncop_getxattr(healer->this, loc, &dict, EC_XATTR_HEAL, NULL, + &xdata); + if (ret == 0) { + if (dict && (dict_get_str(dict, EC_XATTR_HEAL, &heal_status) == 0)) { + if (ec_is_heal_completed(heal_status)) { + GF_ATOMIC_INC(ec->stats.shd.completed); + } + } + } + + if (!full && (loc->inode->ia_type == IA_IFDIR)) { + /* If we have just healed a directory, it's possible that + * other index entries have appeared to be healed. */ + if ((xdata != NULL) && + (dict_get_uint32(xdata, EC_XATTR_HEAL_NEW, &count) == 0) && + (count > 0)) { + /* Force a rerun of the index healer. */ + gf_msg_debug(healer->this->name, 0, "%d more entries to heal", + count); + + healer->rerun = _gf_true; + } + } + + if (xdata != NULL) { + dict_unref(xdata); + } + + if (dict) { + dict_unref(dict); + } + + return ret; +} + +int +ec_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent, + void *data) +{ + struct subvol_healer *healer = data; + ec_t *ec = NULL; + loc_t loc = {0}; + int ret = 0; + + ec = healer->this->private; + if (ec->xl_up_count <= ec->fragments) { + return -ENOTCONN; + } + if (!ec->shd.enabled) + return -EBUSY; + + gf_msg_debug(healer->this->name, 0, "got entry: %s", entry->d_name); + + ret = gf_uuid_parse(entry->d_name, loc.gfid); + if (ret) + return 0; + + /* If this fails with ENOENT/ESTALE index is stale */ + ret = syncop_gfid_to_path(healer->this->itable, subvol, loc.gfid, + (char **)&loc.path); + if (ret < 0) + goto out; + + ret = syncop_inode_find(healer->this, healer->this, loc.gfid, &loc.inode, + NULL, NULL); + if (ret < 0) + goto out; + + ec_shd_selfheal(healer, healer->subvol, &loc, _gf_false); +out: + if (ret == -ENOENT || ret == -ESTALE) { + gf_msg(healer->this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL, + "Purging index for gfid %s:", uuid_utoa(loc.gfid)); + ec_shd_index_purge(subvol, parent->inode, entry->d_name); + } + loc_wipe(&loc); + + return 0; +} + +int +ec_shd_index_sweep(struct subvol_healer *healer) +{ + loc_t loc = {0}; + ec_t *ec = NULL; + int ret = 0; + xlator_t *subvol = NULL; + dict_t *xdata = NULL; + + ec = healer->this->private; + subvol = ec->xl_list[healer->subvol]; + + ret = ec_shd_index_inode(healer->this, subvol, &loc.inode); + if (ret < 0) { + gf_msg(healer->this->name, GF_LOG_WARNING, errno, + EC_MSG_INDEX_DIR_GET_FAIL, "unable to get index-dir on %s", + subvol->name); + goto out; + } + + xdata = dict_new(); + if (!xdata || dict_set_int32(xdata, "get-gfid-type", 1)) { + ret = -ENOMEM; + goto out; + } + + _mask_cancellation(); + ret = syncop_mt_dir_scan(NULL, subvol, &loc, GF_CLIENT_PID_SELF_HEALD, + healer, ec_shd_index_heal, xdata, + ec->shd.max_threads, ec->shd.wait_qlength); + _unmask_cancellation(); +out: + if (xdata) + dict_unref(xdata); + loc_wipe(&loc); + + return ret; +} + +int +ec_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent, + void *data) +{ + struct subvol_healer *healer = data; + xlator_t *this = healer->this; + ec_t *ec = NULL; + loc_t loc = {0}; + int ret = 0; + + ec = this->private; + + if (this->cleanup_starting) { + return -ENOTCONN; + } + + if (ec->xl_up_count <= ec->fragments) { + return -ENOTCONN; + } + if (!ec->shd.enabled) + return -EBUSY; + + if (gf_uuid_is_null(entry->d_stat.ia_gfid)) { + /* It's possible that an entry has been removed just after + * being seen in a directory but before getting its stat info. + * In this case we'll receive a NULL gfid here. Since the file + * doesn't exist anymore, we can safely ignore it. */ + return 0; + } + + loc.parent = inode_ref(parent->inode); + loc.name = entry->d_name; + gf_uuid_copy(loc.gfid, entry->d_stat.ia_gfid); + + /* If this fails with ENOENT/ESTALE index is stale */ + ret = syncop_gfid_to_path(this->itable, subvol, loc.gfid, + (char **)&loc.path); + if (ret < 0) + goto out; + + ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL); + if (ret < 0) + goto out; + + ec_shd_selfheal(healer, healer->subvol, &loc, _gf_true); + + ret = 0; + +out: + loc_wipe(&loc); + return ret; +} + +int +ec_shd_full_sweep(struct subvol_healer *healer, inode_t *inode) +{ + ec_t *ec = NULL; + loc_t loc = {0}; + int ret = -1; + + ec = healer->this->private; + loc.inode = inode; + _mask_cancellation(); + ret = syncop_ftw(ec->xl_list[healer->subvol], &loc, + GF_CLIENT_PID_SELF_HEALD, healer, ec_shd_full_heal); + _unmask_cancellation(); + return ret; +} + +void * +ec_shd_index_healer(void *data) +{ + struct subvol_healer *healer = NULL; + xlator_t *this = NULL; + int run = 0; + + healer = data; + THIS = this = healer->this; + ec_t *ec = this->private; + + for (;;) { + run = ec_shd_healer_wait(healer); + if (run == -1) + break; + + if (ec->xl_up_count > ec->fragments) { + gf_msg_debug(this->name, 0, "starting index sweep on subvol %s", + ec_subvol_name(this, healer->subvol)); + ec_shd_index_sweep(healer); + } + gf_msg_debug(this->name, 0, "finished index sweep on subvol %s", + ec_subvol_name(this, healer->subvol)); + } + + return NULL; +} + +void * +ec_shd_full_healer(void *data) +{ + struct subvol_healer *healer = NULL; + xlator_t *this = NULL; + loc_t rootloc = {0}; + + int run = 0; + + healer = data; + THIS = this = healer->this; + ec_t *ec = this->private; + + rootloc.inode = this->itable->root; + for (;;) { + run = ec_shd_healer_wait(healer); + if (run < 0) { + break; + } else if (run == 0) { + continue; + } + + if (ec->xl_up_count > ec->fragments) { + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_START, + "starting full sweep on subvol %s", + ec_subvol_name(this, healer->subvol)); + + ec_shd_selfheal(healer, healer->subvol, &rootloc, _gf_true); + ec_shd_full_sweep(healer, this->itable->root); + } + + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_STOP, + "finished full sweep on subvol %s", + ec_subvol_name(this, healer->subvol)); + } + + return NULL; +} + +int +ec_shd_healer_init(xlator_t *this, struct subvol_healer *healer) +{ + int ret = 0; + + ret = pthread_mutex_init(&healer->mutex, NULL); + if (ret) + goto out; + + ret = pthread_cond_init(&healer->cond, NULL); + if (ret) + goto out; + + healer->this = this; + healer->running = _gf_false; + healer->rerun = _gf_false; +out: + return ret; +} + +int +ec_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer, + void *(threadfn)(void *)) +{ + int ret = 0; + + pthread_mutex_lock(&healer->mutex); + { + if (healer->running) { + pthread_cond_signal(&healer->cond); + } else { + ret = gf_thread_create(&healer->thread, NULL, threadfn, healer, + "ecshd"); + if (ret) + goto unlock; + healer->running = 1; + } + + healer->rerun = 1; + } +unlock: + pthread_mutex_unlock(&healer->mutex); + + return ret; +} + +int +ec_shd_full_healer_spawn(xlator_t *this, int subvol) +{ + if (xlator_is_cleanup_starting(this)) + return -1; + + return ec_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol), + ec_shd_full_healer); +} + +int +ec_shd_index_healer_spawn(xlator_t *this, int subvol) +{ + if (xlator_is_cleanup_starting(this)) + return -1; + + return ec_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol), + ec_shd_index_healer); +} + +void +ec_shd_index_healer_wake(ec_t *ec) +{ + int32_t i; + + for (i = 0; i < ec->nodes; i++) { + if (((ec->xl_up >> i) & 1) != 0) { + ec_shd_index_healer_spawn(ec->xl, i); + } + } +} + +int +ec_selfheal_daemon_init(xlator_t *this) +{ + ec_t *ec = NULL; + ec_self_heald_t *shd = NULL; + int ret = -1; + int i = 0; + + ec = this->private; + shd = &ec->shd; + + shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers), ec->nodes, + ec_mt_subvol_healer_t); + if (!shd->index_healers) + goto out; + + for (i = 0; i < ec->nodes; i++) { + shd->index_healers[i].subvol = i; + ret = ec_shd_healer_init(this, &shd->index_healers[i]); + if (ret) + goto out; + } + + shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), ec->nodes, + ec_mt_subvol_healer_t); + if (!shd->full_healers) + goto out; + + for (i = 0; i < ec->nodes; i++) { + shd->full_healers[i].subvol = i; + ret = ec_shd_healer_init(this, &shd->full_healers[i]); + if (ret) + goto out; + } + + ret = 0; +out: + return ret; +} + +int +ec_heal_op(xlator_t *this, dict_t *output, gf_xl_afr_op_t op, int xl_id) +{ + char key[64] = {0}; + int op_ret = 0; + ec_t *ec = NULL; + int i = 0; + GF_UNUSED int ret = 0; + + ec = this->private; + + op_ret = -1; + for (i = 0; i < ec->nodes; i++) { + snprintf(key, sizeof(key), "%d-%d-status", xl_id, i); + + if (((ec->xl_up >> i) & 1) == 0) { + ret = dict_set_str(output, key, "Brick is not connected"); + } else if (!ec->up) { + ret = dict_set_str(output, key, "Disperse subvolume is not up"); + } else if (!ec_shd_is_subvol_local(this, i)) { + ret = dict_set_str(output, key, "Brick is remote"); + } else { + ret = dict_set_str(output, key, "Started self-heal"); + if (op == GF_SHD_OP_HEAL_FULL) { + ec_shd_full_healer_spawn(this, i); + } else if (op == GF_SHD_OP_HEAL_INDEX) { + ec_shd_index_healer_spawn(this, i); + } + op_ret = 0; + } + } + return op_ret; +} + +int +ec_xl_op(xlator_t *this, dict_t *input, dict_t *output) +{ + gf_xl_afr_op_t op = GF_SHD_OP_INVALID; + int ret = 0; + int xl_id = 0; + + ret = dict_get_int32(input, "xl-op", (int32_t *)&op); + if (ret) + goto out; + + ret = dict_get_int32(input, this->name, &xl_id); + if (ret) + goto out; + + ret = dict_set_int32(output, this->name, xl_id); + if (ret) + goto out; + + switch (op) { + case GF_SHD_OP_HEAL_FULL: + ret = ec_heal_op(this, output, op, xl_id); + break; + + case GF_SHD_OP_HEAL_INDEX: + ret = ec_heal_op(this, output, op, xl_id); + break; + + default: + ret = -1; + break; + } +out: + dict_del(output, this->name); + return ret; +} + +void +ec_destroy_healer_object(xlator_t *this, struct subvol_healer *healer) +{ + if (!healer) + return; + + pthread_cond_destroy(&healer->cond); + pthread_mutex_destroy(&healer->mutex); +} + +void +ec_selfheal_daemon_fini(xlator_t *this) +{ + struct subvol_healer *healer = NULL; + ec_self_heald_t *shd = NULL; + ec_t *priv = NULL; + int i = 0; + + priv = this->private; + if (!priv) + return; + + shd = &priv->shd; + if (!shd->iamshd) + return; + + for (i = 0; i < priv->nodes; i++) { + healer = &shd->index_healers[i]; + ec_destroy_healer_object(this, healer); + + healer = &shd->full_healers[i]; + ec_destroy_healer_object(this, healer); + } + + GF_FREE(shd->index_healers); + GF_FREE(shd->full_healers); +} diff --git a/xlators/cluster/ec/src/ec-heald.h b/xlators/cluster/ec/src/ec-heald.h new file mode 100644 index 00000000000..6c7da4edc10 --- /dev/null +++ b/xlators/cluster/ec/src/ec-heald.h @@ -0,0 +1,30 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_HEALD_H__ +#define __EC_HEALD_H__ + +#include "ec-types.h" // for ec_t +#include "glusterfs/dict.h" // for dict_t +#include "glusterfs/globals.h" // for xlator_t + +int +ec_xl_op(xlator_t *this, dict_t *input, dict_t *output); + +int +ec_selfheal_daemon_init(xlator_t *this); + +void +ec_shd_index_healer_wake(ec_t *ec); + +void +ec_selfheal_daemon_fini(xlator_t *this); + +#endif /* __EC_HEALD_H__ */ diff --git a/xlators/cluster/ec/src/ec-helpers.c b/xlators/cluster/ec/src/ec-helpers.c new file mode 100644 index 00000000000..48f54475e01 --- /dev/null +++ b/xlators/cluster/ec/src/ec-helpers.c @@ -0,0 +1,867 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <libgen.h> + +#include <glusterfs/byte-order.h> + +#include "ec.h" +#include "ec-mem-types.h" +#include "ec-messages.h" +#include "ec-fops.h" +#include "ec-method.h" +#include "ec-helpers.h" + +static const char *ec_fop_list[] = {[-EC_FOP_HEAL] = "HEAL"}; + +const char * +ec_bin(char *str, size_t size, uint64_t value, int32_t digits) +{ + str += size; + + if (size-- < 1) { + goto failed; + } + *--str = 0; + + while ((value != 0) || (digits > 0)) { + if (size-- < 1) { + goto failed; + } + *--str = '0' + (value & 1); + digits--; + value >>= 1; + } + + return str; + +failed: + return "<buffer too small>"; +} + +const char * +ec_fop_name(int32_t id) +{ + if (id >= 0) { + return gf_fop_list[id]; + } + + return ec_fop_list[-id]; +} + +void +ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...) +{ + char str1[32], str2[32], str3[32]; + char *msg; + ec_t *ec = fop->xl->private; + va_list args; + int32_t ret; + + va_start(args, fmt); + ret = vasprintf(&msg, fmt, args); + va_end(args); + + if (ret < 0) { + msg = "<memory allocation error>"; + } + + gf_msg_trace("ec", 0, + "%s(%s) %p(%p) [refs=%d, winds=%d, jobs=%d] " + "frame=%p/%p, min/exp=%d/%d, err=%d state=%d " + "{%s:%s:%s} %s", + event, ec_fop_name(fop->id), fop, fop->parent, fop->refs, + fop->winds, fop->jobs, fop->req_frame, fop->frame, + fop->minimum, fop->expected, fop->error, fop->state, + ec_bin(str1, sizeof(str1), fop->mask, ec->nodes), + ec_bin(str2, sizeof(str2), fop->remaining, ec->nodes), + ec_bin(str3, sizeof(str3), fop->good, ec->nodes), msg); + + if (ret >= 0) { + free(msg); + } +} + +int32_t +ec_bits_consume(uint64_t *n) +{ + uint64_t tmp; + + tmp = *n; + tmp &= -tmp; + *n ^= tmp; + + return gf_bits_index(tmp); +} + +size_t +ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset, + size_t size) +{ + int32_t i = 0; + size_t total = 0, len = 0; + + while (i < count) { + if (offset < vector[i].iov_len) { + while ((i < count) && (size > 0)) { + len = size; + if (len > vector[i].iov_len - offset) { + len = vector[i].iov_len - offset; + } + memcpy(dst, vector[i++].iov_base + offset, len); + offset = 0; + dst += len; + total += len; + size -= len; + } + + break; + } + + offset -= vector[i].iov_len; + i++; + } + + return total; +} + +int32_t +ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr) +{ + struct iobref *iobref = NULL; + struct iobuf *iobuf = NULL; + int32_t ret = -ENOMEM; + + iobuf = iobuf_get_page_aligned(xl->ctx->iobuf_pool, size, + EC_METHOD_WORD_SIZE); + if (iobuf == NULL) { + goto out; + } + + iobref = *piobref; + if (iobref == NULL) { + iobref = iobref_new(); + if (iobref == NULL) { + goto out; + } + } + + ret = iobref_add(iobref, iobuf); + if (ret != 0) { + if (iobref != *piobref) { + iobref_unref(iobref); + } + iobref = NULL; + + goto out; + } + + GF_ASSERT(EC_ALIGN_CHECK(iobuf->ptr, EC_METHOD_WORD_SIZE)); + + *ptr = iobuf->ptr; + +out: + if (iobuf != NULL) { + iobuf_unref(iobuf); + } + + if (iobref != NULL) { + *piobref = iobref; + } + + return ret; +} + +int32_t +ec_dict_set_array(dict_t *dict, char *key, uint64_t value[], int32_t size) +{ + int ret = -1; + uint64_t *ptr = NULL; + int32_t vindex; + + if (value == NULL) { + return -EINVAL; + } + + ptr = GF_MALLOC(sizeof(uint64_t) * size, gf_common_mt_char); + if (ptr == NULL) { + return -ENOMEM; + } + for (vindex = 0; vindex < size; vindex++) { + ptr[vindex] = hton64(value[vindex]); + } + ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t) * size); + if (ret) + GF_FREE(ptr); + return ret; +} + +int32_t +ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size) +{ + void *ptr; + int32_t len; + int32_t vindex; + int32_t old_size = 0; + int32_t err; + + if (dict == NULL) { + return -EINVAL; + } + err = dict_get_ptr_and_len(dict, key, &ptr, &len); + if (err != 0) { + return err; + } + + if (len > (size * sizeof(uint64_t)) || (len % sizeof(uint64_t))) { + return -EINVAL; + } + + /* 3.6 version ec would have stored version in 64 bit. In that case treat + * metadata versions same as data*/ + old_size = min(size, len / sizeof(uint64_t)); + for (vindex = 0; vindex < old_size; vindex++) { + value[vindex] = ntoh64(*((uint64_t *)ptr + vindex)); + } + + if (old_size < size) { + for (vindex = old_size; vindex < size; vindex++) { + value[vindex] = value[old_size - 1]; + } + } + + return 0; +} + +int32_t +ec_dict_del_array(dict_t *dict, char *key, uint64_t value[], int32_t size) +{ + int ret = 0; + + ret = ec_dict_get_array(dict, key, value, size); + if (ret == 0) + dict_del(dict, key); + + return ret; +} + +int32_t +ec_dict_set_number(dict_t *dict, char *key, uint64_t value) +{ + int ret = -1; + uint64_t *ptr; + + ptr = GF_MALLOC(sizeof(value), gf_common_mt_char); + if (ptr == NULL) { + return -ENOMEM; + } + + *ptr = hton64(value); + + ret = dict_set_bin(dict, key, ptr, sizeof(value)); + if (ret) + GF_FREE(ptr); + + return ret; +} + +int32_t +ec_dict_del_number(dict_t *dict, char *key, uint64_t *value) +{ + void *ptr; + int32_t len, err; + + if (dict == NULL) { + return -EINVAL; + } + err = dict_get_ptr_and_len(dict, key, &ptr, &len); + if (err != 0) { + return err; + } + if (len != sizeof(uint64_t)) { + return -EINVAL; + } + + *value = ntoh64(*(uint64_t *)ptr); + + dict_del(dict, key); + + return 0; +} + +int32_t +ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config) +{ + int ret = -1; + uint64_t *ptr, data; + + if (config->version > EC_CONFIG_VERSION) { + gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION, + "Trying to store an unsupported config " + "version (%u)", + config->version); + + return -EINVAL; + } + + ptr = GF_MALLOC(sizeof(uint64_t), gf_common_mt_char); + if (ptr == NULL) { + return -ENOMEM; + } + + data = ((uint64_t)config->version) << 56; + data |= ((uint64_t)config->algorithm) << 48; + data |= ((uint64_t)config->gf_word_size) << 40; + data |= ((uint64_t)config->bricks) << 32; + data |= ((uint64_t)config->redundancy) << 24; + data |= config->chunk_size; + + *ptr = hton64(data); + + ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t)); + if (ret) + GF_FREE(ptr); + + return ret; +} + +int32_t +ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config) +{ + void *ptr; + uint64_t data; + int32_t len, err; + + if (dict == NULL) { + return -EINVAL; + } + err = dict_get_ptr_and_len(dict, key, &ptr, &len); + if (err != 0) { + return err; + } + if (len != sizeof(uint64_t)) { + return -EINVAL; + } + + data = ntoh64(*(uint64_t *)ptr); + /* Currently we need to get the config xattr for entries of type IA_INVAL. + * These entries can later become IA_DIR entries (after inode_link()), + * which don't have a config xattr. However, since the xattr is requested + * using an xattrop() fop, it will always return a config full of 0's + * instead of saying that it doesn't exist. + * + * We need to filter out this case and consider that a config xattr == 0 is + * the same as a non-existent xattr. Otherwise ec_config_check() will fail. + */ + if (data == 0) { + return -ENODATA; + } + + config->version = (data >> 56) & 0xff; + if (config->version > EC_CONFIG_VERSION) { + gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION, + "Found an unsupported config version (%u)", config->version); + + return -EINVAL; + } + + config->algorithm = (data >> 48) & 0xff; + config->gf_word_size = (data >> 40) & 0xff; + config->bricks = (data >> 32) & 0xff; + config->redundancy = (data >> 24) & 0xff; + config->chunk_size = data & 0xffffff; + + dict_del(dict, key); + + return 0; +} + +gf_boolean_t +ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src) +{ + if (gf_uuid_is_null(src)) { + return _gf_true; + } + + if (gf_uuid_is_null(dst)) { + gf_uuid_copy(dst, src); + + return _gf_true; + } + + if (gf_uuid_compare(dst, src) != 0) { + gf_msg(xl->name, GF_LOG_WARNING, 0, EC_MSG_GFID_MISMATCH, + "Mismatching GFID's in loc"); + + return _gf_false; + } + + return _gf_true; +} + +int32_t +ec_loc_setup_inode(xlator_t *xl, inode_table_t *table, loc_t *loc) +{ + int32_t ret = -EINVAL; + + if (loc->inode != NULL) { + if (!ec_loc_gfid_check(xl, loc->gfid, loc->inode->gfid)) { + goto out; + } + } else if (table != NULL) { + if (!gf_uuid_is_null(loc->gfid)) { + loc->inode = inode_find(table, loc->gfid); + } else if (loc->path && strchr(loc->path, '/')) { + loc->inode = inode_resolve(table, (char *)loc->path); + } + } + + ret = 0; + +out: + return ret; +} + +int32_t +ec_loc_setup_parent(xlator_t *xl, inode_table_t *table, loc_t *loc) +{ + char *path, *parent; + int32_t ret = -EINVAL; + + if (loc->parent != NULL) { + if (!ec_loc_gfid_check(xl, loc->pargfid, loc->parent->gfid)) { + goto out; + } + } else if (table != NULL) { + if (!gf_uuid_is_null(loc->pargfid)) { + loc->parent = inode_find(table, loc->pargfid); + } else if (loc->path && strchr(loc->path, '/')) { + path = gf_strdup(loc->path); + if (path == NULL) { + gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Unable to duplicate path '%s'", loc->path); + + ret = -ENOMEM; + + goto out; + } + parent = dirname(path); + loc->parent = inode_resolve(table, parent); + if (loc->parent != NULL) { + gf_uuid_copy(loc->pargfid, loc->parent->gfid); + } + GF_FREE(path); + } + } + + /* If 'pargfid' has not been determined, clear 'name' to avoid resolutions + based on <gfid:pargfid>/name. */ + if (gf_uuid_is_null(loc->pargfid)) { + loc->name = NULL; + } + + ret = 0; + +out: + return ret; +} + +int32_t +ec_loc_setup_path(xlator_t *xl, loc_t *loc) +{ + static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}; + char *name; + int32_t ret = -EINVAL; + + if (loc->path != NULL) { + name = strrchr(loc->path, '/'); + if (name == NULL) { + /* Allow gfid paths: <gfid:...> */ + if (strncmp(loc->path, "<gfid:", 6) == 0) { + ret = 0; + } + goto out; + } + if (name == loc->path) { + if (name[1] == 0) { + if (!ec_loc_gfid_check(xl, loc->gfid, root)) { + goto out; + } + } else { + if (!ec_loc_gfid_check(xl, loc->pargfid, root)) { + goto out; + } + } + } + name++; + + if (loc->name != NULL) { + if (strcmp(loc->name, name) != 0) { + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_LOC_NAME, + "Invalid name '%s' in loc", loc->name); + + goto out; + } + } else { + loc->name = name; + } + } + + ret = 0; + +out: + return ret; +} + +int32_t +ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent) +{ + inode_table_t *table = NULL; + char *str = NULL; + int32_t ret = -ENOMEM; + + memset(parent, 0, sizeof(loc_t)); + + if (loc->parent != NULL) { + table = loc->parent->table; + parent->inode = inode_ref(loc->parent); + } else if (loc->inode != NULL) { + table = loc->inode->table; + } + if (!gf_uuid_is_null(loc->pargfid)) { + gf_uuid_copy(parent->gfid, loc->pargfid); + } + if (loc->path && strchr(loc->path, '/')) { + str = gf_strdup(loc->path); + if (str == NULL) { + gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Unable to duplicate path '%s'", loc->path); + + goto out; + } + parent->path = gf_strdup(dirname(str)); + if (parent->path == NULL) { + gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Unable to duplicate path '%s'", dirname(str)); + + goto out; + } + } + + ret = ec_loc_setup_path(xl, parent); + if (ret == 0) { + ret = ec_loc_setup_inode(xl, table, parent); + } + if (ret == 0) { + ret = ec_loc_setup_parent(xl, table, parent); + } + if (ret != 0) { + goto out; + } + + if ((parent->inode == NULL) && (parent->path == NULL) && + gf_uuid_is_null(parent->gfid)) { + gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_LOC_PARENT_INODE_MISSING, + "Parent inode missing for loc_t"); + + ret = -EINVAL; + + goto out; + } + + ret = 0; + +out: + GF_FREE(str); + + if (ret != 0) { + loc_wipe(parent); + } + + return ret; +} + +int32_t +ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt) +{ + inode_table_t *table = NULL; + int32_t ret = -EINVAL; + + if (inode != NULL) { + table = inode->table; + if (loc->inode != inode) { + if (loc->inode != NULL) { + inode_unref(loc->inode); + } + loc->inode = inode_ref(inode); + gf_uuid_copy(loc->gfid, inode->gfid); + } + } else if (loc->inode != NULL) { + table = loc->inode->table; + } else if (loc->parent != NULL) { + table = loc->parent->table; + } + + if (iatt != NULL) { + if (!ec_loc_gfid_check(xl, loc->gfid, iatt->ia_gfid)) { + goto out; + } + } + + ret = ec_loc_setup_path(xl, loc); + if (ret == 0) { + ret = ec_loc_setup_inode(xl, table, loc); + } + if (ret == 0) { + ret = ec_loc_setup_parent(xl, table, loc); + } + if (ret != 0) { + goto out; + } + +out: + return ret; +} + +int32_t +ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd) +{ + ec_fd_t *ctx; + int32_t ret = -ENOMEM; + + memset(loc, 0, sizeof(*loc)); + + ctx = ec_fd_get(fd, xl); + if (ctx != NULL) { + if (loc_copy(loc, &ctx->loc) != 0) { + goto out; + } + } + + ret = ec_loc_update(xl, loc, fd->inode, NULL); + if (ret != 0) { + goto out; + } + +out: + if (ret != 0) { + loc_wipe(loc); + } + + return ret; +} + +int32_t +ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src) +{ + int32_t ret = -ENOMEM; + + memset(dst, 0, sizeof(*dst)); + + if (loc_copy(dst, src) != 0) { + goto out; + } + + ret = ec_loc_update(xl, dst, NULL, NULL); + if (ret != 0) { + goto out; + } + +out: + if (ret != 0) { + loc_wipe(dst); + } + + return ret; +} + +void +ec_owner_set(call_frame_t *frame, void *owner) +{ + set_lk_owner_from_ptr(&frame->root->lk_owner, owner); +} + +void +ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner) +{ + lk_owner_copy(&frame->root->lk_owner, owner); +} + +static void +ec_stripe_cache_init(ec_t *ec, ec_inode_t *ctx) +{ + ec_stripe_list_t *stripe_cache = NULL; + + stripe_cache = &(ctx->stripe_cache); + if (stripe_cache->max == 0) { + stripe_cache->max = ec->stripe_cache; + } +} + +ec_inode_t * +__ec_inode_get(inode_t *inode, xlator_t *xl) +{ + ec_inode_t *ctx = NULL; + uint64_t value = 0; + + if ((__inode_ctx_get(inode, xl, &value) != 0) || (value == 0)) { + ctx = GF_MALLOC(sizeof(*ctx), ec_mt_ec_inode_t); + if (ctx != NULL) { + memset(ctx, 0, sizeof(*ctx)); + INIT_LIST_HEAD(&ctx->heal); + INIT_LIST_HEAD(&ctx->stripe_cache.lru); + ctx->heal_count = 0; + value = (uint64_t)(uintptr_t)ctx; + if (__inode_ctx_set(inode, xl, &value) != 0) { + GF_FREE(ctx); + + return NULL; + } + } + } else { + ctx = (ec_inode_t *)(uintptr_t)value; + } + if (ctx) + ec_stripe_cache_init(xl->private, ctx); + + return ctx; +} + +ec_inode_t * +ec_inode_get(inode_t *inode, xlator_t *xl) +{ + ec_inode_t *ctx = NULL; + + LOCK(&inode->lock); + + ctx = __ec_inode_get(inode, xl); + + UNLOCK(&inode->lock); + + return ctx; +} + +ec_fd_t * +__ec_fd_get(fd_t *fd, xlator_t *xl) +{ + int i = 0; + ec_fd_t *ctx = NULL; + ec_inode_t *ictx = NULL; + uint64_t value = 0; + ec_t *ec = xl->private; + + if ((__fd_ctx_get(fd, xl, &value) != 0) || (value == 0)) { + ctx = GF_MALLOC(sizeof(*ctx) + (sizeof(ec_fd_status_t) * ec->nodes), + ec_mt_ec_fd_t); + if (ctx != NULL) { + memset(ctx, 0, sizeof(*ctx)); + + for (i = 0; i < ec->nodes; i++) { + if (fd_is_anonymous(fd)) { + ctx->fd_status[i] = EC_FD_OPENED; + } else { + ctx->fd_status[i] = EC_FD_NOT_OPENED; + } + } + + value = (uint64_t)(uintptr_t)ctx; + if (__fd_ctx_set(fd, xl, value) != 0) { + GF_FREE(ctx); + return NULL; + } + /* Only refering bad-version so no need for lock + * */ + ictx = __ec_inode_get(fd->inode, xl); + if (ictx) { + ctx->bad_version = ictx->bad_version; + } + } + } else { + ctx = (ec_fd_t *)(uintptr_t)value; + } + + /* Treat anonymous fd specially */ + if (fd->anonymous && ctx) { + /* Mark the fd open for all subvolumes. */ + ctx->open = -1; + /* Try to populate ctx->loc with fd->inode information. */ + ec_loc_update(xl, &ctx->loc, fd->inode, NULL); + } + + return ctx; +} + +ec_fd_t * +ec_fd_get(fd_t *fd, xlator_t *xl) +{ + ec_fd_t *ctx = NULL; + + LOCK(&fd->lock); + + ctx = __ec_fd_get(fd, xl); + + UNLOCK(&fd->lock); + + return ctx; +} + +gf_boolean_t +ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data) +{ + if (key && (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0)) + return _gf_true; + + return _gf_false; +} + +void +ec_filter_internal_xattrs(dict_t *xattr) +{ + dict_foreach_match(xattr, ec_is_internal_xattr, NULL, + dict_remove_foreach_fn, NULL); +} + +gf_boolean_t +ec_is_data_fop(glusterfs_fop_t fop) +{ + switch (fop) { + case GF_FOP_WRITE: + case GF_FOP_TRUNCATE: + case GF_FOP_FTRUNCATE: + case GF_FOP_FALLOCATE: + case GF_FOP_DISCARD: + case GF_FOP_ZEROFILL: + return _gf_true; + default: + return _gf_false; + } + return _gf_false; +} +/* +gf_boolean_t +ec_is_metadata_fop (int32_t lock_kind, glusterfs_fop_t fop) +{ + if (lock_kind == EC_LOCK_ENTRY) { + return _gf_false; + } + + switch (fop) { + case GF_FOP_SETATTR: + case GF_FOP_FSETATTR: + case GF_FOP_SETXATTR: + case GF_FOP_FSETXATTR: + case GF_FOP_REMOVEXATTR: + case GF_FOP_FREMOVEXATTR: + return _gf_true; + default: + return _gf_false; + } + return _gf_false; +}*/ diff --git a/xlators/cluster/ec/src/ec-helpers.h b/xlators/cluster/ec/src/ec-helpers.h new file mode 100644 index 00000000000..015db514e05 --- /dev/null +++ b/xlators/cluster/ec/src/ec-helpers.h @@ -0,0 +1,200 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_HELPERS_H__ +#define __EC_HELPERS_H__ + +#include "ec-types.h" + +#define EC_ERR(_x) ((void *)-(intptr_t)(_x)) +#define EC_IS_ERR(_x) (((uintptr_t)(_x) & ~0xfffULL) == ~0xfffULL) +#define EC_GET_ERR(_x) ((int32_t)(intptr_t)(_x)) + +#define EC_ALIGN_CHECK(_ptr, _align) ((((uintptr_t)(_ptr)) & ((_align)-1)) == 0) + +const char * +ec_bin(char *str, size_t size, uint64_t value, int32_t digits); +const char * +ec_fop_name(int32_t id); +void +ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...); +int32_t +ec_bits_consume(uint64_t *n); +size_t +ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset, + size_t size); +int32_t +ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr); +int32_t +ec_dict_set_array(dict_t *dict, char *key, uint64_t *value, int32_t size); +int32_t +ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size); + +int32_t +ec_dict_del_array(dict_t *dict, char *key, uint64_t *value, int32_t size); +int32_t +ec_dict_set_number(dict_t *dict, char *key, uint64_t value); +int32_t +ec_dict_del_number(dict_t *dict, char *key, uint64_t *value); +int32_t +ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config); +int32_t +ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config); + +int32_t +ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent); +int32_t +ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt); + +int32_t +ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd); +int32_t +ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src); + +void +ec_owner_set(call_frame_t *frame, void *owner); +void +ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner); + +ec_inode_t * +__ec_inode_get(inode_t *inode, xlator_t *xl); +ec_inode_t * +ec_inode_get(inode_t *inode, xlator_t *xl); +ec_fd_t * +__ec_fd_get(fd_t *fd, xlator_t *xl); +ec_fd_t * +ec_fd_get(fd_t *fd, xlator_t *xl); + +static inline uint32_t +ec_adjust_size_down(ec_t *ec, uint64_t *value, gf_boolean_t scale) +{ + uint64_t head, tmp; + + tmp = *value; + head = tmp % ec->stripe_size; + tmp -= head; + + if (scale) { + tmp /= ec->fragments; + } + + *value = tmp; + + return (uint32_t)head; +} + +/* This function can cause an overflow if the passed value is too near to the + * uint64_t limit. If this happens, it returns the tail in negative form and + * the value is set to UINT64_MAX. */ +static inline int32_t +ec_adjust_size_up(ec_t *ec, uint64_t *value, gf_boolean_t scale) +{ + uint64_t tmp; + int32_t tail; + + tmp = *value; + /* We first adjust the value down. This never causes overflow. */ + tail = ec_adjust_size_down(ec, &tmp, scale); + + /* If the value was already aligned, tail will be 0 and nothing else + * needs to be done. */ + if (tail != 0) { + /* Otherwise, we need to compute the real tail and adjust the + * returned value to the next stripe. */ + tail = ec->stripe_size - tail; + if (scale) { + tmp += ec->fragment_size; + } else { + tmp += ec->stripe_size; + /* If no scaling is requested there's a possibility of + * overflow. */ + if (tmp < ec->stripe_size) { + tmp = UINT64_MAX; + tail = -tail; + } + } + } + + *value = tmp; + + return tail; +} + +/* This function is equivalent to ec_adjust_size_down() but with a potentially + * different parameter size (off_t vs uint64_t). */ +static inline uint32_t +ec_adjust_offset_down(ec_t *ec, off_t *value, gf_boolean_t scale) +{ + off_t head, tmp; + + tmp = *value; + head = tmp % ec->stripe_size; + tmp -= head; + + if (scale) { + tmp /= ec->fragments; + } + + *value = tmp; + + return (uint32_t)head; +} + +/* This function is equivalent to ec_adjust_size_up() but with a potentially + * different parameter size (off_t vs uint64_t). */ +static inline int32_t +ec_adjust_offset_up(ec_t *ec, off_t *value, gf_boolean_t scale) +{ + uint64_t tail, tmp; + + /* An offset is a signed type that can only have positive values, so + * we take advantage of this to avoid overflows. We simply convert it + * to an unsigned integer and operate normally. This won't cause an + * overflow. Overflow is only checked when converting back to an + * off_t. */ + tmp = *value; + tail = ec->stripe_size; + tail -= (tmp + tail - 1) % tail + 1; + tmp += tail; + if (scale) { + /* If we are scaling, we'll never get an overflow. */ + tmp /= ec->fragments; + } else { + /* Check if there has been an overflow. */ + if ((off_t)tmp < 0) { + tmp = GF_OFF_MAX; + tail = -tail; + } + } + + *value = (off_t)tmp; + + return (int32_t)tail; +} + +static inline int32_t +ec_is_power_of_2(uint32_t value) +{ + return (value != 0) && ((value & (value - 1)) == 0); +} + +gf_boolean_t +ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data); + +void +ec_filter_internal_xattrs(dict_t *xattr); + +gf_boolean_t +ec_is_data_fop(glusterfs_fop_t fop); + +int32_t +ec_launch_replace_heal(ec_t *ec); + +#endif /* __EC_HELPERS_H__ */ diff --git a/xlators/cluster/ec/src/ec-inode-read.c b/xlators/cluster/ec/src/ec-inode-read.c new file mode 100644 index 00000000000..dad5f4d7018 --- /dev/null +++ b/xlators/cluster/ec/src/ec-inode-read.c @@ -0,0 +1,2046 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "ec.h" +#include "ec-messages.h" +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-combine.h" +#include "ec-method.h" +#include "ec-fops.h" + +/* FOP: access */ + +int32_t +ec_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ACCESS, idx, op_ret, + op_errno); + if (cbk) { + if (xdata) + cbk->xdata = dict_ref(xdata); + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_access(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_access_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->access, + &fop->loc[0], fop->int32, fop->xdata); +} + +int32_t +ec_manager_access(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk = NULL; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_one(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + if (ec_dispatch_one_retry(fop, NULL)) { + return EC_STATE_DISPATCH; + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + GF_ASSERT(cbk); + if (fop->cbks.access != NULL) { + if (cbk) { + fop->cbks.access(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); + } + } + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + if (fop->cbks.access != NULL) { + fop->cbks.access(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL); + } + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc, + int32_t mask, dict_t *xdata) +{ + ec_cbk_t callback = {.access = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(ACCESS) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_ACCESS, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_access, + ec_manager_access, callback, data); + if (fop == NULL) { + goto out; + } + + fop->int32 = mask; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: getxattr */ + +int32_t +ec_combine_getxattr(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (!ec_dict_compare(dst->dict, src->dict)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_DICT_MISMATCH, + "Mismatching dictionary in " + "answers of 'GF_FOP_GETXATTR'"); + + return 0; + } + + return 1; +} + +int32_t +ec_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_GETXATTR, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (dict != NULL) { + cbk->dict = dict_ref(dict); + if (cbk->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, ec_combine_getxattr); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_getxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_getxattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->getxattr, + &fop->loc[0], fop->str[0], fop->xdata); +} + +void +ec_handle_special_xattrs(ec_fop_data_t *fop) +{ + ec_cbk_data_t *cbk = NULL; + /* Stime may not be available on all the bricks, so even if some of the + * subvols succeed the operation, treat it as answer.*/ + if (fop->str[0] && fnmatch(GF_XATTR_STIME_PATTERN, fop->str[0], 0) == 0) { + if (!fop->answer || (fop->answer->op_ret < 0)) { + list_for_each_entry(cbk, &fop->cbk_list, list) + { + if (cbk->op_ret >= 0) { + fop->answer = cbk; + break; + } + } + } + } +} + +int32_t +ec_manager_getxattr(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + /* clear-locks commands must be done without any locks acquired + to avoid interferences. */ + if ((fop->str[0] == NULL) || + (strncmp(fop->str[0], GF_XATTR_CLRLK_CMD, + SLEN(GF_XATTR_CLRLK_CMD)) != 0)) { + if (fop->fd == NULL) { + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, + EC_RANGE_FULL); + } + ec_lock(fop); + } + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + if (fop->minimum == EC_MINIMUM_ALL) { + ec_dispatch_all(fop); + } else { + ec_dispatch_one(fop); + } + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + ec_handle_special_xattrs(fop); + if (fop->minimum == EC_MINIMUM_ALL) { + cbk = ec_fop_prepare_answer(fop, _gf_true); + } else { + if (ec_dispatch_one_retry(fop, &cbk)) { + return EC_STATE_DISPATCH; + } + } + if (cbk != NULL) { + int32_t err; + + err = ec_dict_combine(cbk, EC_COMBINE_DICT); + if (!ec_cbk_set_error(cbk, -err, _gf_true)) { + if (cbk->xdata != NULL) + ec_filter_internal_xattrs(cbk->xdata); + + if (cbk->dict != NULL) + ec_filter_internal_xattrs(cbk->dict); + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.getxattr != NULL) { + fop->cbks.getxattr(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->dict, cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.getxattr != NULL) { + fop->cbks.getxattr(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +int32_t +ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, + int32_t op_ret, int32_t op_errno, uintptr_t mask, + uintptr_t good, uintptr_t bad, uint32_t pending, + dict_t *xdata) +{ + fop_getxattr_cbk_t func = cookie; + ec_t *ec = xl->private; + dict_t *dict = NULL; + char *str; + char bin1[65], bin2[65]; + + /* We try to return the 'pending' information in xdata, but if this cannot + * be set, we will ignore it silently. We prefer to report the success or + * failure of the heal itself. */ + if (xdata == NULL) { + xdata = dict_new(); + } else { + dict_ref(xdata); + } + if (xdata != NULL) { + if (dict_set_uint32(xdata, EC_XATTR_HEAL_NEW, pending) != 0) { + /* dict_set_uint32() is marked as 'warn_unused_result' and gcc + * enforces to check the result in this case. However we don't + * really care if it succeeded or not. We'll just do the same. + * + * This empty 'if' avoids the warning, and it will be removed by + * the optimizer. */ + } + } + + if (op_ret >= 0) { + dict = dict_new(); + if (dict == NULL) { + op_ret = -1; + op_errno = ENOMEM; + } else { + if (gf_asprintf(&str, "Good: %s, Bad: %s", + ec_bin(bin1, sizeof(bin1), good, ec->nodes), + ec_bin(bin2, sizeof(bin2), mask & ~(good | bad), + ec->nodes)) < 0) { + dict_unref(dict); + dict = NULL; + + op_ret = -1; + op_errno = ENOMEM; + + goto out; + } + + if (dict_set_dynstr(dict, EC_XATTR_HEAL, str) != 0) { + GF_FREE(str); + dict_unref(dict); + dict = NULL; + + op_ret = -1; + op_errno = ENOMEM; + + goto out; + } + } + } + +out: + func(frame, NULL, xl, op_ret, op_errno, dict, xdata); + + if (dict != NULL) { + dict_unref(dict); + } + if (xdata != NULL) { + dict_unref(xdata); + } + + return 0; +} + +void +ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc, + const char *name, dict_t *xdata) +{ + ec_cbk_t callback = {.getxattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(GETXATTR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + /* Special handling of an explicit self-heal request */ + if ((name != NULL) && (strcmp(name, EC_XATTR_HEAL) == 0)) { + ec_heal(frame, this, target, EC_MINIMUM_ONE, ec_getxattr_heal_cbk, func, + loc, 0, NULL); + + return; + } + + fop = ec_fop_data_allocate( + frame, this, GF_FOP_GETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags, + ec_wind_getxattr, ec_manager_getxattr, callback, data); + if (fop == NULL) { + goto out; + } + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (name != NULL) { + /* In case of list-node-uuids xattr, set flag to indicate + * the same and use node-uuid xattr for winding fop */ + if (XATTR_IS_NODE_UUID_LIST(name)) { + fop->int32 = 1; + fop->str[0] = gf_strdup(GF_XATTR_NODE_UUID_KEY); + } else { + fop->str[0] = gf_strdup(name); + } + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} + +/* FOP: fgetxattr */ + +int32_t +ec_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FGETXATTR, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (dict != NULL) { + cbk->dict = dict_ref(dict); + if (cbk->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, ec_combine_getxattr); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_fgetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fgetxattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fgetxattr, + fop->fd, fop->str[0], fop->xdata); +} + +void +ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd, + const char *name, dict_t *xdata) +{ + ec_cbk_t callback = {.fgetxattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FGETXATTR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate( + frame, this, GF_FOP_FGETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags, + ec_wind_fgetxattr, ec_manager_getxattr, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (name != NULL) { + fop->str[0] = gf_strdup(name); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} + +/* FOP: open */ + +int32_t +ec_combine_open(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (dst->fd != src->fd) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH, + "Mismatching fd in answers " + "of 'GF_FOP_OPEN': %p <-> %p", + dst->fd, src->fd); + + return 0; + } + + return 1; +} + +int32_t +ec_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, fd_t *fd, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPEN, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (fd != NULL) { + cbk->fd = fd_ref(fd); + if (cbk->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, + EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, ec_combine_open); + + ec_update_fd_status(fd, this, idx, op_ret); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_open(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_open_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->open, + &fop->loc[0], fop->int32, fop->fd, fop->xdata); +} + +int32_t +ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + int32_t error = 0; + + fop = fop->data; + if (op_ret >= 0) { + fop->answer->iatt[0] = *postbuf; + } else { + error = op_errno; + } + + ec_resume(fop, error); + + return 0; +} + +int32_t +ec_manager_open(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + ec_fd_t *ctx; + int32_t err; + + switch (state) { + case EC_STATE_INIT: + LOCK(&fop->fd->lock); + + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx == NULL) { + UNLOCK(&fop->fd->lock); + + fop->error = ENOMEM; + + return EC_STATE_REPORT; + } + if (!ctx->loc.inode) { + err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]); + if (err != 0) { + UNLOCK(&fop->fd->lock); + + fop->error = -err; + + return EC_STATE_REPORT; + } + } + + ctx->flags = fop->int32; + + UNLOCK(&fop->fd->lock); + + /* We need to write to specific offsets on the bricks, so we + need to remove O_APPEND from flags (if present). + If O_TRUNC is specified, we remove it from open and an + ftruncate will be executed later, which will correctly update + the file size taking appropriate locks. O_TRUNC flag is saved + into fop->uint32 to use it later.*/ + fop->uint32 = fop->int32 & O_TRUNC; + fop->int32 &= ~(O_APPEND | O_TRUNC); + + /* Fall through */ + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + int32_t err; + + err = ec_loc_update(fop->xl, &fop->loc[0], cbk->fd->inode, + NULL); + if (!ec_cbk_set_error(cbk, -err, _gf_true)) { + LOCK(&fop->fd->lock); + + ctx = __ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + ctx->open |= cbk->mask; + } + + UNLOCK(&fop->fd->lock); + + /* If O_TRUNC was specified, call ftruncate to + effectively trunc the file with appropriate locks + acquired. We don't use ctx->flags because self-heal + can use the same fd with different flags. */ + if (fop->uint32 != 0) { + ec_sleep(fop); + ec_ftruncate(fop->req_frame, fop->xl, cbk->mask, + fop->minimum, ec_open_truncate_cbk, fop, + cbk->fd, 0, NULL); + } + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.open != NULL) { + fop->cbks.open(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->fd, cbk->xdata); + } + + return EC_STATE_END; + + case -EC_STATE_INIT: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.open != NULL) { + fop->cbks.open(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL); + } + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc, + int32_t flags, fd_t *fd, dict_t *xdata) +{ + ec_cbk_t callback = {.open = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(OPEN) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_OPEN, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_open, ec_manager_open, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->int32 = flags; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} + +/* FOP: readlink */ + +int32_t +ec_combine_readlink(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_READLINK'"); + + return 0; + } + + return 1; +} + +int32_t +ec_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, const char *path, + struct iatt *buf, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, + op_errno); + if (cbk) { + if (xdata) + cbk->xdata = dict_ref(xdata); + + if (cbk->op_ret >= 0) { + cbk->iatt[0] = *buf; + cbk->str = gf_strdup(path); + if (!cbk->str) { + ec_cbk_set_error(cbk, ENOMEM, _gf_true); + } + } + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) + ec_complete(fop); + + return 0; +} + +void +ec_wind_readlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_readlink_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->readlink, + &fop->loc[0], fop->size, fop->xdata); +} + +int32_t +ec_manager_readlink(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk = NULL; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); + ec_lock(fop); + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_one(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + if (ec_dispatch_one_retry(fop, &cbk)) { + return EC_STATE_DISPATCH; + } + + if ((cbk != NULL) && (cbk->op_ret >= 0)) { + ec_iatt_rebuild(fop->xl->private, &cbk->iatt[0], 1, 1); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + GF_ASSERT(cbk); + if (fop->cbks.readlink != NULL) { + fop->cbks.readlink(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->str, &cbk->iatt[0], + cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + if (fop->cbks.readlink != NULL) { + fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL); + } + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc, + size_t size, dict_t *xdata) +{ + ec_cbk_t callback = {.readlink = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(READLINK) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate( + frame, this, GF_FOP_READLINK, EC_FLAG_LOCK_SHARED, target, fop_flags, + ec_wind_readlink, ec_manager_readlink, callback, data); + if (fop == NULL) { + goto out; + } + + fop->size = size; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} + +/* FOP: readv */ + +int32_t +ec_readv_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk) +{ + struct iovec vector[1]; + ec_cbk_data_t *ans = NULL; + struct iobref *iobref = NULL; + void *ptr; + uint64_t fsize = 0, size = 0, max = 0; + int32_t pos, err = -ENOMEM; + + if (cbk->op_ret < 0) { + err = -cbk->op_errno; + + goto out; + } + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, &cbk->iatt[0].ia_size)); + + if (cbk->op_ret > 0) { + void *blocks[cbk->count]; + uint32_t values[cbk->count]; + + fsize = cbk->op_ret; + size = fsize * ec->fragments; + for (ans = cbk; ans != NULL; ans = ans->next) { + pos = gf_bits_count(cbk->mask & ((1 << ans->idx) - 1)); + values[pos] = ans->idx + 1; + blocks[pos] = ans->vector[0].iov_base; + if ((ans->int32 != 1) || + !EC_ALIGN_CHECK(blocks[pos], EC_METHOD_WORD_SIZE)) { + if (iobref == NULL) { + err = ec_buffer_alloc(ec->xl, size, &iobref, &ptr); + if (err != 0) { + goto out; + } + } + ec_iov_copy_to(ptr, ans->vector, ans->int32, 0, fsize); + blocks[pos] = ptr; + ptr += fsize; + } + } + + err = ec_buffer_alloc(ec->xl, size, &iobref, &ptr); + if (err != 0) { + goto out; + } + + err = ec_method_decode(&ec->matrix, fsize, cbk->mask, values, blocks, + ptr); + if (err != 0) { + goto out; + } + + vector[0].iov_base = ptr + fop->head; + vector[0].iov_len = size - fop->head; + + max = fop->offset * ec->fragments + size; + if (max > cbk->iatt[0].ia_size) { + max = cbk->iatt[0].ia_size; + } + max -= fop->offset * ec->fragments + fop->head; + if (max > fop->user_size) { + max = fop->user_size; + } + size -= fop->head; + if (size > max) { + vector[0].iov_len -= size - max; + size = max; + } + + cbk->op_ret = size; + cbk->int32 = 1; + + iobref_unref(cbk->buffers); + cbk->buffers = iobref; + + GF_FREE(cbk->vector); + cbk->vector = iov_dup(vector, 1); + if (cbk->vector == NULL) { + return -ENOMEM; + } + } + + return 0; + +out: + if (iobref != NULL) { + iobref_unref(iobref); + } + + return err; +} + +int32_t +ec_combine_readv(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (!ec_vector_compare(dst->vector, dst->int32, src->vector, src->int32)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_VECTOR_MISMATCH, + "Mismatching vector in " + "answers of 'GF_FOP_READ'"); + + return 0; + } + + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_READ'"); + + return 0; + } + + return 1; +} + +int32_t +ec_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iovec *vector, int32_t count, + struct iatt *stbuf, struct iobref *iobref, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + ec_t *ec = this->private; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_READ, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + cbk->int32 = count; + + if (count > 0) { + cbk->vector = iov_dup(vector, count); + if (cbk->vector == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a " + "vector list."); + + goto out; + } + cbk->int32 = count; + } + if (stbuf != NULL) { + cbk->iatt[0] = *stbuf; + } + if (iobref != NULL) { + cbk->buffers = iobref_ref(iobref); + if (cbk->buffers == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL, + "Failed to reference a " + "buffer."); + + goto out; + } + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) { + ec_cbk_set_error(cbk, EIO, _gf_true); + } + + ec_combine(cbk, ec_combine_readv); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_readv(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_readv_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->readv, fop->fd, + fop->size, fop->offset, fop->uint32, fop->xdata); +} + +int32_t +ec_manager_readv(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + ec_t *ec = fop->xl->private; + + switch (state) { + case EC_STATE_INIT: + fop->user_size = fop->size; + fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset, + _gf_true); + fop->size += fop->head; + ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true); + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset, + fop->size); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + if (ec->read_mask) { + fop->mask &= ec->read_mask; + } + ec_dispatch_min(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_true); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count); + + err = ec_readv_rebuild(fop->xl->private, fop, cbk); + if (err != 0) { + ec_cbk_set_error(cbk, -err, _gf_true); + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.readv != NULL) { + fop->cbks.readv(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->vector, cbk->int32, + &cbk->iatt[0], cbk->buffers, cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.readv != NULL) { + fop->cbks.readv(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, 0, NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd, + size_t size, off_t offset, uint32_t flags, dict_t *xdata) +{ + ec_cbk_t callback = {.readv = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(READ) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_READ, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_readv, + ec_manager_readv, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->size = size; + fop->offset = offset; + fop->uint32 = flags; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, 0, NULL, NULL, NULL); + } +} + +/* FOP: seek */ + +int32_t +ec_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, off_t offset, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + ec_t *ec = this->private; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_SEEK, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + cbk->offset = offset; + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + } + + if ((op_ret > 0) && ((cbk->offset % ec->fragment_size) != 0)) { + cbk->op_ret = -1; + cbk->op_errno = EIO; + } + + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_seek(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_seek_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->seek, fop->fd, + fop->offset, fop->seek, fop->xdata); +} + +int32_t +ec_manager_seek(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + uint64_t size; + + switch (state) { + case EC_STATE_INIT: + fop->user_size = fop->offset; + fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset, + _gf_true); + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset, + EC_RANGE_FULL); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT( + ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, &size)); + + if (fop->user_size >= size) { + ec_fop_set_error(fop, ENXIO); + + return EC_STATE_REPORT; + } + + ec_dispatch_one(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + if (ec_dispatch_one_retry(fop, &cbk)) { + return EC_STATE_DISPATCH; + } + if ((cbk != NULL) && (cbk->op_ret >= 0)) { + ec_t *ec = fop->xl->private; + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, + &size)); + + cbk->offset *= ec->fragments; + if (cbk->offset < fop->user_size) { + cbk->offset = fop->user_size; + } + if (cbk->offset > size) { + cbk->offset = size; + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.seek != NULL) { + fop->cbks.seek(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->offset, cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.seek != NULL) { + fop->cbks.seek(fop->req_frame, fop, fop->xl, -1, fop->error, 0, + NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd, + off_t offset, gf_seek_what_t what, dict_t *xdata) +{ + ec_cbk_t callback = {.seek = func}; + ec_fop_data_t *fop = NULL; + int32_t error = EIO; + + gf_msg_trace("ec", 0, "EC(SEEK) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_SEEK, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_seek, ec_manager_seek, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->offset = offset; + fop->seek = what; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, EIO, 0, NULL); + } +} + +/* FOP: stat */ + +int32_t +ec_combine_stat(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH, + "Mismatching iatt in " + "answers of 'GF_FOP_STAT'"); + + return 0; + } + + return 1; +} + +int32_t +ec_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *buf, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STAT, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (buf != NULL) { + cbk->iatt[0] = *buf; + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, ec_combine_stat); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_stat(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_stat_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->stat, + &fop->loc[0], fop->xdata); +} + +int32_t +ec_manager_stat(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + if (fop->fd == NULL) { + ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0, + EC_RANGE_FULL); + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, + EC_RANGE_FULL); + } + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_true); + + if (cbk != NULL) { + if (cbk->iatt[0].ia_type == IA_IFREG) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, + fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->id == GF_FOP_STAT) { + if (fop->cbks.stat != NULL) { + fop->cbks.stat(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, &cbk->iatt[0], cbk->xdata); + } + } else { + if (fop->cbks.fstat != NULL) { + fop->cbks.fstat(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, &cbk->iatt[0], cbk->xdata); + } + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->id == GF_FOP_STAT) { + if (fop->cbks.stat != NULL) { + fop->cbks.stat(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL); + } + } else { + if (fop->cbks.fstat != NULL) { + fop->cbks.fstat(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL); + } + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc, + dict_t *xdata) +{ + ec_cbk_t callback = {.stat = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(STAT) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_STAT, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_stat, ec_manager_stat, + callback, data); + if (fop == NULL) { + goto out; + } + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} + +/* FOP: fstat */ + +int32_t +ec_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *buf, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSTAT, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (buf != NULL) { + cbk->iatt[0] = *buf; + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, ec_combine_stat); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_fstat(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fstat_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fstat, fop->fd, + fop->xdata); +} + +void +ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd, + dict_t *xdata) +{ + ec_cbk_t callback = {.fstat = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FSTAT) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSTAT, EC_FLAG_LOCK_SHARED, + target, fop_flags, ec_wind_fstat, + ec_manager_stat, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} diff --git a/xlators/cluster/ec/src/ec-inode-write.c b/xlators/cluster/ec/src/ec-inode-write.c new file mode 100644 index 00000000000..9b5fe2a7fdc --- /dev/null +++ b/xlators/cluster/ec/src/ec-inode-write.c @@ -0,0 +1,2369 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "ec-messages.h" +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-combine.h" +#include "ec-method.h" +#include "ec-fops.h" +#include "ec-mem-types.h" + +int32_t +ec_update_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + ec_cbk_data_t *cbk = NULL; + ec_fop_data_t *parent = fop->parent; + int i = 0; + + ec_trace("UPDATE_WRITEV_CBK", cookie, "ret=%d, errno=%d, parent-fop=%s", + op_ret, op_errno, ec_fop_name(parent->id)); + + if (op_ret < 0) { + ec_fop_set_error(parent, op_errno); + goto out; + } + cbk = ec_cbk_data_allocate(parent->frame, this, parent, parent->id, 0, + op_ret, op_errno); + if (!cbk) { + ec_fop_set_error(parent, ENOMEM); + goto out; + } + + if (xdata) + cbk->xdata = dict_ref(xdata); + + if (prebuf) + cbk->iatt[i++] = *prebuf; + + if (postbuf) + cbk->iatt[i++] = *postbuf; + + LOCK(&parent->lock); + { + parent->good &= fop->good; + + if (gf_bits_count(parent->good) < parent->minimum) { + __ec_fop_set_error(parent, EIO); + } else if (fop->error == 0 && parent->answer == NULL) { + parent->answer = cbk; + } + } + UNLOCK(&parent->lock); +out: + return 0; +} + +static int32_t +ec_update_write(ec_fop_data_t *fop, uintptr_t mask, off_t offset, uint64_t size) +{ + struct iobref *iobref = NULL; + struct iobuf *iobuf = NULL; + struct iovec vector; + int32_t err = -ENOMEM; + + iobref = iobref_new(); + if (iobref == NULL) { + goto out; + } + iobuf = iobuf_get(fop->xl->ctx->iobuf_pool); + if (iobuf == NULL) { + goto out; + } + err = iobref_add(iobref, iobuf); + if (err != 0) { + goto out; + } + + if (fop->locks[0].lock) + ec_lock_update_good(fop->locks[0].lock, fop); + vector.iov_base = iobuf->ptr; + vector.iov_len = size; + memset(vector.iov_base, 0, vector.iov_len); + + ec_writev(fop->frame, fop->xl, mask, fop->minimum, ec_update_writev_cbk, + NULL, fop->fd, &vector, 1, offset, 0, iobref, NULL); + + err = 0; + +out: + if (iobuf != NULL) { + iobuf_unref(iobuf); + } + if (iobref != NULL) { + iobref_unref(iobref); + } + + return err; +} + +int +ec_inode_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie, + int op_ret, int op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int i = 0; + int idx = 0; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + idx = (int32_t)(uintptr_t)cookie; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret, + op_errno); + if (!cbk) + goto out; + + if (op_ret < 0) + goto out; + + if (xdata) + cbk->xdata = dict_ref(xdata); + + if (prestat) + cbk->iatt[i++] = *prestat; + + if (poststat) + cbk->iatt[i++] = *poststat; + +out: + if (cbk) + ec_combine(cbk, ec_combine_write); + + if (fop) + ec_complete(fop); + return 0; +} +/* FOP: removexattr */ + +int32_t +ec_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL, + xdata); +} + +void +ec_wind_removexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_removexattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->removexattr, + &fop->loc[0], fop->str[0], fop->xdata); +} + +void +ec_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + switch (fop->id) { + case GF_FOP_SETXATTR: + if (fop->cbks.setxattr) { + QUORUM_CBK(fop->cbks.setxattr, fop, frame, cookie, this, op_ret, + op_errno, xdata); + } + break; + case GF_FOP_REMOVEXATTR: + if (fop->cbks.removexattr) { + QUORUM_CBK(fop->cbks.removexattr, fop, frame, cookie, this, + op_ret, op_errno, xdata); + } + break; + case GF_FOP_FSETXATTR: + if (fop->cbks.fsetxattr) { + QUORUM_CBK(fop->cbks.fsetxattr, fop, frame, cookie, this, + op_ret, op_errno, xdata); + } + break; + case GF_FOP_FREMOVEXATTR: + if (fop->cbks.fremovexattr) { + QUORUM_CBK(fop->cbks.fremovexattr, fop, frame, cookie, this, + op_ret, op_errno, xdata); + } + break; + } +} + +int32_t +ec_manager_xattr(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + if (fop->fd == NULL) { + ec_lock_prepare_inode(fop, &fop->loc[0], + EC_UPDATE_META | EC_QUERY_INFO, 0, + EC_RANGE_FULL); + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO, + 0, EC_RANGE_FULL); + } + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + ec_fop_prepare_answer(fop, _gf_false); + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + ec_xattr_cbk(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + ec_xattr_cbk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL); + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_removexattr_cbk_t func, void *data, + loc_t *loc, const char *name, dict_t *xdata) +{ + ec_cbk_t callback = {.removexattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(REMOVEXATTR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_REMOVEXATTR, 0, target, + fop_flags, ec_wind_removexattr, ec_manager_xattr, + callback, data); + if (fop == NULL) { + goto out; + } + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (name != NULL) { + fop->str[0] = gf_strdup(name); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: fremovexattr */ + +int32_t +ec_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL, + xdata); +} + +void +ec_wind_fremovexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fremovexattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fremovexattr, + fop->fd, fop->str[0], fop->xdata); +} + +void +ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data, + fd_t *fd, const char *name, dict_t *xdata) +{ + ec_cbk_t callback = {.fremovexattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FREMOVEXATTR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FREMOVEXATTR, 0, target, + fop_flags, ec_wind_fremovexattr, + ec_manager_xattr, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (name != NULL) { + fop->str[0] = gf_strdup(name); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: setattr */ + +int32_t +ec_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); +} + +void +ec_wind_setattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_setattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->setattr, + &fop->loc[0], &fop->iatt, fop->int32, fop->xdata); +} + +int32_t +ec_manager_setattr(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + if (fop->fd == NULL) { + ec_lock_prepare_inode(fop, &fop->loc[0], + EC_UPDATE_META | EC_QUERY_INFO, 0, + EC_RANGE_FULL); + } else { + ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO, + 0, EC_RANGE_FULL); + } + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + if (cbk->iatt[0].ia_type == IA_IFREG) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, + fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->id == GF_FOP_SETATTR) { + if (fop->cbks.setattr != NULL) { + QUORUM_CBK(fop->cbks.setattr, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, + &cbk->iatt[0], &cbk->iatt[1], cbk->xdata); + } + } else { + if (fop->cbks.fsetattr != NULL) { + QUORUM_CBK(fop->cbks.fsetattr, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, + &cbk->iatt[0], &cbk->iatt[1], cbk->xdata); + } + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->id == GF_FOP_SETATTR) { + if (fop->cbks.setattr != NULL) { + fop->cbks.setattr(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL, NULL); + } + } else { + if (fop->cbks.fsetattr != NULL) { + fop->cbks.fsetattr(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL, NULL); + } + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc, + struct iatt *stbuf, int32_t valid, dict_t *xdata) +{ + ec_cbk_t callback = {.setattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(SETATTR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_SETATTR, 0, target, + fop_flags, ec_wind_setattr, ec_manager_setattr, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->int32 = valid; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (stbuf != NULL) { + fop->iatt = *stbuf; + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} + +/* FOP: fsetattr */ + +int32_t +ec_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); +} + +void +ec_wind_fsetattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fsetattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fsetattr, + fop->fd, &fop->iatt, fop->int32, fop->xdata); +} + +void +ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd, + struct iatt *stbuf, int32_t valid, dict_t *xdata) +{ + ec_cbk_t callback = {.fsetattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FSETATTR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETATTR, 0, target, + fop_flags, ec_wind_fsetattr, ec_manager_setattr, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->int32 = valid; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (stbuf != NULL) { + fop->iatt = *stbuf; + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} + +/* FOP: setxattr */ + +int32_t +ec_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL, + xdata); +} + +void +ec_wind_setxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_setxattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->setxattr, + &fop->loc[0], fop->dict, fop->int32, fop->xdata); +} + +void +ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc, + dict_t *dict, int32_t flags, dict_t *xdata) +{ + ec_cbk_t callback = {.setxattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(SETXATTR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_SETXATTR, 0, target, + fop_flags, ec_wind_setxattr, ec_manager_xattr, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->int32 = flags; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (dict != NULL) { + fop->dict = dict_copy_with_ref(dict, NULL); + if (fop->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: fsetxattr */ + +int32_t +ec_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSETXATTR, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_fsetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fsetxattr_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fsetxattr, + fop->fd, fop->dict, fop->int32, fop->xdata); +} + +void +ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd, + dict_t *dict, int32_t flags, dict_t *xdata) +{ + ec_cbk_t callback = {.fsetxattr = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FSETXATTR) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETXATTR, 0, target, + fop_flags, ec_wind_fsetxattr, ec_manager_xattr, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->int32 = flags; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (dict != NULL) { + fop->dict = dict_copy_with_ref(dict, NULL); + if (fop->dict == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/********************************************************************* + * + * File Operation : fallocate + * + *********************************************************************/ + +int32_t +ec_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf, + postbuf, xdata); +} + +void +ec_wind_fallocate(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fallocate_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fallocate, + fop->fd, fop->int32, fop->offset, fop->size, fop->xdata); +} + +int32_t +ec_manager_fallocate(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk = NULL; + + switch (state) { + case EC_STATE_INIT: + if (fop->size == 0) { + ec_fop_set_error(fop, EINVAL); + return EC_STATE_REPORT; + } + if (fop->int32 & + (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE | + FALLOC_FL_ZERO_RANGE | FALLOC_FL_PUNCH_HOLE)) { + ec_fop_set_error(fop, ENOTSUP); + return EC_STATE_REPORT; + } + fop->user_size = fop->offset + fop->size; + fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset, + _gf_true); + fop->size += fop->head; + ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true); + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + fop->offset, fop->size); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + LOCK(&fop->locks[0].lock->loc.inode->lock); + { + GF_ASSERT(__ec_get_inode_size(fop, + fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); + + /*If mode has FALLOC_FL_KEEP_SIZE keep the size */ + if (fop->int32 & FALLOC_FL_KEEP_SIZE) { + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; + } else if (fop->user_size > cbk->iatt[0].ia_size) { + cbk->iatt[1].ia_size = fop->user_size; + + /* This shouldn't fail because we have the inode + * locked. */ + GF_ASSERT(__ec_set_inode_size( + fop, fop->locks[0].lock->loc.inode, + cbk->iatt[1].ia_size)); + } else { + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; + } + } + UNLOCK(&fop->locks[0].lock->loc.inode->lock); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.fallocate != NULL) { + QUORUM_CBK(fop->cbks.fallocate, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.fallocate != NULL) { + fop->cbks.fallocate(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd, + int32_t mode, off_t offset, size_t len, dict_t *xdata) +{ + ec_cbk_t callback = {.fallocate = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FALLOCATE) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FALLOCATE, 0, target, + fop_flags, ec_wind_fallocate, + ec_manager_fallocate, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + fop->int32 = mode; + fop->offset = offset; + fop->size = len; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + goto out; + } + } + + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} + +/********************************************************************* + * + * File Operation : Discard + * + *********************************************************************/ +void +ec_update_discard_write(ec_fop_data_t *fop, uintptr_t mask) +{ + ec_t *ec = fop->xl->private; + off_t off_head = 0; + off_t off_tail = 0; + uint64_t size_head = 0; + uint64_t size_tail = 0; + int error = 0; + + off_head = fop->offset * ec->fragments - fop->int32; + if (fop->size == 0) { + error = ec_update_write(fop, mask, off_head, fop->user_size); + } else { + size_head = fop->int32; + size_tail = (off_head + fop->user_size) % ec->stripe_size; + off_tail = off_head + fop->user_size - size_tail; + if (size_head) { + error = ec_update_write(fop, mask, off_head, size_head); + if (error) { + goto out; + } + } + if (size_tail) { + error = ec_update_write(fop, mask, off_tail, size_tail); + } + } +out: + if (error) + ec_fop_set_error(fop, -error); +} + +void +ec_discard_adjust_offset_size(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + + fop->user_size = fop->size; + /* If discard length covers at least a fragment on brick, we will + * perform discard operation(when fop->size is non-zero) else we just + * write zeros. + */ + fop->int32 = ec_adjust_offset_up(ec, &fop->offset, _gf_true); + fop->frag_range.first = fop->offset; + if (fop->size < fop->int32) { + fop->size = 0; + } else { + fop->size -= fop->int32; + ec_adjust_size_down(ec, &fop->size, _gf_true); + } + fop->frag_range.last = fop->offset + fop->size; +} + +int32_t +ec_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prebuf, + struct iatt *postbuf, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf, + postbuf, xdata); +} + +void +ec_wind_discard(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_discard_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->discard, + fop->fd, fop->offset, fop->size, fop->xdata); +} + +int32_t +ec_manager_discard(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk = NULL; + off_t fl_start = 0; + uint64_t fl_size = 0; + + switch (state) { + case EC_STATE_INIT: + if ((fop->size <= 0) || (fop->offset < 0)) { + ec_fop_set_error(fop, EINVAL); + return EC_STATE_REPORT; + } + /* Because of the head/tail writes, "discard" happens on the + * remaining regions, but we need to compute region including + * head/tail writes so compute them separately*/ + fl_start = fop->offset; + fl_size = fop->size; + fl_size += ec_adjust_offset_down(fop->xl->private, &fl_start, + _gf_true); + ec_adjust_size_up(fop->xl->private, &fl_size, _gf_true); + + ec_discard_adjust_offset_size(fop); + + /* Fall through */ + + case EC_STATE_LOCK: + ec_lock_prepare_fd(fop, fop->fd, + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + fl_start, fl_size); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + + /* Dispatch discard fop only if we have whole fragment + * to deallocate */ + if (fop->size) { + ec_dispatch_all(fop); + return EC_STATE_DELAYED_START; + } else { + /* Assume discard to have succeeded on all bricks */ + ec_succeed_all(fop); + } + + /* Fall through */ + + case EC_STATE_DELAYED_START: + + if (fop->size) { + if (fop->answer && fop->answer->op_ret == 0) + ec_update_discard_write(fop, fop->answer->mask); + } else { + ec_update_discard_write(fop, fop->mask); + } + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); + + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; + } + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.discard != NULL) { + QUORUM_CBK(fop->cbks.discard, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_DELAYED_START: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.discard != NULL) { + fop->cbks.discard(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd, + off_t offset, size_t len, dict_t *xdata) +{ + ec_cbk_t callback = {.discard = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(DISCARD) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_DISCARD, 0, target, + fop_flags, ec_wind_discard, ec_manager_discard, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + fop->offset = offset; + fop->size = len; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + } + + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} + +/********************************************************************* + * + * File Operation : truncate + * + *********************************************************************/ + +int32_t +ec_update_truncate_write(ec_fop_data_t *fop, uintptr_t mask) +{ + ec_t *ec = fop->xl->private; + uint64_t size = fop->offset * ec->fragments - fop->user_size; + return ec_update_write(fop, mask, fop->user_size, size); +} + +int32_t +ec_truncate_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata) +{ + ec_fop_data_t *fop = cookie; + int32_t err; + + fop->parent->good &= fop->good; + if (op_ret >= 0) { + fd_bind(fd); + err = ec_update_truncate_write(fop->parent, fop->answer->mask); + if (err != 0) { + ec_fop_set_error(fop->parent, -err); + } + } + + return 0; +} + +int32_t +ec_truncate_clean(ec_fop_data_t *fop) +{ + if (fop->fd == NULL) { + fop->fd = fd_create(fop->loc[0].inode, fop->frame->root->pid); + if (fop->fd == NULL) { + return -ENOMEM; + } + + ec_open(fop->frame, fop->xl, fop->answer->mask, fop->minimum, + ec_truncate_open_cbk, fop, &fop->loc[0], O_RDWR, fop->fd, NULL); + + return 0; + } else { + return ec_update_truncate_write(fop, fop->answer->mask); + } +} + +int32_t +ec_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); +} + +void +ec_wind_truncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_truncate_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->truncate, + &fop->loc[0], fop->offset, fop->xdata); +} + +int32_t +ec_manager_truncate(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + off_t offset_down; + + switch (state) { + case EC_STATE_INIT: + fop->user_size = fop->offset; + ec_adjust_offset_up(fop->xl->private, &fop->offset, _gf_true); + fop->frag_range.first = fop->offset; + fop->frag_range.last = UINT64_MAX; + + /* Fall through */ + + case EC_STATE_LOCK: + offset_down = fop->user_size; + ec_adjust_offset_down(fop->xl->private, &offset_down, _gf_true); + + if (fop->id == GF_FOP_TRUNCATE) { + ec_lock_prepare_inode( + fop, &fop->loc[0], + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + offset_down, EC_RANGE_FULL); + } else { + ec_lock_prepare_fd( + fop, fop->fd, + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + offset_down, EC_RANGE_FULL); + } + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + int32_t err; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + /* Inode size doesn't need to be updated under locks, because + * conflicting operations won't be in-flight + */ + GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, + &cbk->iatt[0].ia_size)); + cbk->iatt[1].ia_size = fop->user_size; + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_set_inode_size(fop, fop->locks[0].lock->loc.inode, + fop->user_size)); + if ((cbk->iatt[0].ia_size > cbk->iatt[1].ia_size) && + (fop->user_size != fop->offset)) { + err = ec_truncate_clean(fop); + if (err != 0) { + ec_cbk_set_error(cbk, -err, _gf_false); + } + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->id == GF_FOP_TRUNCATE) { + if (fop->cbks.truncate != NULL) { + QUORUM_CBK(fop->cbks.truncate, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, + &cbk->iatt[0], &cbk->iatt[1], cbk->xdata); + } + } else { + if (fop->cbks.ftruncate != NULL) { + QUORUM_CBK(fop->cbks.ftruncate, fop, fop->req_frame, fop, + fop->xl, cbk->op_ret, cbk->op_errno, + &cbk->iatt[0], &cbk->iatt[1], cbk->xdata); + } + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->id == GF_FOP_TRUNCATE) { + if (fop->cbks.truncate != NULL) { + fop->cbks.truncate(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL, NULL); + } + } else { + if (fop->cbks.ftruncate != NULL) { + fop->cbks.ftruncate(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL, NULL, NULL); + } + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc, + off_t offset, dict_t *xdata) +{ + ec_cbk_t callback = {.truncate = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(TRUNCATE) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_TRUNCATE, 0, target, + fop_flags, ec_wind_truncate, ec_manager_truncate, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->offset = offset; + + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} + +/* FOP: ftruncate */ + +int32_t +ec_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iatt *prestat, + struct iatt *poststat, dict_t *xdata) +{ + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); +} + +void +ec_wind_ftruncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_ftruncate_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->ftruncate, + fop->fd, fop->offset, fop->xdata); +} + +void +ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd, + off_t offset, dict_t *xdata) +{ + ec_cbk_t callback = {.ftruncate = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FTRUNCATE) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FTRUNCATE, 0, target, + fop_flags, ec_wind_ftruncate, + ec_manager_truncate, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->offset = offset; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} + +/* FOP: writev */ +static ec_stripe_t * +ec_allocate_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache) +{ + ec_stripe_t *stripe = NULL; + + if (stripe_cache->count >= stripe_cache->max) { + GF_ASSERT(!list_empty(&stripe_cache->lru)); + stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru); + list_move_tail(&stripe->lru, &stripe_cache->lru); + GF_ATOMIC_INC(ec->stats.stripe_cache.evicts); + } else { + stripe = GF_MALLOC(sizeof(ec_stripe_t) + ec->stripe_size, + ec_mt_ec_stripe_t); + if (stripe != NULL) { + stripe_cache->count++; + list_add_tail(&stripe->lru, &stripe_cache->lru); + GF_ATOMIC_INC(ec->stats.stripe_cache.allocs); + } else { + GF_ATOMIC_INC(ec->stats.stripe_cache.errors); + } + } + + return stripe; +} + +static void +ec_write_stripe_data(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe) +{ + off_t base; + + base = fop->size - ec->stripe_size; + memcpy(stripe->data, fop->vector[0].iov_base + base, ec->stripe_size); + stripe->frag_offset = fop->frag_range.last - ec->fragment_size; +} + +static void +ec_add_stripe_in_cache(ec_t *ec, ec_fop_data_t *fop) +{ + ec_inode_t *ctx = NULL; + ec_stripe_t *stripe = NULL; + ec_stripe_list_t *stripe_cache = NULL; + gf_boolean_t failed = _gf_true; + + LOCK(&fop->fd->inode->lock); + + ctx = __ec_inode_get(fop->fd->inode, fop->xl); + if (ctx == NULL) { + goto out; + } + + stripe_cache = &ctx->stripe_cache; + if (stripe_cache->max > 0) { + stripe = ec_allocate_stripe(ec, stripe_cache); + if (stripe == NULL) { + goto out; + } + + ec_write_stripe_data(ec, fop, stripe); + } + + failed = _gf_false; + +out: + UNLOCK(&fop->fd->inode->lock); + + if (failed) { + gf_msg(ec->xl->name, GF_LOG_DEBUG, ENOMEM, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to create and add stripe in cache"); + } +} + +int32_t +ec_writev_merge_tail(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iovec *vector, + int32_t count, struct iatt *stbuf, struct iobref *iobref, + dict_t *xdata) +{ + ec_t *ec = this->private; + ec_fop_data_t *fop = frame->local; + uint64_t size, base, tmp; + + if (op_ret >= 0) { + tmp = 0; + size = fop->size - fop->user_size - fop->head; + base = ec->stripe_size - size; + if (op_ret > base) { + tmp = min(op_ret - base, size); + ec_iov_copy_to(fop->vector[0].iov_base + fop->size - size, vector, + count, base, tmp); + + size -= tmp; + } + + if (size > 0) { + memset(fop->vector[0].iov_base + fop->size - size, 0, size); + } + + if (ec->stripe_cache) { + ec_add_stripe_in_cache(ec, fop); + } + } + return 0; +} + +int32_t +ec_writev_merge_head(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct iovec *vector, + int32_t count, struct iatt *stbuf, struct iobref *iobref, + dict_t *xdata) +{ + ec_t *ec = this->private; + ec_fop_data_t *fop = frame->local; + uint64_t size, base; + + if (op_ret >= 0) { + size = fop->head; + base = 0; + + if (op_ret > 0) { + base = min(op_ret, size); + ec_iov_copy_to(fop->vector[0].iov_base, vector, count, 0, base); + + size -= base; + } + + if (size > 0) { + memset(fop->vector[0].iov_base + base, 0, size); + } + + size = fop->size - fop->user_size - fop->head; + if ((size > 0) && (fop->size == ec->stripe_size)) { + ec_writev_merge_tail(frame, cookie, this, op_ret, op_errno, vector, + count, stbuf, iobref, xdata); + } + } + + return 0; +} + +static int +ec_make_internal_fop_xdata(dict_t **xdata) +{ + dict_t *dict = NULL; + + if (*xdata) + return 0; + + dict = dict_new(); + if (!dict) + goto out; + + if (dict_set_str(dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes")) + goto out; + + *xdata = dict; + return 0; +out: + if (dict) + dict_unref(dict); + return -1; +} + +static int32_t +ec_writev_prepare_buffers(ec_t *ec, ec_fop_data_t *fop) +{ + struct iobref *iobref = NULL; + struct iovec *iov; + void *ptr; + int32_t err; + + fop->user_size = iov_length(fop->vector, fop->int32); + fop->head = ec_adjust_offset_down(ec, &fop->offset, _gf_false); + fop->frag_range.first = fop->offset / ec->fragments; + fop->size = fop->user_size + fop->head; + ec_adjust_size_up(ec, &fop->size, _gf_false); + fop->frag_range.last = fop->frag_range.first + fop->size / ec->fragments; + + if ((fop->int32 != 1) || (fop->head != 0) || (fop->size > fop->user_size) || + !EC_ALIGN_CHECK(fop->vector[0].iov_base, EC_METHOD_WORD_SIZE)) { + err = ec_buffer_alloc(ec->xl, fop->size, &iobref, &ptr); + if (err != 0) { + goto out; + } + + ec_iov_copy_to(ptr + fop->head, fop->vector, fop->int32, 0, + fop->user_size); + + fop->vector[0].iov_base = ptr; + fop->vector[0].iov_len = fop->size; + + iobref_unref(fop->buffers); + fop->buffers = iobref; + } + + if (fop->int32 != 2) { + iov = GF_MALLOC(VECTORSIZE(2), gf_common_mt_iovec); + if (iov == NULL) { + err = -ENOMEM; + + goto out; + } + iov[0].iov_base = fop->vector[0].iov_base; + iov[0].iov_len = fop->vector[0].iov_len; + + GF_FREE(fop->vector); + fop->vector = iov; + } + + fop->vector[1].iov_len = fop->size / ec->fragments; + err = ec_buffer_alloc(ec->xl, fop->vector[1].iov_len * ec->nodes, + &fop->buffers, &fop->vector[1].iov_base); + if (err != 0) { + goto out; + } + + err = 0; + +out: + return err; +} + +static void +ec_merge_stripe_head_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe) +{ + uint32_t head, size; + + head = fop->head; + memcpy(fop->vector[0].iov_base, stripe->data, head); + + size = ec->stripe_size - head; + if (size > fop->user_size) { + head += fop->user_size; + size = ec->stripe_size - head; + memcpy(fop->vector[0].iov_base + head, stripe->data + head, size); + } +} + +static void +ec_merge_stripe_tail_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe) +{ + uint32_t head, tail; + off_t offset; + + offset = fop->user_size + fop->head; + tail = fop->size - offset; + head = ec->stripe_size - tail; + + memcpy(fop->vector[0].iov_base + offset, stripe->data + head, tail); +} + +static ec_stripe_t * +ec_get_stripe_from_cache_locked(ec_t *ec, ec_fop_data_t *fop, + uint64_t frag_offset) +{ + ec_inode_t *ctx = NULL; + ec_stripe_t *stripe = NULL; + ec_stripe_list_t *stripe_cache = NULL; + + ctx = __ec_inode_get(fop->fd->inode, fop->xl); + if (ctx == NULL) { + GF_ATOMIC_INC(ec->stats.stripe_cache.errors); + return NULL; + } + + stripe_cache = &ctx->stripe_cache; + list_for_each_entry(stripe, &stripe_cache->lru, lru) + { + if (stripe->frag_offset == frag_offset) { + list_move_tail(&stripe->lru, &stripe_cache->lru); + GF_ATOMIC_INC(ec->stats.stripe_cache.hits); + return stripe; + } + } + + GF_ATOMIC_INC(ec->stats.stripe_cache.misses); + + return NULL; +} + +static gf_boolean_t +ec_get_and_merge_stripe(ec_t *ec, ec_fop_data_t *fop, ec_stripe_part_t which) +{ + uint64_t frag_offset; + ec_stripe_t *stripe = NULL; + gf_boolean_t found = _gf_false; + + if (!ec->stripe_cache) { + return found; + } + + LOCK(&fop->fd->inode->lock); + if (which == EC_STRIPE_HEAD) { + frag_offset = fop->frag_range.first; + stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset); + if (stripe) { + ec_merge_stripe_head_locked(ec, fop, stripe); + found = _gf_true; + } + } + + if (which == EC_STRIPE_TAIL) { + frag_offset = fop->frag_range.last - ec->fragment_size; + stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset); + if (stripe) { + ec_merge_stripe_tail_locked(ec, fop, stripe); + found = _gf_true; + } + } + UNLOCK(&fop->fd->inode->lock); + + return found; +} + +static uintptr_t +ec_get_lock_good_mask(inode_t *inode, xlator_t *xl) +{ + ec_lock_t *lock = NULL; + ec_inode_t *ictx = NULL; + LOCK(&inode->lock); + { + ictx = __ec_inode_get(inode, xl); + if (ictx) + lock = ictx->inode_lock; + } + UNLOCK(&inode->lock); + if (lock) + return lock->good_mask; + return 0; +} + +void +ec_writev_start(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + ec_fd_t *ctx; + fd_t *fd; + dict_t *xdata = NULL; + uint64_t tail, current; + int32_t err = -ENOMEM; + gf_boolean_t found_stripe = _gf_false; + + /* This shouldn't fail because we have the inode locked. */ + GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, ¤t)); + + fd = fd_anonymous(fop->fd->inode); + if (fd == NULL) { + goto failed; + } + + fop->frame->root->uid = 0; + fop->frame->root->gid = 0; + + ctx = ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + if ((ctx->flags & O_APPEND) != 0) { + /* Appending writes take full locks so size won't change because + * of any parallel operations + */ + fop->offset = current; + } + } + + err = ec_writev_prepare_buffers(ec, fop); + if (err != 0) { + goto failed_fd; + } + tail = fop->size - fop->user_size - fop->head; + if (fop->head > 0) { + if (current > fop->offset) { + found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_HEAD); + if (!found_stripe) { + if (ec_make_internal_fop_xdata(&xdata)) { + err = -ENOMEM; + goto failed_xdata; + } + ec_readv(fop->frame, fop->xl, + ec_get_lock_good_mask(fop->fd->inode, fop->xl), + EC_MINIMUM_MIN, ec_writev_merge_head, NULL, fd, + ec->stripe_size, fop->offset, 0, xdata); + } + } else { + memset(fop->vector[0].iov_base, 0, fop->head); + memset(fop->vector[0].iov_base + fop->size - tail, 0, tail); + if (ec->stripe_cache && (fop->size <= ec->stripe_size)) { + ec_add_stripe_in_cache(ec, fop); + } + } + } + + if ((tail > 0) && ((fop->head == 0) || (fop->size > ec->stripe_size))) { + /* Current locking scheme will make sure the 'current' below will + * never decrease while the fop is in progress, so the checks will + * work as expected + */ + if (current > fop->offset + fop->head + fop->user_size) { + found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_TAIL); + if (!found_stripe) { + if (ec_make_internal_fop_xdata(&xdata)) { + err = -ENOMEM; + goto failed_xdata; + } + ec_readv(fop->frame, fop->xl, + ec_get_lock_good_mask(fop->fd->inode, fop->xl), + EC_MINIMUM_MIN, ec_writev_merge_tail, NULL, fd, + ec->stripe_size, + fop->offset + fop->size - ec->stripe_size, 0, xdata); + } + } else { + memset(fop->vector[0].iov_base + fop->size - tail, 0, tail); + if (ec->stripe_cache) { + ec_add_stripe_in_cache(ec, fop); + } + } + } + + err = 0; + +failed_xdata: + if (xdata) { + dict_unref(xdata); + } +failed_fd: + fd_unref(fd); +failed: + ec_fop_set_error(fop, -err); +} + +int32_t +ec_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct iatt *prestat, struct iatt *poststat, + dict_t *xdata) +{ + ec_t *ec = NULL; + if (this && this->private) { + ec = this->private; + if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) { + op_ret = -1; + op_errno = EIO; + } + } + return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat, + poststat, xdata); +} + +void +ec_wind_writev(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + struct iovec vector[1]; + size_t size; + + size = fop->vector[1].iov_len; + + vector[0].iov_base = fop->vector[1].iov_base + idx * size; + vector[0].iov_len = size; + + STACK_WIND_COOKIE(fop->frame, ec_writev_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->writev, fop->fd, + vector, 1, fop->offset / ec->fragments, fop->uint32, + fop->buffers, fop->xdata); +} + +static void +ec_writev_encode(ec_fop_data_t *fop) +{ + ec_t *ec = fop->xl->private; + void *blocks[ec->nodes]; + uint32_t i; + + blocks[0] = fop->vector[1].iov_base; + for (i = 1; i < ec->nodes; i++) { + blocks[i] = blocks[i - 1] + fop->vector[1].iov_len; + } + ec_method_encode(&ec->matrix, fop->vector[0].iov_len, + fop->vector[0].iov_base, blocks); +} + +int32_t +ec_manager_writev(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + ec_fd_t *ctx = NULL; + ec_t *ec = fop->xl->private; + off_t fl_start = 0; + uint64_t fl_size = LONG_MAX; + + switch (state) { + case EC_STATE_INIT: + case EC_STATE_LOCK: + ctx = ec_fd_get(fop->fd, fop->xl); + if (ctx != NULL) { + if ((ctx->flags & O_APPEND) == 0) { + off_t user_size = 0; + off_t head = 0; + + fl_start = fop->offset; + user_size = iov_length(fop->vector, fop->int32); + head = ec_adjust_offset_down(ec, &fl_start, _gf_true); + fl_size = user_size + head; + ec_adjust_size_up(ec, &fl_size, _gf_true); + } + } + ec_lock_prepare_fd(fop, fop->fd, + EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO, + fl_start, fl_size); + ec_lock(fop); + + return EC_STATE_DISPATCH; + + case EC_STATE_DISPATCH: + ec_writev_start(fop); + + return EC_STATE_DELAYED_START; + + case EC_STATE_DELAYED_START: + /* Restore uid, gid if they were changed to do some partial + * reads. */ + fop->frame->root->uid = fop->uid; + fop->frame->root->gid = fop->gid; + + ec_writev_encode(fop); + + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + cbk = ec_fop_prepare_answer(fop, _gf_false); + if (cbk != NULL) { + ec_t *ec = fop->xl->private; + uint64_t size; + + ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count); + + /* This shouldn't fail because we have the inode locked. */ + LOCK(&fop->fd->inode->lock); + { + GF_ASSERT(__ec_get_inode_size(fop, fop->fd->inode, + &cbk->iatt[0].ia_size)); + cbk->iatt[1].ia_size = cbk->iatt[0].ia_size; + size = fop->offset + fop->head + fop->user_size; + if (size > cbk->iatt[0].ia_size) { + /* Only update inode size if this is a top level fop. + * Otherwise this is an internal write and the top + * level fop should take care of the real inode size. + */ + if (fop->parent == NULL) { + /* This shouldn't fail because we have the inode + * locked. */ + GF_ASSERT( + __ec_set_inode_size(fop, fop->fd->inode, size)); + } + cbk->iatt[1].ia_size = size; + } + } + UNLOCK(&fop->fd->inode->lock); + + if (fop->error == 0) { + cbk->op_ret *= ec->fragments; + if (cbk->op_ret < fop->head) { + cbk->op_ret = 0; + } else { + cbk->op_ret -= fop->head; + } + if (cbk->op_ret > fop->user_size) { + cbk->op_ret = fop->user_size; + } + } + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.writev != NULL) { + QUORUM_CBK(fop->cbks.writev, fop, fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, &cbk->iatt[0], + &cbk->iatt[1], cbk->xdata); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_DELAYED_START: + /* We have failed while doing partial reads. We need to restore + * original uid, gid. */ + fop->frame->root->uid = fop->uid; + fop->frame->root->gid = fop->gid; + + /* Fall through */ + + case -EC_STATE_INIT: + case -EC_STATE_LOCK: + case -EC_STATE_DISPATCH: + case -EC_STATE_PREPARE_ANSWER: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.writev != NULL) { + fop->cbks.writev(fop->req_frame, fop, fop->xl, -1, fop->error, + NULL, NULL, NULL); + } + + return EC_STATE_LOCK_REUSE; + + case -EC_STATE_LOCK_REUSE: + case EC_STATE_LOCK_REUSE: + ec_lock_reuse(fop); + + return EC_STATE_UNLOCK; + + case -EC_STATE_UNLOCK: + case EC_STATE_UNLOCK: + ec_unlock(fop); + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd, + struct iovec *vector, int32_t count, off_t offset, uint32_t flags, + struct iobref *iobref, dict_t *xdata) +{ + ec_cbk_t callback = {.writev = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(WRITE) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_WRITE, 0, target, fop_flags, + ec_wind_writev, ec_manager_writev, callback, + data); + if (fop == NULL) { + goto out; + } + + fop->int32 = count; + fop->offset = offset; + fop->uint32 = flags; + + fop->use_fd = 1; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (count > 0) { + fop->vector = iov_dup(vector, count); + if (fop->vector == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a " + "vector list."); + + goto out; + } + fop->int32 = count; + } + if (iobref != NULL) { + fop->buffers = iobref_ref(iobref); + if (fop->buffers == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL, + "Failed to reference a " + "buffer."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_copy_with_ref(xdata, NULL); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL, NULL); + } +} diff --git a/xlators/cluster/ec/src/ec-locks.c b/xlators/cluster/ec/src/ec-locks.c new file mode 100644 index 00000000000..601960d6154 --- /dev/null +++ b/xlators/cluster/ec/src/ec-locks.c @@ -0,0 +1,1128 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-combine.h" +#include "ec-fops.h" +#include "ec-messages.h" + +#define EC_LOCK_MODE_NONE 0 +#define EC_LOCK_MODE_INC 1 +#define EC_LOCK_MODE_ALL 2 + +int32_t +ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask) +{ + ec_t *ec = fop->xl->private; + ec_cbk_data_t *ans = NULL; + ec_cbk_data_t *cbk = NULL; + uintptr_t locked = 0; + int32_t good = 0; + int32_t eagain = 0; + int32_t estale = 0; + int32_t error = -1; + + /* There are some errors that we'll handle in an special way while trying + * to acquire a lock. + * + * EAGAIN: If it's found during a parallel non-blocking lock request, we + * consider that there's contention on the inode, so we consider + * the acquisition a failure and try again with a sequential + * blocking lock request. This will ensure that we get a lock on + * as many bricks as possible (ignoring EAGAIN here would cause + * unnecessary triggers of self-healing). + * + * If it's found during a sequential blocking lock request, it's + * considered an error. Lock will only succeed if there are + * enough other bricks locked. + * + * ESTALE: This can appear during parallel or sequential lock request if + * the inode has just been unlinked. We consider this error is + * not recoverable, but we also don't consider it as fatal. So, + * if it happens during parallel lock, we won't attempt a + * sequential one unless there are EAGAIN errors on other + * bricks (and are enough to form a quorum), but if we reach + * quorum counting the ESTALE bricks, we consider the whole + * result of the operation is ESTALE instead of EIO. + */ + + list_for_each_entry(ans, &fop->cbk_list, list) + { + if (ans->op_ret >= 0) { + if (locked != 0) { + error = EIO; + } + locked |= ans->mask; + good = ans->count; + cbk = ans; + } else if (ans->op_errno == ESTALE) { + estale += ans->count; + } else if ((ans->op_errno == EAGAIN) && + (fop->uint32 != EC_LOCK_MODE_INC)) { + eagain += ans->count; + } + } + + if (error == -1) { + /* If we have enough quorum with succeeded and EAGAIN answers, we + * ignore for now any ESTALE answer. If there are EAGAIN answers, + * we retry with a sequential blocking lock request if needed. + * Otherwise we succeed. */ + if ((good + eagain) >= ec->fragments) { + if (eagain == 0) { + if (fop->answer == NULL) { + fop->answer = cbk; + } + + ec_update_good(fop, locked); + + error = 0; + } else { + switch (fop->uint32) { + case EC_LOCK_MODE_NONE: + error = EAGAIN; + break; + case EC_LOCK_MODE_ALL: + fop->uint32 = EC_LOCK_MODE_INC; + break; + default: + /* This shouldn't happen because eagain cannot be > 0 + * when fop->uint32 is EC_LOCK_MODE_INC. */ + error = EIO; + break; + } + } + } else { + /* We have been unable to find enough candidates that will be able + * to take the lock. If we have quorum on some answer, we return + * it. Otherwise we check if ESTALE answers allow us to reach + * quorum. If so, we return ESTALE. */ + if (fop->answer && fop->answer->op_ret < 0) { + error = fop->answer->op_errno; + } else if ((good + eagain + estale) >= ec->fragments) { + error = ESTALE; + } else { + error = EIO; + } + } + } + + *mask = locked; + + return error; +} + +int32_t +ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + if (op_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED, + "Failed to unlock an entry/inode"); + } + + return 0; +} + +int32_t +ec_lock_lk_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, struct gf_flock *flock, + dict_t *xdata) +{ + if (op_ret < 0) { + gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_LK_UNLOCK_FAILED, + "Failed to unlock an lk"); + } + + return 0; +} + +/* FOP: entrylk */ + +int32_t +ec_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ENTRYLK, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_entrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_entrylk_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->entrylk, + fop->str[0], &fop->loc[0], fop->str[1], fop->entrylk_cmd, + fop->entrylk_type, fop->xdata); +} + +int32_t +ec_manager_entrylk(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + if (fop->entrylk_cmd == ENTRYLK_LOCK) { + fop->uint32 = EC_LOCK_MODE_ALL; + fop->entrylk_cmd = ENTRYLK_LOCK_NB; + } + + /* Fall through */ + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + case -EC_STATE_PREPARE_ANSWER: + if (fop->entrylk_cmd != ENTRYLK_UNLOCK) { + uintptr_t mask; + + ec_fop_set_error(fop, ec_lock_check(fop, &mask)); + if (fop->error != 0) { + if (mask != 0) { + if (fop->id == GF_FOP_ENTRYLK) { + ec_entrylk( + fop->frame, fop->xl, mask, 1, ec_lock_unlocked, + NULL, fop->str[0], &fop->loc[0], fop->str[1], + ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata); + } else { + ec_fentrylk(fop->frame, fop->xl, mask, 1, + ec_lock_unlocked, NULL, fop->str[0], + fop->fd, fop->str[1], ENTRYLK_UNLOCK, + fop->entrylk_type, fop->xdata); + } + } + if (fop->error < 0) { + fop->error = 0; + + fop->entrylk_cmd = ENTRYLK_LOCK; + + ec_dispatch_inc(fop); + + return EC_STATE_PREPARE_ANSWER; + } + } + } else { + ec_fop_prepare_answer(fop, _gf_true); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->id == GF_FOP_ENTRYLK) { + if (fop->cbks.entrylk != NULL) { + fop->cbks.entrylk(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); + } + } else { + if (fop->cbks.fentrylk != NULL) { + fop->cbks.fentrylk(fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, cbk->xdata); + } + } + + return EC_STATE_END; + + case -EC_STATE_INIT: + case -EC_STATE_DISPATCH: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->id == GF_FOP_ENTRYLK) { + if (fop->cbks.entrylk != NULL) { + fop->cbks.entrylk(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL); + } + } else { + if (fop->cbks.fentrylk != NULL) { + fop->cbks.fentrylk(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL); + } + } + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_entrylk_cbk_t func, void *data, + const char *volume, loc_t *loc, const char *basename, + entrylk_cmd cmd, entrylk_type type, dict_t *xdata) +{ + ec_cbk_t callback = {.entrylk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(ENTRYLK) %p", frame); + + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_ENTRYLK, 0, target, + fop_flags, ec_wind_entrylk, ec_manager_entrylk, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->entrylk_cmd = cmd; + fop->entrylk_type = type; + + if (volume != NULL) { + fop->str[0] = gf_strdup(volume); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (basename != NULL) { + fop->str[1] = gf_strdup(basename); + if (fop->str[1] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: fentrylk */ + +int32_t +ec_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FENTRYLK, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_fentrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_fentrylk_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->fentrylk, + fop->str[0], fop->fd, fop->str[1], fop->entrylk_cmd, + fop->entrylk_type, fop->xdata); +} + +void +ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target, + uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data, + const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd, + entrylk_type type, dict_t *xdata) +{ + ec_cbk_t callback = {.fentrylk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FENTRYLK) %p", frame); + + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FENTRYLK, 0, target, + fop_flags, ec_wind_fentrylk, ec_manager_entrylk, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->entrylk_cmd = cmd; + fop->entrylk_type = type; + + if (volume != NULL) { + fop->str[0] = gf_strdup(volume); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (basename != NULL) { + fop->str[1] = gf_strdup(basename); + if (fop->str[1] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: inodelk */ + +int32_t +ec_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_INODELK, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_inodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_inodelk_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->inodelk, + fop->str[0], &fop->loc[0], fop->int32, &fop->flock, + fop->xdata); +} + +int32_t +ec_manager_inodelk(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + fop->flock.l_len += ec_adjust_offset_down( + fop->xl->private, &fop->flock.l_start, _gf_true); + ec_adjust_offset_up(fop->xl->private, &fop->flock.l_len, _gf_true); + if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) { + fop->uint32 = EC_LOCK_MODE_ALL; + fop->int32 = F_SETLK; + } + + /* Fall through */ + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + case -EC_STATE_PREPARE_ANSWER: + if (fop->flock.l_type != F_UNLCK) { + uintptr_t mask; + + ec_fop_set_error(fop, ec_lock_check(fop, &mask)); + if (fop->error != 0) { + if (mask != 0) { + ec_t *ec = fop->xl->private; + struct gf_flock flock; + + flock.l_type = F_UNLCK; + flock.l_whence = fop->flock.l_whence; + flock.l_start = fop->flock.l_start * ec->fragments; + flock.l_len = fop->flock.l_len * ec->fragments; + flock.l_pid = 0; + flock.l_owner.len = 0; + + if (fop->id == GF_FOP_INODELK) { + ec_inodelk(fop->frame, fop->xl, + &fop->frame->root->lk_owner, mask, 1, + ec_lock_unlocked, NULL, fop->str[0], + &fop->loc[0], F_SETLK, &flock, + fop->xdata); + } else { + ec_finodelk(fop->frame, fop->xl, + &fop->frame->root->lk_owner, mask, 1, + ec_lock_unlocked, NULL, fop->str[0], + fop->fd, F_SETLK, &flock, fop->xdata); + } + } + if (fop->error < 0) { + fop->error = 0; + + fop->int32 = F_SETLKW; + + ec_dispatch_inc(fop); + + return EC_STATE_PREPARE_ANSWER; + } + } + } else { + ec_fop_prepare_answer(fop, _gf_true); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->id == GF_FOP_INODELK) { + if (fop->cbks.inodelk != NULL) { + fop->cbks.inodelk(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, cbk->xdata); + } + } else { + if (fop->cbks.finodelk != NULL) { + fop->cbks.finodelk(fop->req_frame, fop, fop->xl, + cbk->op_ret, cbk->op_errno, cbk->xdata); + } + } + + return EC_STATE_END; + + case -EC_STATE_INIT: + case -EC_STATE_DISPATCH: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->id == GF_FOP_INODELK) { + if (fop->cbks.inodelk != NULL) { + fop->cbks.inodelk(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL); + } + } else { + if (fop->cbks.finodelk != NULL) { + fop->cbks.finodelk(fop->req_frame, fop, fop->xl, -1, + fop->error, NULL); + } + } + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner, + uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func, + void *data, const char *volume, loc_t *loc, int32_t cmd, + struct gf_flock *flock, dict_t *xdata) +{ + ec_cbk_t callback = {.inodelk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(INODELK) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_INODELK, 0, target, + fop_flags, ec_wind_inodelk, ec_manager_inodelk, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->int32 = cmd; + ec_owner_copy(fop->frame, owner); + + if (volume != NULL) { + fop->str[0] = gf_strdup(volume); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (loc != NULL) { + if (loc_copy(&fop->loc[0], loc) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL, + "Failed to copy a location."); + + goto out; + } + } + if (flock != NULL) { + fop->flock.l_type = flock->l_type; + fop->flock.l_whence = flock->l_whence; + fop->flock.l_start = flock->l_start; + fop->flock.l_len = flock->l_len; + fop->flock.l_pid = flock->l_pid; + fop->flock.l_owner.len = flock->l_owner.len; + if (flock->l_owner.len > 0) { + memcpy(fop->flock.l_owner.data, flock->l_owner.data, + flock->l_owner.len); + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: finodelk */ + +int32_t +ec_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, + int32_t op_ret, int32_t op_errno, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FINODELK, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, NULL); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_finodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_finodelk_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->finodelk, + fop->str[0], fop->fd, fop->int32, &fop->flock, + fop->xdata); +} + +void +ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner, + uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func, + void *data, const char *volume, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata) +{ + ec_cbk_t callback = {.finodelk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(FINODELK) %p", frame); + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_FINODELK, 0, target, + fop_flags, ec_wind_finodelk, ec_manager_inodelk, + callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->int32 = cmd; + ec_owner_copy(fop->frame, owner); + + if (volume != NULL) { + fop->str[0] = gf_strdup(volume); + if (fop->str[0] == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to duplicate a string."); + + goto out; + } + } + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (flock != NULL) { + fop->flock.l_type = flock->l_type; + fop->flock.l_whence = flock->l_whence; + fop->flock.l_start = flock->l_start; + fop->flock.l_len = flock->l_len; + fop->flock.l_pid = flock->l_pid; + fop->flock.l_owner.len = flock->l_owner.len; + if (flock->l_owner.len > 0) { + memcpy(fop->flock.l_owner.data, flock->l_owner.data, + flock->l_owner.len); + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL); + } +} + +/* FOP: lk */ + +int32_t +ec_combine_lk(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src) +{ + if (!ec_flock_compare(&dst->flock, &src->flock)) { + gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_LOCK_MISMATCH, + "Mismatching lock in " + "answers of 'GF_FOP_LK'"); + + return 0; + } + + return 1; +} + +int32_t +ec_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret, + int32_t op_errno, struct gf_flock *flock, dict_t *xdata) +{ + ec_fop_data_t *fop = NULL; + ec_cbk_data_t *cbk = NULL; + int32_t idx = (int32_t)(uintptr_t)cookie; + + VALIDATE_OR_GOTO(this, out); + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, frame->local, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = frame->local; + + ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame, + op_ret, op_errno); + + cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LK, idx, op_ret, + op_errno); + if (cbk != NULL) { + if (op_ret >= 0) { + if (flock != NULL) { + cbk->flock.l_type = flock->l_type; + cbk->flock.l_whence = flock->l_whence; + cbk->flock.l_start = flock->l_start; + cbk->flock.l_len = flock->l_len; + cbk->flock.l_pid = flock->l_pid; + cbk->flock.l_owner.len = flock->l_owner.len; + if (flock->l_owner.len > 0) { + memcpy(cbk->flock.l_owner.data, flock->l_owner.data, + flock->l_owner.len); + } + } + } + if (xdata != NULL) { + cbk->xdata = dict_ref(xdata); + if (cbk->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + ec_combine(cbk, ec_combine_lk); + } + +out: + if (fop != NULL) { + ec_complete(fop); + } + + return 0; +} + +void +ec_wind_lk(ec_t *ec, ec_fop_data_t *fop, int32_t idx) +{ + ec_trace("WIND", fop, "idx=%d", idx); + + STACK_WIND_COOKIE(fop->frame, ec_lk_cbk, (void *)(uintptr_t)idx, + ec->xl_list[idx], ec->xl_list[idx]->fops->lk, fop->fd, + fop->int32, &fop->flock, fop->xdata); +} + +int32_t +ec_manager_lk(ec_fop_data_t *fop, int32_t state) +{ + ec_cbk_data_t *cbk; + + switch (state) { + case EC_STATE_INIT: + if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) { + fop->uint32 = EC_LOCK_MODE_ALL; + fop->int32 = F_SETLK; + } + + /* Fall through */ + + case EC_STATE_DISPATCH: + ec_dispatch_all(fop); + + return EC_STATE_PREPARE_ANSWER; + + case EC_STATE_PREPARE_ANSWER: + case -EC_STATE_PREPARE_ANSWER: + if (fop->flock.l_type != F_UNLCK) { + uintptr_t mask; + + ec_fop_set_error(fop, ec_lock_check(fop, &mask)); + if (fop->error != 0) { + if (mask != 0) { + struct gf_flock flock = {0}; + + flock.l_type = F_UNLCK; + flock.l_whence = fop->flock.l_whence; + flock.l_start = fop->flock.l_start; + flock.l_len = fop->flock.l_len; + flock.l_pid = fop->flock.l_pid; + lk_owner_copy(&flock.l_owner, &fop->flock.l_owner); + + ec_lk(fop->frame, fop->xl, mask, 1, ec_lock_lk_unlocked, + NULL, fop->fd, F_SETLK, &flock, fop->xdata); + } + + if (fop->error < 0) { + fop->error = 0; + + fop->int32 = F_SETLKW; + + ec_dispatch_inc(fop); + + return EC_STATE_PREPARE_ANSWER; + } + } + } else { + ec_fop_prepare_answer(fop, _gf_true); + } + + return EC_STATE_REPORT; + + case EC_STATE_REPORT: + cbk = fop->answer; + + GF_ASSERT(cbk != NULL); + + if (fop->cbks.lk != NULL) { + fop->cbks.lk(fop->req_frame, fop, fop->xl, cbk->op_ret, + cbk->op_errno, &cbk->flock, cbk->xdata); + } + + return EC_STATE_END; + + case -EC_STATE_INIT: + case -EC_STATE_DISPATCH: + case -EC_STATE_REPORT: + GF_ASSERT(fop->error != 0); + + if (fop->cbks.lk != NULL) { + fop->cbks.lk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL, + NULL); + } + + return EC_STATE_END; + + default: + gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE, + "Unhandled state %d for %s", state, ec_fop_name(fop->id)); + + return EC_STATE_END; + } +} + +void +ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags, + fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata) +{ + ec_cbk_t callback = {.lk = func}; + ec_fop_data_t *fop = NULL; + int32_t error = ENOMEM; + + gf_msg_trace("ec", 0, "EC(LK) %p", frame); + + GF_VALIDATE_OR_GOTO(this->name, frame, out); + GF_VALIDATE_OR_GOTO(this->name, this->private, out); + + fop = ec_fop_data_allocate(frame, this, GF_FOP_LK, 0, target, fop_flags, + ec_wind_lk, ec_manager_lk, callback, data); + if (fop == NULL) { + goto out; + } + + fop->use_fd = 1; + + fop->int32 = cmd; + + if (fd != NULL) { + fop->fd = fd_ref(fd); + if (fop->fd == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL, + "Failed to reference a " + "file descriptor."); + + goto out; + } + } + if (flock != NULL) { + fop->flock.l_type = flock->l_type; + fop->flock.l_whence = flock->l_whence; + fop->flock.l_start = flock->l_start; + fop->flock.l_len = flock->l_len; + fop->flock.l_pid = flock->l_pid; + fop->flock.l_owner.len = flock->l_owner.len; + if (flock->l_owner.len > 0) { + memcpy(fop->flock.l_owner.data, flock->l_owner.data, + flock->l_owner.len); + } + } + if (xdata != NULL) { + fop->xdata = dict_ref(xdata); + if (fop->xdata == NULL) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL, + "Failed to reference a " + "dictionary."); + + goto out; + } + } + + error = 0; + +out: + if (fop != NULL) { + ec_manager(fop, error); + } else { + func(frame, NULL, this, -1, error, NULL, NULL); + } +} diff --git a/xlators/cluster/ec/src/ec-mem-types.h b/xlators/cluster/ec/src/ec-mem-types.h new file mode 100644 index 00000000000..3252c4c1c58 --- /dev/null +++ b/xlators/cluster/ec/src/ec-mem-types.h @@ -0,0 +1,30 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_MEM_TYPES_H__ +#define __EC_MEM_TYPES_H__ + +#include <glusterfs/mem-types.h> + +enum gf_ec_mem_types_ { + ec_mt_ec_t = gf_common_mt_end + 1, + ec_mt_xlator_t, + ec_mt_ec_inode_t, + ec_mt_ec_fd_t, + ec_mt_subvol_healer_t, + ec_mt_ec_gf_t, + ec_mt_ec_code_t, + ec_mt_ec_code_builder_t, + ec_mt_ec_matrix_t, + ec_mt_ec_stripe_t, + ec_mt_end +}; + +#endif /* __EC_MEM_TYPES_H__ */ diff --git a/xlators/cluster/ec/src/ec-messages.h b/xlators/cluster/ec/src/ec-messages.h new file mode 100644 index 00000000000..72e98f11286 --- /dev/null +++ b/xlators/cluster/ec/src/ec-messages.h @@ -0,0 +1,61 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _EC_MESSAGES_H_ +#define _EC_MESSAGES_H_ + +#include <glusterfs/glfs-message-id.h> + +/* To add new message IDs, append new identifiers at the end of the list. + * + * Never remove a message ID. If it's not used anymore, you can rename it or + * leave it as it is, but not delete it. This is to prevent reutilization of + * IDs by other messages. + * + * The component name must match one of the entries defined in + * glfs-message-id.h. + */ + +GLFS_MSGID(EC, EC_MSG_INVALID_CONFIG, EC_MSG_HEAL_FAIL, + EC_MSG_DICT_COMBINE_FAIL, EC_MSG_STIME_COMBINE_FAIL, + EC_MSG_INVALID_DICT_NUMS, EC_MSG_IATT_COMBINE_FAIL, + EC_MSG_INVALID_FORMAT, EC_MSG_DICT_GET_FAILED, + EC_MSG_UNHANDLED_STATE, EC_MSG_FILE_DESC_REF_FAIL, + EC_MSG_LOC_COPY_FAIL, EC_MSG_BUF_REF_FAIL, EC_MSG_DICT_REF_FAIL, + EC_MSG_LK_UNLOCK_FAILED, EC_MSG_UNLOCK_FAILED, + EC_MSG_LOC_PARENT_INODE_MISSING, EC_MSG_INVALID_LOC_NAME, + EC_MSG_NO_MEMORY, EC_MSG_GFID_MISMATCH, EC_MSG_UNSUPPORTED_VERSION, + EC_MSG_FD_CREATE_FAIL, EC_MSG_READDIRP_REQ_PREP_FAIL, + EC_MSG_LOOKUP_REQ_PREP_FAIL, EC_MSG_INODE_REF_FAIL, + EC_MSG_LOOKUP_READAHEAD_FAIL, EC_MSG_FRAME_MISMATCH, + EC_MSG_XLATOR_MISMATCH, EC_MSG_VECTOR_MISMATCH, EC_MSG_IATT_MISMATCH, + EC_MSG_FD_MISMATCH, EC_MSG_DICT_MISMATCH, EC_MSG_INDEX_DIR_GET_FAIL, + EC_MSG_PREOP_LOCK_FAILED, EC_MSG_CHILDS_INSUFFICIENT, + EC_MSG_OP_EXEC_UNAVAIL, EC_MSG_UNLOCK_DELAY_FAILED, + EC_MSG_SIZE_VERS_UPDATE_FAIL, EC_MSG_INVALID_REQUEST, + EC_MSG_INVALID_LOCK_TYPE, EC_MSG_SIZE_VERS_GET_FAIL, + EC_MSG_FILE_SIZE_GET_FAIL, EC_MSG_FOP_MISMATCH, + EC_MSG_SUBVOL_ID_DICT_SET_FAIL, EC_MSG_SUBVOL_BUILD_FAIL, + EC_MSG_XLATOR_INIT_FAIL, EC_MSG_NO_PARENTS, EC_MSG_TIMER_CREATE_FAIL, + EC_MSG_TOO_MANY_SUBVOLS, EC_MSG_DATA_UNAVAILABLE, + EC_MSG_INODE_REMOVE_FAIL, EC_MSG_INVALID_REDUNDANCY, + EC_MSG_XLATOR_PARSE_OPT_FAIL, EC_MSG_OP_FAIL_ON_SUBVOLS, + EC_MSG_INVALID_INODE, EC_MSG_LOCK_MISMATCH, EC_MSG_XDATA_MISMATCH, + EC_MSG_HEALING_INFO, EC_MSG_HEAL_SUCCESS, EC_MSG_FULL_SWEEP_START, + EC_MSG_FULL_SWEEP_STOP, EC_MSG_INVALID_FOP, EC_MSG_EC_UP, + EC_MSG_EC_DOWN, EC_MSG_SIZE_XATTR_GET_FAIL, + EC_MSG_VER_XATTR_GET_FAIL, EC_MSG_CONFIG_XATTR_GET_FAIL, + EC_MSG_CONFIG_XATTR_INVALID, EC_MSG_EXTENSION, EC_MSG_EXTENSION_NONE, + EC_MSG_EXTENSION_UNKNOWN, EC_MSG_EXTENSION_UNSUPPORTED, + EC_MSG_EXTENSION_FAILED, EC_MSG_NO_GF, EC_MSG_MATRIX_FAILED, + EC_MSG_DYN_CREATE_FAILED, EC_MSG_DYN_CODEGEN_FAILED, + EC_MSG_THREAD_CLEANUP_FAILED, EC_MSG_FD_BAD); + +#endif /* !_EC_MESSAGES_H_ */ diff --git a/xlators/cluster/ec/src/ec-method.c b/xlators/cluster/ec/src/ec-method.c new file mode 100644 index 00000000000..55faed0b193 --- /dev/null +++ b/xlators/cluster/ec/src/ec-method.c @@ -0,0 +1,433 @@ +/* + Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <string.h> +#include <inttypes.h> + +#include "ec-types.h" +#include "ec-mem-types.h" +#include "ec-galois.h" +#include "ec-code.h" +#include "ec-method.h" +#include "ec-helpers.h" + +static void +ec_method_matrix_normal(ec_gf_t *gf, uint32_t *matrix, uint32_t columns, + uint32_t *values, uint32_t count) +{ + uint32_t i, j, v, tmp; + + columns--; + for (i = 0; i < count; i++) { + v = *values++; + *matrix++ = tmp = ec_gf_exp(gf, v, columns); + for (j = 0; j < columns; j++) { + *matrix++ = tmp = ec_gf_div(gf, tmp, v); + } + } +} + +static void +ec_method_matrix_inverse(ec_gf_t *gf, uint32_t *matrix, uint32_t *values, + uint32_t count) +{ + uint32_t a[count]; + uint32_t i, j, p, last, tmp; + + last = count - 1; + for (i = 0; i < last; i++) { + a[i] = 1; + } + a[i] = values[0]; + for (i = last; i > 0; i--) { + for (j = i - 1; j < last; j++) { + a[j] = a[j + 1] ^ ec_gf_mul(gf, values[i], a[j]); + } + a[j] = ec_gf_mul(gf, values[i], a[j]); + } + for (i = 0; i < count; i++) { + p = a[0]; + matrix += count; + *matrix = tmp = p ^ values[i]; + for (j = 1; j < last; j++) { + matrix += count; + *matrix = tmp = a[j] ^ ec_gf_mul(gf, values[i], tmp); + p = tmp ^ ec_gf_mul(gf, values[i], p); + } + for (j = 0; j < last; j++) { + *matrix = ec_gf_div(gf, *matrix, p); + matrix -= count; + } + *matrix = ec_gf_div(gf, 1, p); + matrix++; + } +} + +static void +ec_method_matrix_init(ec_matrix_list_t *list, ec_matrix_t *matrix, + uintptr_t mask, uint32_t *rows, gf_boolean_t inverse) +{ + uint32_t i; + + matrix->refs = 1; + matrix->mask = mask; + matrix->code = list->code; + matrix->columns = list->columns; + INIT_LIST_HEAD(&matrix->lru); + + if (inverse) { + matrix->rows = list->columns; + ec_method_matrix_inverse(matrix->code->gf, matrix->values, rows, + matrix->rows); + for (i = 0; i < matrix->rows; i++) { + matrix->row_data[i].values = matrix->values + i * matrix->columns; + matrix->row_data[i].func.interleaved = ec_code_build_interleaved( + matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values, + matrix->columns); + } + } else { + matrix->rows = list->rows; + ec_method_matrix_normal(matrix->code->gf, matrix->values, + matrix->columns, rows, matrix->rows); + for (i = 0; i < matrix->rows; i++) { + matrix->row_data[i].values = matrix->values + i * matrix->columns; + matrix->row_data[i].func.linear = ec_code_build_linear( + matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values, + matrix->columns); + } + } +} + +static void +ec_method_matrix_release(ec_matrix_t *matrix) +{ + uint32_t i; + + for (i = 0; i < matrix->rows; i++) { + if (matrix->row_data[i].func.linear != NULL) { + ec_code_release(matrix->code, &matrix->row_data[i].func); + matrix->row_data[i].func.linear = NULL; + } + } +} + +static void +ec_method_matrix_destroy(ec_matrix_list_t *list, ec_matrix_t *matrix) +{ + list_del_init(&matrix->lru); + + ec_method_matrix_release(matrix); + + mem_put(matrix); + + list->count--; +} + +static void +ec_method_matrix_unref(ec_matrix_list_t *list, ec_matrix_t *matrix) +{ + if (--matrix->refs == 0) { + list_add_tail(&matrix->lru, &list->lru); + if (list->count > list->max) { + matrix = list_first_entry(&list->lru, ec_matrix_t, lru); + ec_method_matrix_destroy(list, matrix); + } + } +} + +static ec_matrix_t * +ec_method_matrix_lookup(ec_matrix_list_t *list, uintptr_t mask, uint32_t *pos) +{ + ec_matrix_t *matrix; + uint32_t i, j, k; + + i = 0; + j = list->count; + while (i < j) { + k = (i + j) >> 1; + matrix = list->objects[k]; + if (matrix->mask == mask) { + *pos = k; + return matrix; + } + if (matrix->mask < mask) { + i = k + 1; + } else { + j = k; + } + } + *pos = i; + + return NULL; +} + +static void +ec_method_matrix_remove(ec_matrix_list_t *list, uintptr_t mask) +{ + uint32_t pos; + + if (ec_method_matrix_lookup(list, mask, &pos) != NULL) { + list->count--; + if (pos < list->count) { + memmove(list->objects + pos, list->objects + pos + 1, + sizeof(ec_matrix_t *) * (list->count - pos)); + } + } +} + +static void +ec_method_matrix_insert(ec_matrix_list_t *list, ec_matrix_t *matrix) +{ + uint32_t pos; + + GF_ASSERT(ec_method_matrix_lookup(list, matrix->mask, &pos) == NULL); + + if (pos < list->count) { + memmove(list->objects + pos + 1, list->objects + pos, + sizeof(ec_matrix_t *) * (list->count - pos)); + } + list->objects[pos] = matrix; + list->count++; +} + +static ec_matrix_t * +ec_method_matrix_get(ec_matrix_list_t *list, uintptr_t mask, uint32_t *rows) +{ + ec_matrix_t *matrix; + uint32_t pos; + + LOCK(&list->lock); + + matrix = ec_method_matrix_lookup(list, mask, &pos); + if (matrix != NULL) { + list_del_init(&matrix->lru); + matrix->refs++; + + goto out; + } + + if ((list->count >= list->max) && !list_empty(&list->lru)) { + matrix = list_first_entry(&list->lru, ec_matrix_t, lru); + list_del_init(&matrix->lru); + + ec_method_matrix_remove(list, matrix->mask); + + ec_method_matrix_release(matrix); + } else { + matrix = mem_get0(list->pool); + if (matrix == NULL) { + matrix = EC_ERR(ENOMEM); + goto out; + } + matrix->values = (uint32_t *)((uintptr_t)matrix + sizeof(ec_matrix_t) + + sizeof(ec_matrix_row_t) * list->columns); + } + + ec_method_matrix_init(list, matrix, mask, rows, _gf_true); + + if (list->count < list->max) { + ec_method_matrix_insert(list, matrix); + } else { + matrix->mask = 0; + } + +out: + UNLOCK(&list->lock); + + return matrix; +} + +static void +ec_method_matrix_put(ec_matrix_list_t *list, ec_matrix_t *matrix) +{ + LOCK(&list->lock); + + ec_method_matrix_unref(list, matrix); + + UNLOCK(&list->lock); +} + +static int32_t +ec_method_setup(xlator_t *xl, ec_matrix_list_t *list, const char *gen) +{ + ec_matrix_t *matrix; + uint32_t values[list->rows]; + uint32_t i; + int32_t err; + + matrix = GF_MALLOC(sizeof(ec_matrix_t) + + sizeof(ec_matrix_row_t) * list->rows + + sizeof(uint32_t) * list->columns * list->rows, + ec_mt_ec_matrix_t); + if (matrix == NULL) { + err = -ENOMEM; + goto failed; + } + memset(matrix, 0, sizeof(ec_matrix_t)); + matrix->values = (uint32_t *)((uintptr_t)matrix + sizeof(ec_matrix_t) + + sizeof(ec_matrix_row_t) * list->rows); + + list->code = ec_code_create(list->gf, ec_code_detect(xl, gen)); + if (EC_IS_ERR(list->code)) { + err = EC_GET_ERR(list->code); + list->code = NULL; + goto failed_matrix; + } + + for (i = 0; i < list->rows; i++) { + values[i] = i + 1; + } + ec_method_matrix_init(list, matrix, 0, values, _gf_false); + + list->encode = matrix; + + return 0; + +failed_matrix: + GF_FREE(matrix); +failed: + return err; +} + +int32_t +ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns, + uint32_t rows, uint32_t max, const char *gen) +{ + list->columns = columns; + list->rows = rows; + list->max = max; + list->stripe = EC_METHOD_CHUNK_SIZE * list->columns; + INIT_LIST_HEAD(&list->lru); + int32_t err; + + list->pool = mem_pool_new_fn(xl->ctx, + sizeof(ec_matrix_t) + + sizeof(ec_matrix_row_t) * columns + + sizeof(uint32_t) * columns * columns, + 128, "ec_matrix_t"); + if (list->pool == NULL) { + err = -ENOMEM; + goto failed; + } + + list->objects = GF_MALLOC(sizeof(ec_matrix_t *) * max, ec_mt_ec_matrix_t); + if (list->objects == NULL) { + err = -ENOMEM; + goto failed_pool; + } + + list->gf = ec_gf_prepare(EC_GF_BITS, EC_GF_MOD); + if (EC_IS_ERR(list->gf)) { + err = EC_GET_ERR(list->gf); + goto failed_objects; + } + + err = ec_method_setup(xl, list, gen); + if (err != 0) { + goto failed_gf; + } + + LOCK_INIT(&list->lock); + + return 0; + +failed_gf: + ec_gf_destroy(list->gf); +failed_objects: + GF_FREE(list->objects); +failed_pool: + mem_pool_destroy(list->pool); +failed: + list->pool = NULL; + list->objects = NULL; + list->gf = NULL; + + return err; +} + +void +ec_method_fini(ec_matrix_list_t *list) +{ + ec_matrix_t *matrix; + + if (list->encode == NULL) { + return; + } + + while (!list_empty(&list->lru)) { + matrix = list_first_entry(&list->lru, ec_matrix_t, lru); + ec_method_matrix_destroy(list, matrix); + } + + GF_ASSERT(list->count == 0); + + if (list->pool) /*Init was successful*/ + LOCK_DESTROY(&list->lock); + + ec_method_matrix_release(list->encode); + GF_FREE(list->encode); + + ec_code_destroy(list->code); + ec_gf_destroy(list->gf); + GF_FREE(list->objects); + + if (list->pool) + mem_pool_destroy(list->pool); +} + +int32_t +ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen) +{ + /* TODO: Allow changing code generator */ + + return 0; +} + +void +ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out) +{ + ec_matrix_t *matrix; + uint64_t pos; + uint32_t i; + + matrix = list->encode; + for (pos = 0; pos < size; pos += list->stripe) { + for (i = 0; i < matrix->rows; i++) { + matrix->row_data[i].func.linear( + out[i], in, pos, matrix->row_data[i].values, list->columns); + out[i] += EC_METHOD_CHUNK_SIZE; + } + } +} + +int32_t +ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask, + uint32_t *rows, void **in, void *out) +{ + ec_matrix_t *matrix; + uint64_t pos; + uint32_t i; + + matrix = ec_method_matrix_get(list, mask, rows); + if (EC_IS_ERR(matrix)) { + return EC_GET_ERR(matrix); + } + for (pos = 0; pos < size; pos += EC_METHOD_CHUNK_SIZE) { + for (i = 0; i < matrix->rows; i++) { + matrix->row_data[i].func.interleaved( + out, in, pos, matrix->row_data[i].values, list->columns); + out += EC_METHOD_CHUNK_SIZE; + } + } + + ec_method_matrix_put(list, matrix); + + return 0; +} diff --git a/xlators/cluster/ec/src/ec-method.h b/xlators/cluster/ec/src/ec-method.h new file mode 100644 index 00000000000..f91233b2f88 --- /dev/null +++ b/xlators/cluster/ec/src/ec-method.h @@ -0,0 +1,48 @@ +/* + Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_METHOD_H__ +#define __EC_METHOD_H__ + +#include "ec-types.h" +#include "ec-galois.h" + +#define EC_GF_BITS 8 +#define EC_GF_MOD 0x11D + +#define EC_GF_SIZE (1 << EC_GF_BITS) + +/* Determines the maximum size of the matrix used to encode/decode data */ +#define EC_METHOD_MAX_FRAGMENTS 16 +/* Determines the maximum number of usable elements in the Galois Field */ +#define EC_METHOD_MAX_NODES (EC_GF_SIZE - 1) + +#define EC_METHOD_WORD_SIZE 64 + +#define EC_METHOD_CHUNK_SIZE (EC_METHOD_WORD_SIZE * EC_GF_BITS) + +int32_t +ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns, + uint32_t rows, uint32_t max, const char *gen); + +void +ec_method_fini(ec_matrix_list_t *list); + +int32_t +ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen); + +void +ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out); + +int32_t +ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask, + uint32_t *rows, void **in, void *out); + +#endif /* __EC_METHOD_H__ */ diff --git a/xlators/cluster/ec/src/ec-types.h b/xlators/cluster/ec/src/ec-types.h new file mode 100644 index 00000000000..de9b89bb2c9 --- /dev/null +++ b/xlators/cluster/ec/src/ec-types.h @@ -0,0 +1,690 @@ +/* + Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_TYPES_H__ +#define __EC_TYPES_H__ + +#include <glusterfs/xlator.h> +#include <glusterfs/timer.h> +#include "libxlator.h" +#include <glusterfs/atomic.h> + +#define EC_GF_MAX_REGS 16 + +enum _ec_heal_need; +typedef enum _ec_heal_need ec_heal_need_t; + +enum _ec_stripe_part; +typedef enum _ec_stripe_part ec_stripe_part_t; + +enum _ec_read_policy; +typedef enum _ec_read_policy ec_read_policy_t; + +struct _ec_config; +typedef struct _ec_config ec_config_t; + +struct _ec_fd; +typedef struct _ec_fd ec_fd_t; + +struct _ec_fragment_range; +typedef struct _ec_fragment_range ec_fragment_range_t; + +struct _ec_inode; +typedef struct _ec_inode ec_inode_t; + +union _ec_cbk; +typedef union _ec_cbk ec_cbk_t; + +struct _ec_lock; +typedef struct _ec_lock ec_lock_t; + +struct _ec_lock_link; +typedef struct _ec_lock_link ec_lock_link_t; + +struct _ec_fop_data; +typedef struct _ec_fop_data ec_fop_data_t; + +struct _ec_cbk_data; +typedef struct _ec_cbk_data ec_cbk_data_t; + +enum _ec_gf_opcode; +typedef enum _ec_gf_opcode ec_gf_opcode_t; + +struct _ec_gf_op; +typedef struct _ec_gf_op ec_gf_op_t; + +struct _ec_gf_mul; +typedef struct _ec_gf_mul ec_gf_mul_t; + +struct _ec_gf; +typedef struct _ec_gf ec_gf_t; + +struct _ec_code_gen; +typedef struct _ec_code_gen ec_code_gen_t; + +struct _ec_code; +typedef struct _ec_code ec_code_t; + +struct _ec_code_arg; +typedef struct _ec_code_arg ec_code_arg_t; + +struct _ec_code_op; +typedef struct _ec_code_op ec_code_op_t; + +struct _ec_code_builder; +typedef struct _ec_code_builder ec_code_builder_t; + +struct _ec_code_chunk; +typedef struct _ec_code_chunk ec_code_chunk_t; + +struct _ec_stripe; +typedef struct _ec_stripe ec_stripe_t; + +struct _ec_stripe_list; +typedef struct _ec_stripe_list ec_stripe_list_t; + +struct _ec_code_space; +typedef struct _ec_code_space ec_code_space_t; + +typedef void (*ec_code_func_linear_t)(void *dst, void *src, uint64_t offset, + uint32_t *values, uint32_t count); + +typedef void (*ec_code_func_interleaved_t)(void *dst, void **src, + uint64_t offset, uint32_t *values, + uint32_t count); + +union _ec_code_func; +typedef union _ec_code_func ec_code_func_t; + +struct _ec_matrix_row; +typedef struct _ec_matrix_row ec_matrix_row_t; + +struct _ec_matrix; +typedef struct _ec_matrix ec_matrix_t; + +struct _ec_matrix_list; +typedef struct _ec_matrix_list ec_matrix_list_t; + +struct _ec_heal; +typedef struct _ec_heal ec_heal_t; + +struct _ec_self_heald; +typedef struct _ec_self_heald ec_self_heald_t; + +struct _ec_statistics; +typedef struct _ec_statistics ec_statistics_t; + +struct _ec; +typedef struct _ec ec_t; + +typedef void (*ec_wind_f)(ec_t *, ec_fop_data_t *, int32_t); +typedef int32_t (*ec_handler_f)(ec_fop_data_t *, int32_t); +typedef void (*ec_resume_f)(ec_fop_data_t *, int32_t); + +enum _ec_read_policy { EC_ROUND_ROBIN, EC_GFID_HASH, EC_READ_POLICY_MAX }; + +enum _ec_heal_need { + EC_HEAL_NONEED, + EC_HEAL_MAYBE, + EC_HEAL_MUST, + EC_HEAL_PURGE_INDEX +}; + +enum _ec_stripe_part { EC_STRIPE_HEAD, EC_STRIPE_TAIL }; + +/* Enumartions to indicate FD status. */ +typedef enum { EC_FD_NOT_OPENED, EC_FD_OPENED, EC_FD_OPENING } ec_fd_status_t; + +struct _ec_config { + uint32_t version; + uint8_t algorithm; + uint8_t gf_word_size; + uint8_t bricks; + uint8_t redundancy; + uint32_t chunk_size; +}; + +struct _ec_fd { + loc_t loc; + uintptr_t open; + int32_t flags; + uint64_t bad_version; + ec_fd_status_t fd_status[0]; +}; + +struct _ec_stripe { + struct list_head lru; /* LRU list member */ + uint64_t frag_offset; /* Fragment offset of this stripe */ + char data[]; /* Contents of the stripe */ +}; + +struct _ec_stripe_list { + struct list_head lru; + uint32_t count; + uint32_t max; +}; + +struct _ec_inode { + ec_lock_t *inode_lock; + gf_boolean_t have_info; + gf_boolean_t have_config; + gf_boolean_t have_version; + gf_boolean_t have_size; + int32_t heal_count; + ec_config_t config; + uint64_t pre_version[2]; + uint64_t post_version[2]; + uint64_t pre_size; + uint64_t post_size; + uint64_t dirty[2]; + struct list_head heal; + ec_stripe_list_t stripe_cache; + uint64_t bad_version; +}; + +typedef int32_t (*fop_heal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t, + int32_t, uintptr_t, uintptr_t, uintptr_t, + uint32_t, dict_t *); +typedef int32_t (*fop_fheal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t, + int32_t, uintptr_t, uintptr_t, uintptr_t, + uint32_t, dict_t *); + +union _ec_cbk { + fop_access_cbk_t access; + fop_create_cbk_t create; + fop_discard_cbk_t discard; + fop_entrylk_cbk_t entrylk; + fop_fentrylk_cbk_t fentrylk; + fop_fallocate_cbk_t fallocate; + fop_flush_cbk_t flush; + fop_fsync_cbk_t fsync; + fop_fsyncdir_cbk_t fsyncdir; + fop_getxattr_cbk_t getxattr; + fop_fgetxattr_cbk_t fgetxattr; + fop_heal_cbk_t heal; + fop_fheal_cbk_t fheal; + fop_inodelk_cbk_t inodelk; + fop_finodelk_cbk_t finodelk; + fop_link_cbk_t link; + fop_lk_cbk_t lk; + fop_lookup_cbk_t lookup; + fop_mkdir_cbk_t mkdir; + fop_mknod_cbk_t mknod; + fop_open_cbk_t open; + fop_opendir_cbk_t opendir; + fop_readdir_cbk_t readdir; + fop_readdirp_cbk_t readdirp; + fop_readlink_cbk_t readlink; + fop_readv_cbk_t readv; + fop_removexattr_cbk_t removexattr; + fop_fremovexattr_cbk_t fremovexattr; + fop_rename_cbk_t rename; + fop_rmdir_cbk_t rmdir; + fop_setattr_cbk_t setattr; + fop_fsetattr_cbk_t fsetattr; + fop_setxattr_cbk_t setxattr; + fop_fsetxattr_cbk_t fsetxattr; + fop_stat_cbk_t stat; + fop_fstat_cbk_t fstat; + fop_statfs_cbk_t statfs; + fop_symlink_cbk_t symlink; + fop_truncate_cbk_t truncate; + fop_ftruncate_cbk_t ftruncate; + fop_unlink_cbk_t unlink; + fop_writev_cbk_t writev; + fop_xattrop_cbk_t xattrop; + fop_fxattrop_cbk_t fxattrop; + fop_zerofill_cbk_t zerofill; + fop_seek_cbk_t seek; + fop_ipc_cbk_t ipc; +}; + +struct _ec_lock { + ec_inode_t *ctx; + gf_timer_t *timer; + + /* List of owners of this lock. All fops added to this list are running + * concurrently. */ + struct list_head owners; + + /* List of fops waiting to be an owner of the lock. Fops are added to this + * list when the current owner has an incompatible access (conflicting lock) + * or the lock is not acquired yet. */ + struct list_head waiting; + + /* List of fops that will wait until the next unlock/lock cycle. This + * happens when the currently acquired lock is decided to be released as + * soon as possible. In this case, all frozen fops will be continued only + * after the lock is reacquired. */ + struct list_head frozen; + + uintptr_t mask; + uintptr_t good_mask; + uintptr_t healing; + uint32_t refs_owners; /* Refs for fops owning the lock */ + uint32_t refs_pending; /* Refs assigned to fops being prepared */ + uint32_t waiting_flags; /*Track xattrop/dirty marking*/ + gf_boolean_t acquired; + gf_boolean_t contention; + gf_boolean_t unlock_now; + gf_boolean_t release; + gf_boolean_t query; + fd_t *fd; + loc_t loc; + union { + entrylk_type type; + struct gf_flock flock; + }; +}; + +struct _ec_lock_link { + ec_lock_t *lock; + ec_fop_data_t *fop; + struct list_head owner_list; + struct list_head wait_list; + gf_boolean_t update[2]; + gf_boolean_t dirty[2]; + gf_boolean_t optimistic_changelog; + loc_t *base; + uint64_t size; + uint32_t waiting_flags; + off_t fl_start; + off_t fl_end; +}; + +/* This structure keeps a range of fragment offsets affected by a fop. Since + * real file offsets can be difficult to handle correctly because of overflows, + * we use the 'scaled' offset, which corresponds to the offset of the fragment + * seen by the bricks, which is always smaller and cannot overflow. */ +struct _ec_fragment_range { + uint64_t first; /* Address of the first affected fragment as seen by the + bricks (offset on brick) */ + uint64_t last; /* Address of the first non affected fragment as seen by + the bricks (offset on brick) */ +}; + +/* EC xlator data structure to collect all the data required to perform + * the file operation.*/ +struct _ec_fop_data { + int32_t id; /* ID of the file operation */ + int32_t refs; + int32_t state; + uint32_t minimum; /* Minimum number of successful + operation required to conclude a + fop as successful */ + int32_t expected; + int32_t winds; + int32_t jobs; + int32_t error; + ec_fop_data_t *parent; + xlator_t *xl; /* points to EC xlator */ + call_frame_t *req_frame; /* frame of the calling xlator */ + call_frame_t *frame; /* frame used by this fop */ + struct list_head cbk_list; /* sorted list of groups of answers */ + struct list_head answer_list; /* list of answers */ + struct list_head pending_list; /* member of ec_t.pending_fops */ + ec_cbk_data_t *answer; /* accepted answer */ + int32_t lock_count; + int32_t locked; + gf_lock_t lock; + ec_lock_link_t locks[2]; + int32_t first_lock; + + uint32_t fop_flags; /* Flags passed by the caller. */ + uint32_t flags; /* Internal flags. */ + uint32_t first; + uintptr_t mask; + uintptr_t healing; /*Dispatch is done but call is successful only + if fop->minimum number of subvolumes succeed + which are not healing*/ + uintptr_t remaining; + uintptr_t received; /* Mask of responses */ + uintptr_t good; + + uid_t uid; + gid_t gid; + + ec_wind_f wind; /* Function to wind to */ + ec_handler_f handler; /* FOP manager function */ + ec_resume_f resume; + ec_cbk_t cbks; /* Callback function for this FOP */ + void *data; + ec_heal_t *heal; + struct list_head healer; + + uint64_t user_size; + uint32_t head; + + int32_t use_fd; /* Indicates whether this FOP uses FD or + not */ + + dict_t *xdata; + dict_t *dict; + int32_t int32; + uint32_t uint32; + uint64_t size; + off_t offset; + mode_t mode[2]; + entrylk_cmd entrylk_cmd; + entrylk_type entrylk_type; + gf_xattrop_flags_t xattrop_flags; + dev_t dev; + inode_t *inode; + fd_t *fd; /* FD of the file on which FOP is + being carried upon */ + struct iatt iatt; + char *str[2]; + loc_t loc[2]; /* Holds the location details for + the file */ + struct gf_flock flock; + struct iovec *vector; + struct iobref *buffers; + gf_seek_what_t seek; + ec_fragment_range_t frag_range; /* This will hold the range of stripes + affected by the fop. */ + char *errstr; /*String of fop name, path and gfid + to be used in gf_msg. */ +}; + +struct _ec_cbk_data { + struct list_head list; /* item in the sorted list of groups */ + struct list_head answer_list; /* item in the list of answers */ + ec_fop_data_t *fop; + ec_cbk_data_t *next; /* next answer in the same group */ + uint32_t idx; + int32_t op_ret; + int32_t op_errno; + int32_t count; + uintptr_t mask; + + dict_t *xdata; + dict_t *dict; + int32_t int32; + uintptr_t uintptr[3]; + uint64_t size; + uint64_t version[2]; + inode_t *inode; + fd_t *fd; + struct statvfs statvfs; + struct iatt iatt[5]; + struct gf_flock flock; + struct iovec *vector; + struct iobref *buffers; + char *str; + gf_dirent_t entries; + off_t offset; + gf_seek_what_t what; +}; + +enum _ec_gf_opcode { + EC_GF_OP_LOAD, + EC_GF_OP_STORE, + EC_GF_OP_COPY, + EC_GF_OP_XOR2, + EC_GF_OP_XOR3, + EC_GF_OP_XORM, + EC_GF_OP_END +}; + +struct _ec_gf_op { + ec_gf_opcode_t op; + uint32_t arg1; + uint32_t arg2; + uint32_t arg3; +}; + +struct _ec_gf_mul { + uint32_t regs; + uint32_t map[EC_GF_MAX_REGS]; + ec_gf_op_t *ops; +}; + +struct _ec_gf { + uint32_t bits; + uint32_t size; + uint32_t mod; + uint32_t min_ops; + uint32_t max_ops; + uint32_t avg_ops; + uint32_t *log; + uint32_t *pow; + ec_gf_mul_t **table; +}; + +struct _ec_code_gen { + char *name; + char **flags; + uint32_t width; + + void (*prolog)(ec_code_builder_t *builder); + void (*epilog)(ec_code_builder_t *builder); + void (*load)(ec_code_builder_t *builder, uint32_t reg, uint32_t offset, + uint32_t bit); + void (*store)(ec_code_builder_t *builder, uint32_t reg, uint32_t bit); + void (*copy)(ec_code_builder_t *builder, uint32_t dst, uint32_t src); + void (*xor2)(ec_code_builder_t *builder, uint32_t dst, uint32_t src); + void (*xor3)(ec_code_builder_t *builder, uint32_t dst, uint32_t src1, + uint32_t src2); + void (*xorm)(ec_code_builder_t *builder, uint32_t dst, uint32_t offset, + uint32_t bit); +}; + +struct _ec_code { + gf_lock_t lock; + struct list_head spaces; + ec_gf_t *gf; + ec_code_gen_t *gen; +}; + +struct _ec_code_arg { + uint32_t value; +}; + +struct _ec_code_op { + ec_gf_opcode_t op; + ec_code_arg_t arg1; + ec_code_arg_t arg2; + ec_code_arg_t arg3; +}; + +struct _ec_code_builder { + ec_code_t *code; + uint64_t address; + uint8_t *data; + uint32_t size; + int32_t error; + uint32_t regs; + uint32_t bits; + uint32_t width; + uint32_t count; + uint32_t base; + uint32_t map[EC_GF_MAX_REGS]; + gf_boolean_t linear; + uint64_t loop; + ec_code_op_t ops[0]; +}; + +struct _ec_code_chunk { + struct list_head list; + size_t size; + ec_code_space_t *space; +}; + +struct _ec_code_space { + struct list_head list; + struct list_head chunks; + ec_code_t *code; + void *exec; + size_t size; +}; + +union _ec_code_func { + ec_code_func_linear_t linear; + ec_code_func_interleaved_t interleaved; +}; + +struct _ec_matrix_row { + ec_code_func_t func; + uint32_t *values; +}; + +struct _ec_matrix { + struct list_head lru; + uint32_t refs; + uint32_t columns; + uint32_t rows; + uintptr_t mask; + ec_code_t *code; + uint32_t *values; + ec_matrix_row_t row_data[0]; +}; + +struct _ec_matrix_list { + struct list_head lru; + gf_lock_t lock; + uint32_t columns; + uint32_t rows; + uint32_t max; + uint32_t count; + uint32_t stripe; + struct mem_pool *pool; + ec_gf_t *gf; + ec_code_t *code; + ec_matrix_t *encode; + ec_matrix_t **objects; +}; + +struct _ec_heal { + struct list_head list; + gf_lock_t lock; + xlator_t *xl; + ec_fop_data_t *fop; + void *data; + ec_fop_data_t *lookup; + loc_t loc; + struct iatt iatt; + char *symlink; + fd_t *fd; + int32_t partial; + int32_t done; + int32_t error; + gf_boolean_t nameheal; + uintptr_t available; + uintptr_t good; + uintptr_t bad; + uintptr_t open; + uintptr_t fixed; + uint64_t offset; + uint64_t size; + uint64_t total_size; + uint64_t version[2]; + uint64_t raw_size; +}; + +struct subvol_healer { + xlator_t *this; + int subvol; + gf_boolean_t running; + gf_boolean_t rerun; + pthread_mutex_t mutex; + pthread_cond_t cond; + pthread_t thread; +}; + +struct _ec_self_heald { + gf_boolean_t iamshd; + gf_boolean_t enabled; + int timeout; + uint32_t max_threads; + uint32_t wait_qlength; + struct subvol_healer *index_healers; + struct subvol_healer *full_healers; +}; + +struct _ec_statistics { + struct { + gf_atomic_t hits; /* Cache hits. */ + gf_atomic_t misses; /* Cache misses. */ + gf_atomic_t updates; /* Number of times an existing stripe has + been updated with new content. */ + gf_atomic_t invals; /* Number of times an existing stripe has + been invalidated because of truncates + or discards. */ + gf_atomic_t evicts; /* Number of times that an existing entry + has been evicted to make room for newer + entries. */ + gf_atomic_t allocs; /* Number of memory allocations made to + store stripes. */ + gf_atomic_t errors; /* Number of errors that have caused extra + requests. (Basically memory allocation + errors). */ + } stripe_cache; + struct { + gf_atomic_t attempted; /*Number of heals attempted on + files/directories*/ + gf_atomic_t completed; /*Number of heals complted on files/directories*/ + } shd; +}; + +struct _ec { + xlator_t *xl; + int32_t healers; + int32_t heal_waiters; + int32_t nodes; /* Total number of bricks(n) */ + int32_t bits_for_nodes; + int32_t fragments; /* Data bricks(k) */ + int32_t redundancy; /* Redundant bricks(m) */ + uint32_t fragment_size; /* Size of fragment/chunk on a + brick. */ + uint32_t stripe_size; /* (fragment_size * fragments) + maximum size of user data + stored in one stripe. */ + int32_t up; /* Represents whether EC volume is + up or not. */ + uint32_t idx; + uint32_t xl_up_count; /* Number of UP bricks. */ + uintptr_t xl_up; /* Bit flag representing UP + bricks */ + uint32_t xl_notify_count; /* Number of notifications. */ + uintptr_t xl_notify; /* Bit flag representing + notification for bricks. */ + uintptr_t node_mask; + uintptr_t read_mask; /*Stores user defined read-mask*/ + gf_atomic_t async_fop_count; /* Number of on going asynchronous fops. */ + xlator_t **xl_list; + gf_lock_t lock; + gf_timer_t *timer; + gf_boolean_t shutdown; + gf_boolean_t eager_lock; + gf_boolean_t other_eager_lock; + gf_boolean_t optimistic_changelog; + gf_boolean_t parallel_writes; + uint32_t stripe_cache; + uint32_t quorum_count; + uint32_t background_heals; + uint32_t heal_wait_qlen; + uint32_t self_heal_window_size; /* max size of read/writes */ + uint32_t eager_lock_timeout; + uint32_t other_eager_lock_timeout; + struct list_head pending_fops; + struct list_head heal_waiting; + struct list_head healing; + struct mem_pool *fop_pool; + struct mem_pool *cbk_pool; + struct mem_pool *lock_pool; + ec_self_heald_t shd; + char vol_uuid[UUID_SIZE + 1]; + dict_t *leaf_to_subvolid; + ec_read_policy_t read_policy; + ec_matrix_list_t matrix; + ec_statistics_t stats; +}; + +#endif /* __EC_TYPES_H__ */ diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c new file mode 100644 index 00000000000..7344be4968d --- /dev/null +++ b/xlators/cluster/ec/src/ec.c @@ -0,0 +1,1873 @@ +/* + Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#include <glusterfs/defaults.h> +#include <glusterfs/statedump.h> +#include <glusterfs/compat-errno.h> +#include <glusterfs/upcall-utils.h> + +#include "ec.h" +#include "ec-messages.h" +#include "ec-mem-types.h" +#include "ec-types.h" +#include "ec-helpers.h" +#include "ec-common.h" +#include "ec-fops.h" +#include "ec-method.h" +#include "ec-code.h" +#include "ec-heald.h" +#include <glusterfs/events.h> + +static char *ec_read_policies[EC_READ_POLICY_MAX + 1] = { + [EC_ROUND_ROBIN] = "round-robin", + [EC_GFID_HASH] = "gfid-hash", + [EC_READ_POLICY_MAX] = NULL}; + +#define EC_INTERNAL_XATTR_OR_GOTO(name, xattr, op_errno, label) \ + do { \ + if (ec_is_internal_xattr(NULL, (char *)name, NULL, NULL)) { \ + op_errno = EPERM; \ + goto label; \ + } \ + if (name && (strlen(name) == 0) && xattr) { \ + /* Bulk [f]removexattr/[f]setxattr */ \ + GF_IF_INTERNAL_XATTR_GOTO(EC_XATTR_PREFIX "*", xattr, op_errno, \ + label); \ + } \ + } while (0) + +int32_t +ec_parse_options(xlator_t *this) +{ + ec_t *ec = this->private; + int32_t error = EINVAL; + uintptr_t mask; + + GF_OPTION_INIT("redundancy", ec->redundancy, int32, out); + ec->fragments = ec->nodes - ec->redundancy; + if ((ec->redundancy < 1) || (ec->redundancy >= ec->fragments) || + (ec->fragments > EC_MAX_FRAGMENTS)) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REDUNDANCY, + "Invalid redundancy (must be between " + "1 and %d)", + (ec->nodes - 1) / 2); + + goto out; + } + + ec->bits_for_nodes = 1; + mask = 2; + while (ec->nodes > mask) { + ec->bits_for_nodes++; + mask <<= 1; + } + ec->node_mask = (1ULL << ec->nodes) - 1ULL; + ec->fragment_size = EC_METHOD_CHUNK_SIZE; + ec->stripe_size = ec->fragment_size * ec->fragments; + + gf_msg_debug("ec", 0, + "Initialized with: nodes=%u, fragments=%u, " + "stripe_size=%u, node_mask=%" PRIxFAST32, + ec->nodes, ec->fragments, ec->stripe_size, ec->node_mask); + + error = 0; + +out: + return error; +} + +int32_t +ec_prepare_childs(xlator_t *this) +{ + ec_t *ec = this->private; + xlator_list_t *child = NULL; + int32_t count = 0; + + for (child = this->children; child != NULL; child = child->next) { + count++; + } + if (count > EC_MAX_NODES) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_TOO_MANY_SUBVOLS, + "Too many subvolumes"); + + return EINVAL; + } + ec->nodes = count; + + ec->xl_list = GF_CALLOC(count, sizeof(ec->xl_list[0]), ec_mt_xlator_t); + if (ec->xl_list == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Allocation of xlator list failed"); + + return ENOMEM; + } + ec->xl_up = 0; + ec->xl_up_count = 0; + + count = 0; + for (child = this->children; child != NULL; child = child->next) { + ec->xl_list[count++] = child->xlator; + } + + return 0; +} + +/* This function transforms the subvol to subvol-id*/ +static int +_subvol_to_subvolid(dict_t *this, char *key, data_t *value, void *data) +{ + ec_t *ec = data; + xlator_t *subvol = NULL; + int i = 0; + int ret = -1; + + subvol = data_to_ptr(value); + for (i = 0; i < ec->nodes; i++) { + if (ec->xl_list[i] == subvol) { + ret = dict_set_int32(this, key, i); + /* -1 stops dict_foreach and returns -1*/ + if (ret < 0) + ret = -1; + goto out; + } + } +out: + return ret; +} + +int +ec_subvol_to_subvol_id_transform(ec_t *ec, dict_t *leaf_to_subvolid) +{ + return dict_foreach(leaf_to_subvolid, _subvol_to_subvolid, ec); +} + +void +__ec_destroy_private(xlator_t *this) +{ + ec_t *ec = this->private; + + if (ec != NULL) { + LOCK(&ec->lock); + + if (ec->timer != NULL) { + gf_timer_call_cancel(this->ctx, ec->timer); + ec->timer = NULL; + } + + UNLOCK(&ec->lock); + + /* There is a race with timer because there is no way to know if + * timer callback has really been cancelled or it has been scheduled + * for execution. If it has been scheduled, it will crash if we + * destroy ec too fast. + * + * Not sure how this can be solved without using global variables or + * having support from gf_timer_call_cancel() + */ + sleep(2); + + this->private = NULL; + if (ec->xl_list != NULL) { + GF_FREE(ec->xl_list); + ec->xl_list = NULL; + } + + if (ec->fop_pool != NULL) { + mem_pool_destroy(ec->fop_pool); + } + + if (ec->cbk_pool != NULL) { + mem_pool_destroy(ec->cbk_pool); + } + + if (ec->lock_pool != NULL) { + mem_pool_destroy(ec->lock_pool); + } + + LOCK_DESTROY(&ec->lock); + + if (ec->leaf_to_subvolid) + dict_unref(ec->leaf_to_subvolid); + + ec_method_fini(&ec->matrix); + + GF_FREE(ec); + } +} + +int32_t +mem_acct_init(xlator_t *this) +{ + if (xlator_mem_acct_init(this, ec_mt_end + 1) != 0) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Memory accounting initialization " + "failed."); + + return -1; + } + + return 0; +} + +void +ec_configure_background_heal_opts(ec_t *ec, int background_heals, + int heal_wait_qlen) +{ + if (background_heals == 0) { + ec->heal_wait_qlen = 0; + } else { + ec->heal_wait_qlen = heal_wait_qlen; + } + ec->background_heals = background_heals; +} + +int +ec_assign_read_policy(ec_t *ec, char *read_policy) +{ + int read_policy_idx = -1; + + read_policy_idx = gf_get_index_by_elem(ec_read_policies, read_policy); + if (read_policy_idx < 0 || read_policy_idx >= EC_READ_POLICY_MAX) + return -1; + + ec->read_policy = read_policy_idx; + return 0; +} + +int32_t +reconfigure(xlator_t *this, dict_t *options) +{ + ec_t *ec = this->private; + char *read_policy = NULL; + char *extensions = NULL; + uint32_t heal_wait_qlen = 0; + uint32_t background_heals = 0; + int32_t ret = -1; + int32_t err; + + GF_OPTION_RECONF("cpu-extensions", extensions, options, str, failed); + + GF_OPTION_RECONF("self-heal-daemon", ec->shd.enabled, options, bool, + failed); + GF_OPTION_RECONF("iam-self-heal-daemon", ec->shd.iamshd, options, bool, + failed); + GF_OPTION_RECONF("eager-lock", ec->eager_lock, options, bool, failed); + GF_OPTION_RECONF("other-eager-lock", ec->other_eager_lock, options, bool, + failed); + GF_OPTION_RECONF("eager-lock-timeout", ec->eager_lock_timeout, options, + uint32, failed); + GF_OPTION_RECONF("other-eager-lock-timeout", ec->other_eager_lock_timeout, + options, uint32, failed); + GF_OPTION_RECONF("background-heals", background_heals, options, uint32, + failed); + GF_OPTION_RECONF("heal-wait-qlength", heal_wait_qlen, options, uint32, + failed); + GF_OPTION_RECONF("self-heal-window-size", ec->self_heal_window_size, + options, uint32, failed); + GF_OPTION_RECONF("heal-timeout", ec->shd.timeout, options, int32, failed); + ec_configure_background_heal_opts(ec, background_heals, heal_wait_qlen); + GF_OPTION_RECONF("shd-max-threads", ec->shd.max_threads, options, uint32, + failed); + GF_OPTION_RECONF("shd-wait-qlength", ec->shd.wait_qlength, options, uint32, + failed); + + GF_OPTION_RECONF("read-policy", read_policy, options, str, failed); + + GF_OPTION_RECONF("optimistic-change-log", ec->optimistic_changelog, options, + bool, failed); + GF_OPTION_RECONF("parallel-writes", ec->parallel_writes, options, bool, + failed); + GF_OPTION_RECONF("stripe-cache", ec->stripe_cache, options, uint32, failed); + GF_OPTION_RECONF("quorum-count", ec->quorum_count, options, uint32, failed); + ret = 0; + if (ec_assign_read_policy(ec, read_policy)) { + ret = -1; + } + + err = ec_method_update(this, &ec->matrix, extensions); + if (err != 0) { + ret = -1; + } + +failed: + return ret; +} + +glusterfs_event_t +ec_get_event_from_state(ec_t *ec) +{ + int down_count = 0; + + if (ec->xl_up_count >= ec->fragments) { + /* If ec is up but some subvolumes are yet to notify, give + * grace time for other subvols to notify to prevent start of + * I/O which may result in self-heals */ + if (ec->xl_notify_count < ec->nodes) + return GF_EVENT_MAXVAL; + + return GF_EVENT_CHILD_UP; + } else { + down_count = ec->xl_notify_count - ec->xl_up_count; + if (down_count > ec->redundancy) + return GF_EVENT_CHILD_DOWN; + } + + return GF_EVENT_MAXVAL; +} + +void +ec_up(xlator_t *this, ec_t *ec) +{ + char str1[32], str2[32]; + + if (ec->timer != NULL) { + gf_timer_call_cancel(this->ctx, ec->timer); + ec->timer = NULL; + } + + ec->up = 1; + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_UP, + "Going UP : Child UP = %s Child Notify = %s", + ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes)); + + gf_event(EVENT_EC_MIN_BRICKS_UP, "subvol=%s", this->name); +} + +void +ec_down(xlator_t *this, ec_t *ec) +{ + char str1[32], str2[32]; + + if (ec->timer != NULL) { + gf_timer_call_cancel(this->ctx, ec->timer); + ec->timer = NULL; + } + + ec->up = 0; + gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_DOWN, + "Going DOWN : Child UP = %s Child Notify = %s", + ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes), + ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes)); + + gf_event(EVENT_EC_MIN_BRICKS_NOT_UP, "subvol=%s", this->name); +} + +void +ec_notify_cbk(void *data) +{ + ec_t *ec = data; + glusterfs_event_t event = GF_EVENT_MAXVAL; + gf_boolean_t propagate = _gf_false; + gf_boolean_t launch_heal = _gf_false; + + LOCK(&ec->lock); + { + if (!ec->timer) { + /* + * Either child_up/child_down is already sent to parent + * This is a spurious wake up. + */ + goto unlock; + } + + gf_timer_call_cancel(ec->xl->ctx, ec->timer); + ec->timer = NULL; + + /* The timeout has expired, so any subvolume that has not + * already reported its state, will be considered to be down. + * We mark as if all bricks had reported. */ + ec->xl_notify = (1ULL << ec->nodes) - 1ULL; + ec->xl_notify_count = ec->nodes; + + /* Since we have marked all subvolumes as notified, it's + * guaranteed that ec_get_event_from_state() will return + * CHILD_UP or CHILD_DOWN, but not MAXVAL. */ + event = ec_get_event_from_state(ec); + if (event == GF_EVENT_CHILD_UP) { + /* We are ready to bring the volume up. If there are + * still bricks DOWN, they will be healed when they + * come up. */ + ec_up(ec->xl, ec); + + if (ec->shd.iamshd && !ec->shutdown) { + launch_heal = _gf_true; + GF_ATOMIC_INC(ec->async_fop_count); + } + } + + propagate = _gf_true; + } +unlock: + UNLOCK(&ec->lock); + + if (launch_heal) { + /* We have just brought the volume UP, so we trigger + * a self-heal check on the root directory. */ + ec_launch_replace_heal(ec); + } + if (propagate) { + default_notify(ec->xl, event, NULL); + } +} + +void +ec_launch_notify_timer(xlator_t *this, ec_t *ec) +{ + struct timespec delay = { + 0, + }; + + gf_msg_debug(this->name, 0, "Initiating child-down timer"); + delay.tv_sec = 10; + delay.tv_nsec = 0; + ec->timer = gf_timer_call_after(this->ctx, delay, ec_notify_cbk, ec); + if (ec->timer == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_TIMER_CREATE_FAIL, + "Cannot create timer " + "for delayed initialization"); + } +} + +gf_boolean_t +ec_disable_delays(ec_t *ec) +{ + ec->shutdown = _gf_true; + + return __ec_is_last_fop(ec); +} + +void +ec_cleanup_healer_object(ec_t *ec) +{ + struct subvol_healer *healer = NULL; + ec_self_heald_t *shd = NULL; + void *res = NULL; + int i = 0; + gf_boolean_t is_join = _gf_false; + + shd = &ec->shd; + if (!shd->iamshd) + return; + + for (i = 0; i < ec->nodes; i++) { + healer = &shd->index_healers[i]; + pthread_mutex_lock(&healer->mutex); + { + healer->rerun = 1; + if (healer->running) { + pthread_cond_signal(&healer->cond); + is_join = _gf_true; + } + } + pthread_mutex_unlock(&healer->mutex); + if (is_join) { + pthread_join(healer->thread, &res); + is_join = _gf_false; + } + + healer = &shd->full_healers[i]; + pthread_mutex_lock(&healer->mutex); + { + healer->rerun = 1; + if (healer->running) { + pthread_cond_signal(&healer->cond); + is_join = _gf_true; + } + } + pthread_mutex_unlock(&healer->mutex); + if (is_join) { + pthread_join(healer->thread, &res); + is_join = _gf_false; + } + } +} +void +ec_pending_fops_completed(ec_t *ec) +{ + if (ec->shutdown) { + default_notify(ec->xl, GF_EVENT_PARENT_DOWN, NULL); + } +} + +static gf_boolean_t +ec_set_up_state(ec_t *ec, uintptr_t index_mask, uintptr_t new_state) +{ + uintptr_t current_state = 0; + + if (xlator_is_cleanup_starting(ec->xl)) + return _gf_false; + + if ((ec->xl_notify & index_mask) == 0) { + ec->xl_notify |= index_mask; + ec->xl_notify_count++; + } + current_state = ec->xl_up & index_mask; + if (current_state != new_state) { + ec->xl_up ^= index_mask; + ec->xl_up_count += (current_state ? -1 : 1); + + return _gf_true; + } + + return _gf_false; +} + +static gf_boolean_t +ec_upcall(ec_t *ec, struct gf_upcall *upcall) +{ + struct gf_upcall_cache_invalidation *ci = NULL; + struct gf_upcall_inodelk_contention *lc = NULL; + inode_t *inode; + inode_table_t *table; + + switch (upcall->event_type) { + case GF_UPCALL_CACHE_INVALIDATION: + ci = upcall->data; + ci->flags |= UP_INVAL_ATTR; + return _gf_true; + + case GF_UPCALL_INODELK_CONTENTION: + lc = upcall->data; + if (strcmp(lc->domain, ec->xl->name) != 0) { + /* The lock is not owned by EC, ignore it. */ + return _gf_true; + } + table = ((xlator_t *)ec->xl->graph->top)->itable; + if (table == NULL) { + /* Self-heal daemon doesn't have an inode table on the top + * xlator because it doesn't need it. In this case we should + * use the inode table managed by EC itself where all inodes + * being healed should be present. However self-heal doesn't + * use eager-locking and inodelk's are already released as + * soon as possible. In this case we can safely ignore these + * notifications. */ + return _gf_false; + } + inode = inode_find(table, upcall->gfid); + /* If inode is not found, it means that it's already released, + * so we can ignore it. Probably it has been released and + * destroyed while the contention notification was being sent. + */ + if (inode != NULL) { + ec_lock_release(ec, inode); + inode_unref(inode); + } + + return _gf_false; + + default: + return _gf_true; + } +} + +int32_t +ec_notify(xlator_t *this, int32_t event, void *data, void *data2) +{ + ec_t *ec = this->private; + int32_t idx = 0; + int32_t error = 0; + glusterfs_event_t old_event = GF_EVENT_MAXVAL; + dict_t *input = NULL; + dict_t *output = NULL; + gf_boolean_t propagate = _gf_true; + gf_boolean_t needs_shd_check = _gf_false; + int32_t orig_event = event; + uintptr_t mask = 0; + + gf_msg_trace(this->name, 0, "NOTIFY(%d): %p, %p", event, data, data2); + + if (event == GF_EVENT_UPCALL) { + propagate = ec_upcall(ec, data); + goto done; + } + + if (event == GF_EVENT_TRANSLATOR_OP) { + if (!ec->up) { + error = -1; + } else { + input = data; + output = data2; + error = ec_xl_op(this, input, output); + } + goto out; + } + + for (idx = 0; idx < ec->nodes; idx++) { + if (ec->xl_list[idx] == data) { + break; + } + } + + LOCK(&ec->lock); + + if (event == GF_EVENT_PARENT_UP) { + /* + * Start a timer which sends appropriate event to parent + * xlator to prevent the 'mount' syscall from hanging. + */ + ec_launch_notify_timer(this, ec); + goto unlock; + } else if (event == GF_EVENT_PARENT_DOWN) { + /* If there aren't pending fops running after we have waken up + * them, we immediately propagate the notification. */ + propagate = ec_disable_delays(ec); + ec_cleanup_healer_object(ec); + goto unlock; + } + + if (idx < ec->nodes) { /* CHILD_* events */ + old_event = ec_get_event_from_state(ec); + + mask = 1ULL << idx; + if (event == GF_EVENT_CHILD_UP) { + /* We need to trigger a selfheal if a brick changes + * to UP state. */ + if (ec_set_up_state(ec, mask, mask) && ec->shd.iamshd && + !ec->shutdown) { + needs_shd_check = _gf_true; + } + } else if (event == GF_EVENT_CHILD_DOWN) { + ec_set_up_state(ec, mask, 0); + } + + event = ec_get_event_from_state(ec); + + if (event == GF_EVENT_CHILD_UP) { + if (!ec->up) { + ec_up(this, ec); + } + } else { + /* If the volume is not UP, it's irrelevant if one + * brick has come up. We cannot heal anything. */ + needs_shd_check = _gf_false; + + if ((event == GF_EVENT_CHILD_DOWN) && ec->up) { + ec_down(this, ec); + } + } + + if (event != GF_EVENT_MAXVAL) { + if (event == old_event) { + if (orig_event == GF_EVENT_CHILD_UP) + event = GF_EVENT_SOME_DESCENDENT_UP; + else /* orig_event has to be GF_EVENT_CHILD_DOWN */ + event = GF_EVENT_SOME_DESCENDENT_DOWN; + } + } else { + propagate = _gf_false; + needs_shd_check = _gf_false; + } + + if (needs_shd_check) { + GF_ATOMIC_INC(ec->async_fop_count); + } + } +unlock: + UNLOCK(&ec->lock); + +done: + if (needs_shd_check) { + ec_launch_replace_heal(ec); + } + if (propagate) { + error = default_notify(this, event, data); + } + +out: + return error; +} + +int32_t +notify(xlator_t *this, int32_t event, void *data, ...) +{ + int ret = -1; + va_list ap; + void *data2 = NULL; + + va_start(ap, data); + data2 = va_arg(ap, dict_t *); + va_end(ap); + ret = ec_notify(this, event, data, data2); + + return ret; +} + +static void +ec_statistics_init(ec_t *ec) +{ + GF_ATOMIC_INIT(ec->stats.stripe_cache.hits, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.misses, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.updates, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.invals, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.evicts, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.allocs, 0); + GF_ATOMIC_INIT(ec->stats.stripe_cache.errors, 0); + GF_ATOMIC_INIT(ec->stats.shd.attempted, 0); + GF_ATOMIC_INIT(ec->stats.shd.completed, 0); +} + +static int +ec_assign_read_mask(ec_t *ec, char *read_mask_str) +{ + char *mask = NULL; + char *maskptr = NULL; + char *saveptr = NULL; + char *id_str = NULL; + int id = 0; + int ret = 0; + uintptr_t read_mask = 0; + + if (!read_mask_str) { + ec->read_mask = 0; + ret = 0; + goto out; + } + + mask = gf_strdup(read_mask_str); + if (!mask) { + ret = -1; + goto out; + } + maskptr = mask; + + for (;;) { + id_str = strtok_r(maskptr, ":", &saveptr); + if (id_str == NULL) + break; + if (gf_string2int(id_str, &id)) { + gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL, + "In read-mask \"%s\" id %s is not a valid integer", + read_mask_str, id_str); + ret = -1; + goto out; + } + + if ((id < 0) || (id >= ec->nodes)) { + gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL, + "In read-mask \"%s\" id %d is not in range [0 - %d]", + read_mask_str, id, ec->nodes - 1); + ret = -1; + goto out; + } + read_mask |= (1UL << id); + maskptr = NULL; + } + + if (gf_bits_count(read_mask) < ec->fragments) { + gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL, + "read-mask \"%s\" should contain at least %d ids", read_mask_str, + ec->fragments); + ret = -1; + goto out; + } + ec->read_mask = read_mask; + ret = 0; +out: + GF_FREE(mask); + return ret; +} + +int32_t +init(xlator_t *this) +{ + ec_t *ec = NULL; + char *read_policy = NULL; + char *extensions = NULL; + int32_t err; + char *read_mask_str = NULL; + + if (this->parents == NULL) { + gf_msg(this->name, GF_LOG_WARNING, 0, EC_MSG_NO_PARENTS, + "Volume does not have parents."); + } + + ec = GF_MALLOC(sizeof(*ec), ec_mt_ec_t); + if (ec == NULL) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to allocate private memory."); + + return -1; + } + memset(ec, 0, sizeof(*ec)); + + this->private = ec; + + ec->xl = this; + LOCK_INIT(&ec->lock); + + GF_ATOMIC_INIT(ec->async_fop_count, 0); + INIT_LIST_HEAD(&ec->pending_fops); + INIT_LIST_HEAD(&ec->heal_waiting); + INIT_LIST_HEAD(&ec->healing); + + ec->fop_pool = mem_pool_new(ec_fop_data_t, 1024); + ec->cbk_pool = mem_pool_new(ec_cbk_data_t, 4096); + ec->lock_pool = mem_pool_new(ec_lock_t, 1024); + if ((ec->fop_pool == NULL) || (ec->cbk_pool == NULL) || + (ec->lock_pool == NULL)) { + gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY, + "Failed to create memory pools."); + + goto failed; + } + + if (ec_prepare_childs(this) != 0) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL, + "Failed to initialize xlator"); + + goto failed; + } + + if (ec_parse_options(this) != 0) { + gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_PARSE_OPT_FAIL, + "Failed to parse xlator options"); + + goto failed; + } + + GF_OPTION_INIT("cpu-extensions", extensions, str, failed); + + err = ec_method_init(this, &ec->matrix, ec->fragments, ec->nodes, + ec->nodes * 2, extensions); + if (err != 0) { + gf_msg(this->name, GF_LOG_ERROR, -err, EC_MSG_MATRIX_FAILED, + "Failed to initialize matrix management"); + + goto failed; + } + + GF_OPTION_INIT("self-heal-daemon", ec->shd.enabled, bool, failed); + GF_OPTION_INIT("iam-self-heal-daemon", ec->shd.iamshd, bool, failed); + GF_OPTION_INIT("eager-lock", ec->eager_lock, bool, failed); + GF_OPTION_INIT("other-eager-lock", ec->other_eager_lock, bool, failed); + GF_OPTION_INIT("eager-lock-timeout", ec->eager_lock_timeout, uint32, + failed); + GF_OPTION_INIT("other-eager-lock-timeout", ec->other_eager_lock_timeout, + uint32, failed); + GF_OPTION_INIT("background-heals", ec->background_heals, uint32, failed); + GF_OPTION_INIT("heal-wait-qlength", ec->heal_wait_qlen, uint32, failed); + GF_OPTION_INIT("self-heal-window-size", ec->self_heal_window_size, uint32, + failed); + ec_configure_background_heal_opts(ec, ec->background_heals, + ec->heal_wait_qlen); + GF_OPTION_INIT("read-policy", read_policy, str, failed); + if (ec_assign_read_policy(ec, read_policy)) + goto failed; + + GF_OPTION_INIT("heal-timeout", ec->shd.timeout, int32, failed); + GF_OPTION_INIT("shd-max-threads", ec->shd.max_threads, uint32, failed); + GF_OPTION_INIT("shd-wait-qlength", ec->shd.wait_qlength, uint32, failed); + GF_OPTION_INIT("optimistic-change-log", ec->optimistic_changelog, bool, + failed); + GF_OPTION_INIT("parallel-writes", ec->parallel_writes, bool, failed); + GF_OPTION_INIT("stripe-cache", ec->stripe_cache, uint32, failed); + GF_OPTION_INIT("quorum-count", ec->quorum_count, uint32, failed); + GF_OPTION_INIT("ec-read-mask", read_mask_str, str, failed); + + if (ec_assign_read_mask(ec, read_mask_str)) + goto failed; + + this->itable = inode_table_new(EC_SHD_INODE_LRU_LIMIT, this); + if (!this->itable) + goto failed; + + if (ec->shd.iamshd) + ec_selfheal_daemon_init(this); + gf_msg_debug(this->name, 0, "Disperse translator initialized."); + + ec->leaf_to_subvolid = dict_new(); + if (!ec->leaf_to_subvolid) + goto failed; + if (glusterfs_reachable_leaves(this, ec->leaf_to_subvolid)) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_BUILD_FAIL, + "Failed to build subvol " + "dictionary"); + goto failed; + } + + if (ec_subvol_to_subvol_id_transform(ec, ec->leaf_to_subvolid) < 0) { + gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_ID_DICT_SET_FAIL, + "Failed to build subvol-id " + "dictionary"); + goto failed; + } + + ec_statistics_init(ec); + + return 0; + +failed: + __ec_destroy_private(this); + + return -1; +} + +void +fini(xlator_t *this) +{ + ec_selfheal_daemon_fini(this); + __ec_destroy_private(this); +} + +int32_t +ec_gf_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask, + dict_t *xdata) +{ + ec_access(frame, this, -1, EC_MINIMUM_ONE, default_access_cbk, NULL, loc, + mask, xdata); + + return 0; +} + +int32_t +ec_gf_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, + mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata) +{ + ec_create(frame, this, -1, EC_MINIMUM_MIN, default_create_cbk, NULL, loc, + flags, mode, umask, fd, xdata); + + return 0; +} + +int32_t +ec_gf_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, + size_t len, dict_t *xdata) +{ + ec_discard(frame, this, -1, EC_MINIMUM_MIN, default_discard_cbk, NULL, fd, + offset, len, xdata); + + return 0; +} + +int32_t +ec_gf_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, + loc_t *loc, const char *basename, entrylk_cmd cmd, + entrylk_type type, dict_t *xdata) +{ + uint32_t fop_flags = EC_MINIMUM_ALL; + + if (cmd == ENTRYLK_UNLOCK) + fop_flags = EC_MINIMUM_ONE; + ec_entrylk(frame, this, -1, fop_flags, default_entrylk_cbk, NULL, volume, + loc, basename, cmd, type, xdata); + + return 0; +} + +int32_t +ec_gf_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, + fd_t *fd, const char *basename, entrylk_cmd cmd, + entrylk_type type, dict_t *xdata) +{ + uint32_t fop_flags = EC_MINIMUM_ALL; + + if (cmd == ENTRYLK_UNLOCK) + fop_flags = EC_MINIMUM_ONE; + ec_fentrylk(frame, this, -1, fop_flags, default_fentrylk_cbk, NULL, volume, + fd, basename, cmd, type, xdata); + + return 0; +} + +int32_t +ec_gf_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode, + off_t offset, size_t len, dict_t *xdata) +{ + ec_fallocate(frame, this, -1, EC_MINIMUM_MIN, default_fallocate_cbk, NULL, + fd, mode, offset, len, xdata); + + return 0; +} + +int32_t +ec_gf_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) +{ + ec_flush(frame, this, -1, EC_MINIMUM_MIN, default_flush_cbk, NULL, fd, + xdata); + + return 0; +} + +int32_t +ec_gf_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync, + dict_t *xdata) +{ + ec_fsync(frame, this, -1, EC_MINIMUM_MIN, default_fsync_cbk, NULL, fd, + datasync, xdata); + + return 0; +} + +int32_t +ec_gf_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync, + dict_t *xdata) +{ + ec_fsyncdir(frame, this, -1, EC_MINIMUM_MIN, default_fsyncdir_cbk, NULL, fd, + datasync, xdata); + + return 0; +} + +int +ec_marker_populate_args(call_frame_t *frame, int type, int *gauge, + xlator_t **subvols) +{ + xlator_t *this = frame->this; + ec_t *ec = this->private; + + memcpy(subvols, ec->xl_list, sizeof(*subvols) * ec->nodes); + + if (type == MARKER_XTIME_TYPE) { + /*Don't error out on ENOENT/ENOTCONN */ + gauge[MCNT_NOTFOUND] = 0; + gauge[MCNT_ENOTCONN] = 0; + } + + return ec->nodes; +} + +int32_t +ec_handle_heal_commands(call_frame_t *frame, xlator_t *this, loc_t *loc, + const char *name, dict_t *xdata) +{ + dict_t *dict_rsp = NULL; + int op_ret = -1; + int op_errno = ENOMEM; + + if (!name || strcmp(name, GF_HEAL_INFO)) + return -1; + + op_errno = -ec_get_heal_info(this, loc, &dict_rsp); + if (op_errno <= 0) { + op_errno = op_ret = 0; + } + + STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict_rsp, NULL); + if (dict_rsp) + dict_unref(dict_rsp); + return 0; +} + +int32_t +ec_gf_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, + const char *name, dict_t *xdata) +{ + int error = 0; + ec_t *ec = this->private; + int32_t fop_flags = EC_MINIMUM_ONE; + + if (name && strcmp(name, EC_XATTR_HEAL) != 0) { + EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out); + } + + if (ec_handle_heal_commands(frame, this, loc, name, xdata) == 0) + return 0; + + if (cluster_handle_marker_getxattr(frame, loc, name, ec->vol_uuid, NULL, + ec_marker_populate_args) == 0) + return 0; + + if (name && ((fnmatch(GF_XATTR_STIME_PATTERN, name, 0) == 0) || + XATTR_IS_NODE_UUID(name) || XATTR_IS_NODE_UUID_LIST(name))) { + fop_flags = EC_MINIMUM_ALL; + } + + ec_getxattr(frame, this, -1, fop_flags, default_getxattr_cbk, NULL, loc, + name, xdata); + + return 0; +out: + error = ENODATA; + STACK_UNWIND_STRICT(getxattr, frame, -1, error, NULL, NULL); + return 0; +} + +int32_t +ec_gf_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name, + dict_t *xdata) +{ + int error = 0; + + EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out); + + ec_fgetxattr(frame, this, -1, EC_MINIMUM_ONE, default_fgetxattr_cbk, NULL, + fd, name, xdata); + return 0; +out: + error = ENODATA; + STACK_UNWIND_STRICT(fgetxattr, frame, -1, error, NULL, NULL); + return 0; +} + +int32_t +ec_gf_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, + loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata) +{ + int32_t fop_flags = EC_MINIMUM_ALL; + + if (flock->l_type == F_UNLCK) + fop_flags = EC_MINIMUM_ONE; + + ec_inodelk(frame, this, &frame->root->lk_owner, -1, fop_flags, + default_inodelk_cbk, NULL, volume, loc, cmd, flock, xdata); + + return 0; +} + +int32_t +ec_gf_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, + fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata) +{ + int32_t fop_flags = EC_MINIMUM_ALL; + + if (flock->l_type == F_UNLCK) + fop_flags = EC_MINIMUM_ONE; + ec_finodelk(frame, this, &frame->root->lk_owner, -1, fop_flags, + default_finodelk_cbk, NULL, volume, fd, cmd, flock, xdata); + + return 0; +} + +int32_t +ec_gf_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, + dict_t *xdata) +{ + ec_link(frame, this, -1, EC_MINIMUM_MIN, default_link_cbk, NULL, oldloc, + newloc, xdata); + + return 0; +} + +int32_t +ec_gf_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd, + struct gf_flock *flock, dict_t *xdata) +{ + int32_t fop_flags = EC_MINIMUM_ALL; + + if (flock->l_type == F_UNLCK) + fop_flags = EC_MINIMUM_ONE; + ec_lk(frame, this, -1, fop_flags, default_lk_cbk, NULL, fd, cmd, flock, + xdata); + + return 0; +} + +int32_t +ec_gf_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) +{ + ec_lookup(frame, this, -1, EC_MINIMUM_MIN, default_lookup_cbk, NULL, loc, + xdata); + + return 0; +} + +int32_t +ec_gf_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, + mode_t umask, dict_t *xdata) +{ + ec_mkdir(frame, this, -1, EC_MINIMUM_MIN, default_mkdir_cbk, NULL, loc, + mode, umask, xdata); + + return 0; +} + +int32_t +ec_gf_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode, + dev_t rdev, mode_t umask, dict_t *xdata) +{ + ec_mknod(frame, this, -1, EC_MINIMUM_MIN, default_mknod_cbk, NULL, loc, + mode, rdev, umask, xdata); + + return 0; +} + +int32_t +ec_gf_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags, + fd_t *fd, dict_t *xdata) +{ + ec_open(frame, this, -1, EC_MINIMUM_MIN, default_open_cbk, NULL, loc, flags, + fd, xdata); + + return 0; +} + +int32_t +ec_gf_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, + dict_t *xdata) +{ + ec_opendir(frame, this, -1, EC_MINIMUM_MIN, default_opendir_cbk, NULL, loc, + fd, xdata); + + return 0; +} + +int32_t +ec_gf_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t offset, dict_t *xdata) +{ + ec_readdir(frame, this, -1, EC_MINIMUM_ONE, default_readdir_cbk, NULL, fd, + size, offset, xdata); + + return 0; +} + +int32_t +ec_gf_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t offset, dict_t *xdata) +{ + ec_readdirp(frame, this, -1, EC_MINIMUM_ONE, default_readdirp_cbk, NULL, fd, + size, offset, xdata); + + return 0; +} + +int32_t +ec_gf_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size, + dict_t *xdata) +{ + ec_readlink(frame, this, -1, EC_MINIMUM_ONE, default_readlink_cbk, NULL, + loc, size, xdata); + + return 0; +} + +int32_t +ec_gf_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, + off_t offset, uint32_t flags, dict_t *xdata) +{ + ec_readv(frame, this, -1, EC_MINIMUM_MIN, default_readv_cbk, NULL, fd, size, + offset, flags, xdata); + + return 0; +} + +int32_t +ec_gf_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc, + const char *name, dict_t *xdata) +{ + int error = 0; + + EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out); + + ec_removexattr(frame, this, -1, EC_MINIMUM_MIN, default_removexattr_cbk, + NULL, loc, name, xdata); + + return 0; +out: + STACK_UNWIND_STRICT(removexattr, frame, -1, error, NULL); + return 0; +} + +int32_t +ec_gf_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, + const char *name, dict_t *xdata) +{ + int error = 0; + + EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out); + + ec_fremovexattr(frame, this, -1, EC_MINIMUM_MIN, default_fremovexattr_cbk, + NULL, fd, name, xdata); + + return 0; +out: + STACK_UNWIND_STRICT(fremovexattr, frame, -1, error, NULL); + return 0; +} + +int32_t +ec_gf_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, + dict_t *xdata) +{ + ec_rename(frame, this, -1, EC_MINIMUM_MIN, default_rename_cbk, NULL, oldloc, + newloc, xdata); + + return 0; +} + +int32_t +ec_gf_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags, + dict_t *xdata) +{ + ec_rmdir(frame, this, -1, EC_MINIMUM_MIN, default_rmdir_cbk, NULL, loc, + xflags, xdata); + + return 0; +} + +int32_t +ec_gf_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, + struct iatt *stbuf, int32_t valid, dict_t *xdata) +{ + ec_setattr(frame, this, -1, EC_MINIMUM_MIN, default_setattr_cbk, NULL, loc, + stbuf, valid, xdata); + + return 0; +} + +int32_t +ec_gf_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, + struct iatt *stbuf, int32_t valid, dict_t *xdata) +{ + ec_fsetattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetattr_cbk, NULL, fd, + stbuf, valid, xdata); + + return 0; +} + +int32_t +ec_gf_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict, + int32_t flags, dict_t *xdata) +{ + int error = 0; + + EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out); + + ec_setxattr(frame, this, -1, EC_MINIMUM_MIN, default_setxattr_cbk, NULL, + loc, dict, flags, xdata); + + return 0; +out: + STACK_UNWIND_STRICT(setxattr, frame, -1, error, NULL); + return 0; +} + +int32_t +ec_gf_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict, + int32_t flags, dict_t *xdata) +{ + int error = 0; + + EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out); + + ec_fsetxattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetxattr_cbk, NULL, + fd, dict, flags, xdata); + + return 0; +out: + STACK_UNWIND_STRICT(fsetxattr, frame, -1, error, NULL); + return 0; +} + +int32_t +ec_gf_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) +{ + ec_stat(frame, this, -1, EC_MINIMUM_MIN, default_stat_cbk, NULL, loc, + xdata); + + return 0; +} + +int32_t +ec_gf_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata) +{ + ec_fstat(frame, this, -1, EC_MINIMUM_MIN, default_fstat_cbk, NULL, fd, + xdata); + + return 0; +} + +int32_t +ec_gf_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata) +{ + ec_statfs(frame, this, -1, EC_MINIMUM_MIN, default_statfs_cbk, NULL, loc, + xdata); + + return 0; +} + +int32_t +ec_gf_symlink(call_frame_t *frame, xlator_t *this, const char *linkname, + loc_t *loc, mode_t umask, dict_t *xdata) +{ + ec_symlink(frame, this, -1, EC_MINIMUM_MIN, default_symlink_cbk, NULL, + linkname, loc, umask, xdata); + + return 0; +} + +int32_t +ec_gf_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, + dict_t *xdata) +{ + ec_truncate(frame, this, -1, EC_MINIMUM_MIN, default_truncate_cbk, NULL, + loc, offset, xdata); + + return 0; +} + +int32_t +ec_gf_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, + dict_t *xdata) +{ + ec_ftruncate(frame, this, -1, EC_MINIMUM_MIN, default_ftruncate_cbk, NULL, + fd, offset, xdata); + + return 0; +} + +int32_t +ec_gf_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags, + dict_t *xdata) +{ + ec_unlink(frame, this, -1, EC_MINIMUM_MIN, default_unlink_cbk, NULL, loc, + xflags, xdata); + + return 0; +} + +int32_t +ec_gf_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, + struct iovec *vector, int32_t count, off_t offset, uint32_t flags, + struct iobref *iobref, dict_t *xdata) +{ + ec_writev(frame, this, -1, EC_MINIMUM_MIN, default_writev_cbk, NULL, fd, + vector, count, offset, flags, iobref, xdata); + + return 0; +} + +int32_t +ec_gf_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata) +{ + ec_xattrop(frame, this, -1, EC_MINIMUM_MIN, default_xattrop_cbk, NULL, loc, + optype, xattr, xdata); + + return 0; +} + +int32_t +ec_gf_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd, + gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata) +{ + ec_fxattrop(frame, this, -1, EC_MINIMUM_MIN, default_fxattrop_cbk, NULL, fd, + optype, xattr, xdata); + + return 0; +} + +int32_t +ec_gf_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, + off_t len, dict_t *xdata) +{ + default_zerofill_failure_cbk(frame, ENOTSUP); + + return 0; +} + +int32_t +ec_gf_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, + gf_seek_what_t what, dict_t *xdata) +{ + ec_seek(frame, this, -1, EC_MINIMUM_ONE, default_seek_cbk, NULL, fd, offset, + what, xdata); + + return 0; +} + +int32_t +ec_gf_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata) +{ + ec_ipc(frame, this, -1, EC_MINIMUM_MIN, default_ipc_cbk, NULL, op, xdata); + return 0; +} + +int32_t +ec_gf_forget(xlator_t *this, inode_t *inode) +{ + uint64_t value = 0; + ec_inode_t *ctx = NULL; + + if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0)) { + ctx = (ec_inode_t *)(uintptr_t)value; + /* We can only forget an inode if it has been unlocked, so the stripe + * cache should also be empty. */ + GF_ASSERT(list_empty(&ctx->stripe_cache.lru)); + GF_FREE(ctx); + } + + return 0; +} + +void +ec_gf_release_fd(xlator_t *this, fd_t *fd) +{ + uint64_t value = 0; + ec_fd_t *ctx = NULL; + + if ((fd_ctx_del(fd, this, &value) == 0) && (value != 0)) { + ctx = (ec_fd_t *)(uintptr_t)value; + loc_wipe(&ctx->loc); + GF_FREE(ctx); + } +} + +int32_t +ec_gf_release(xlator_t *this, fd_t *fd) +{ + ec_gf_release_fd(this, fd); + + return 0; +} + +int32_t +ec_gf_releasedir(xlator_t *this, fd_t *fd) +{ + ec_gf_release_fd(this, fd); + + return 0; +} + +int32_t +ec_dump_private(xlator_t *this) +{ + ec_t *ec = NULL; + char key_prefix[GF_DUMP_MAX_BUF_LEN]; + char tmp[65]; + + GF_ASSERT(this); + + ec = this->private; + GF_ASSERT(ec); + + snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name); + gf_proc_dump_add_section("%s", key_prefix); + gf_proc_dump_write("up", "%u", ec->up); + gf_proc_dump_write("nodes", "%u", ec->nodes); + gf_proc_dump_write("redundancy", "%u", ec->redundancy); + gf_proc_dump_write("fragment_size", "%u", ec->fragment_size); + gf_proc_dump_write("stripe_size", "%u", ec->stripe_size); + gf_proc_dump_write("childs_up", "%u", ec->xl_up_count); + gf_proc_dump_write("childs_up_mask", "%s", + ec_bin(tmp, sizeof(tmp), ec->xl_up, ec->nodes)); + if (ec->read_mask) { + gf_proc_dump_write("read-mask", "%s", + ec_bin(tmp, sizeof(tmp), ec->read_mask, ec->nodes)); + } + gf_proc_dump_write("background-heals", "%d", ec->background_heals); + gf_proc_dump_write("heal-wait-qlength", "%d", ec->heal_wait_qlen); + gf_proc_dump_write("self-heal-window-size", "%" PRIu32, + ec->self_heal_window_size); + gf_proc_dump_write("healers", "%d", ec->healers); + gf_proc_dump_write("heal-waiters", "%d", ec->heal_waiters); + gf_proc_dump_write("read-policy", "%s", ec_read_policies[ec->read_policy]); + gf_proc_dump_write("parallel-writes", "%d", ec->parallel_writes); + gf_proc_dump_write("quorum-count", "%u", ec->quorum_count); + + snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s.stats.stripe_cache", + this->type, this->name); + gf_proc_dump_add_section("%s", key_prefix); + + gf_proc_dump_write("hits", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.hits)); + gf_proc_dump_write("misses", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.misses)); + gf_proc_dump_write("updates", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.updates)); + gf_proc_dump_write("invalidations", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.invals)); + gf_proc_dump_write("evicts", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.evicts)); + gf_proc_dump_write("allocations", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.allocs)); + gf_proc_dump_write("errors", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.stripe_cache.errors)); + gf_proc_dump_write("heals-attempted", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.shd.attempted)); + gf_proc_dump_write("heals-completed", "%" GF_PRI_ATOMIC, + GF_ATOMIC_GET(ec->stats.shd.completed)); + + return 0; +} + +struct xlator_fops fops = {.lookup = ec_gf_lookup, + .stat = ec_gf_stat, + .fstat = ec_gf_fstat, + .truncate = ec_gf_truncate, + .ftruncate = ec_gf_ftruncate, + .access = ec_gf_access, + .readlink = ec_gf_readlink, + .mknod = ec_gf_mknod, + .mkdir = ec_gf_mkdir, + .unlink = ec_gf_unlink, + .rmdir = ec_gf_rmdir, + .symlink = ec_gf_symlink, + .rename = ec_gf_rename, + .link = ec_gf_link, + .create = ec_gf_create, + .open = ec_gf_open, + .readv = ec_gf_readv, + .writev = ec_gf_writev, + .flush = ec_gf_flush, + .fsync = ec_gf_fsync, + .opendir = ec_gf_opendir, + .readdir = ec_gf_readdir, + .readdirp = ec_gf_readdirp, + .fsyncdir = ec_gf_fsyncdir, + .statfs = ec_gf_statfs, + .setxattr = ec_gf_setxattr, + .getxattr = ec_gf_getxattr, + .fsetxattr = ec_gf_fsetxattr, + .fgetxattr = ec_gf_fgetxattr, + .removexattr = ec_gf_removexattr, + .fremovexattr = ec_gf_fremovexattr, + .lk = ec_gf_lk, + .inodelk = ec_gf_inodelk, + .finodelk = ec_gf_finodelk, + .entrylk = ec_gf_entrylk, + .fentrylk = ec_gf_fentrylk, + .xattrop = ec_gf_xattrop, + .fxattrop = ec_gf_fxattrop, + .setattr = ec_gf_setattr, + .fsetattr = ec_gf_fsetattr, + .fallocate = ec_gf_fallocate, + .discard = ec_gf_discard, + .zerofill = ec_gf_zerofill, + .seek = ec_gf_seek, + .ipc = ec_gf_ipc}; + +struct xlator_cbks cbks = {.forget = ec_gf_forget, + .release = ec_gf_release, + .releasedir = ec_gf_releasedir}; + +struct xlator_dumpops dumpops = {.priv = ec_dump_private}; + +struct volume_options options[] = { + {.key = {"redundancy"}, + .type = GF_OPTION_TYPE_INT, + .default_value = "{{ volume.redundancy }}", + .description = "Maximum number of bricks that can fail " + "simultaneously without losing data."}, + { + .key = {"self-heal-daemon"}, + .type = GF_OPTION_TYPE_BOOL, + .description = "self-heal daemon enable/disable", + .default_value = "enable", + .op_version = {GD_OP_VERSION_3_7_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC, + .tags = {"disperse"}, + }, + {.key = {"iam-self-heal-daemon"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "off", + .description = "This option differentiates if the disperse " + "translator is running as part of self-heal-daemon " + "or not."}, + {.key = {"eager-lock"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "on", + .op_version = {GD_OP_VERSION_3_7_10}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "Enable/Disable eager lock for regular files on a " + "disperse volume. If a fop takes a lock and completes " + "its operation, it waits for next 1 second before " + "releasing the lock, to see if the lock can be reused " + "for next fop from the same client. If ec finds any lock " + "contention within 1 second it releases the lock " + "immediately before time expires. This improves the " + "performance of file operations. However, as it takes " + "lock on first brick, for few operations like read, " + "discovery of lock contention might take long time and " + "can actually degrade the performance. If eager lock is " + "disabled, lock will be released as soon as fop " + "completes."}, + {.key = {"other-eager-lock"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "on", + .op_version = {GD_OP_VERSION_3_13_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "It's equivalent to the eager-lock option but for non " + "regular files."}, + {.key = {"eager-lock-timeout"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 60, + .default_value = "1", + .op_version = {GD_OP_VERSION_4_0_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse", "locks", "timeout"}, + .description = "Maximum time (in seconds) that a lock on an inode is " + "kept held if no new operations on the inode are " + "received."}, + {.key = {"other-eager-lock-timeout"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 60, + .default_value = "1", + .op_version = {GD_OP_VERSION_4_0_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse", "locks", "timeout"}, + .description = "It's equivalent to eager-lock-timeout option but for " + "non regular files."}, + { + .key = {"background-heals"}, + .type = GF_OPTION_TYPE_INT, + .min = 0, /*Disabling background heals*/ + .max = 256, + .default_value = "8", + .op_version = {GD_OP_VERSION_3_7_3}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "This option can be used to control number of parallel" + " heals", + }, + { + .key = {"heal-wait-qlength"}, + .type = GF_OPTION_TYPE_INT, + .min = 0, + .max = + 65536, /*Around 100MB as of now with sizeof(ec_fop_data_t) at 1800*/ + .default_value = "128", + .op_version = {GD_OP_VERSION_3_7_3}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "This option can be used to control number of heals" + " that can wait", + }, + {.key = {"heal-timeout"}, + .type = GF_OPTION_TYPE_INT, + .min = 60, + .max = INT_MAX, + .default_value = "600", + .op_version = {GD_OP_VERSION_3_7_3}, + .flags = OPT_FLAG_SETTABLE, + .tags = {"disperse"}, + .description = "time interval for checking the need to self-heal " + "in self-heal-daemon"}, + { + .key = {"read-policy"}, + .type = GF_OPTION_TYPE_STR, + .value = {"round-robin", "gfid-hash"}, + .default_value = "gfid-hash", + .op_version = {GD_OP_VERSION_3_7_6}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = + "inode-read fops happen only on 'k' number of bricks in" + " n=k+m disperse subvolume. 'round-robin' selects the read" + " subvolume using round-robin algo. 'gfid-hash' selects read" + " subvolume based on hash of the gfid of that file/directory.", + }, + {.key = {"shd-max-threads"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 64, + .default_value = "1", + .op_version = {GD_OP_VERSION_3_9_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "Maximum number of parallel heals SHD can do per local " + "brick. This can substantially lower heal times, " + "but can also crush your bricks if you don't have " + "the storage hardware to support this."}, + {.key = {"shd-wait-qlength"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 65536, + .default_value = "1024", + .op_version = {GD_OP_VERSION_3_9_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "This option can be used to control number of heals" + " that can wait in SHD per subvolume"}, + {.key = {"cpu-extensions"}, + .type = GF_OPTION_TYPE_STR, + .value = {"none", "auto", "x64", "sse", "avx"}, + .default_value = "auto", + .op_version = {GD_OP_VERSION_3_9_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "force the cpu extensions to be used to accelerate the " + "galois field computations."}, + {.key = {"self-heal-window-size"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 1024, + .default_value = "1", + .op_version = {GD_OP_VERSION_3_11_0}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC, + .tags = {"disperse"}, + .description = "Maximum number blocks(128KB) per file for which " + "self-heal process would be applied simultaneously."}, + {.key = {"optimistic-change-log"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "on", + .op_version = {GD_OP_VERSION_3_10_1}, + .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT, + .tags = {"disperse"}, + .description = "Set/Unset dirty flag for every update fop at the start" + "of the fop. If OFF, this option impacts performance of" + "entry operations or metadata operations as it will" + "set dirty flag at the start and unset it at the end of" + "ALL update fop. If ON and all the bricks are good," + "dirty flag will be set at the start only for file fops" + "For metadata and entry fops dirty flag will not be set" + "at the start, if all the bricks are good. This does" + "not impact performance for metadata operations and" + "entry operation but has a very small window to miss" + "marking entry as dirty in case it is required to be" + "healed"}, + {.key = {"parallel-writes"}, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "on", + .description = "This controls if writes can be wound in parallel as long" + "as it doesn't modify same stripes"}, + {.key = {"stripe-cache"}, + .type = GF_OPTION_TYPE_INT, + .min = 0, /*Disabling stripe_cache*/ + .max = EC_STRIPE_CACHE_MAX_SIZE, + .default_value = "4", + .description = "This option will keep the last stripe of write fop" + "in memory. If next write falls in this stripe, we need" + "not to read it again from backend and we can save READ" + "fop going over the network. This will improve performance," + "specially for sequential writes. However, this will also" + "lead to extra memory consumption, maximum " + "(cache size * stripe size) Bytes per open file."}, + { + .key = {"quorum-count"}, + .type = GF_OPTION_TYPE_INT, + .default_value = "0", + .description = + "This option can be used to define how many successes on" + "the bricks constitute a success to the application. This" + " count should be in the range" + "[disperse-data-count, disperse-count] (inclusive)", + }, + { + .key = {"ec-read-mask"}, + .type = GF_OPTION_TYPE_STR, + .default_value = NULL, + .description = "This option can be used to choose which bricks can be" + " used for reading data/metadata of a file/directory", + }, + { + .key = {NULL}, + }, +}; + +xlator_api_t xlator_api = { + .init = init, + .fini = fini, + .notify = notify, + .reconfigure = reconfigure, + .mem_acct_init = mem_acct_init, + .op_version = {1}, + .dumpops = &dumpops, + .fops = &fops, + .cbks = &cbks, + .options = options, + .identifier = "disperse", + .category = GF_MAINTAINED, +}; diff --git a/xlators/cluster/ec/src/ec.h b/xlators/cluster/ec/src/ec.h new file mode 100644 index 00000000000..6f6de6d5981 --- /dev/null +++ b/xlators/cluster/ec/src/ec.h @@ -0,0 +1,34 @@ +/* + Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef __EC_H__ +#define __EC_H__ + +#include "ec-method.h" + +#define EC_XATTR_PREFIX "trusted.ec." +#define EC_XATTR_CONFIG EC_XATTR_PREFIX "config" +#define EC_XATTR_SIZE EC_XATTR_PREFIX "size" +#define EC_XATTR_VERSION EC_XATTR_PREFIX "version" +#define EC_XATTR_HEAL EC_XATTR_PREFIX "heal" +#define EC_XATTR_HEAL_NEW EC_XATTR_PREFIX "heal-new" +#define EC_XATTR_DIRTY EC_XATTR_PREFIX "dirty" +#define EC_STRIPE_CACHE_MAX_SIZE 10 +#define EC_VERSION_SIZE 2 +#define EC_SHD_INODE_LRU_LIMIT 10 + +#define EC_MAX_FRAGMENTS EC_METHOD_MAX_FRAGMENTS +/* The maximum number of nodes is derived from the maximum allowed fragments + * using the rule that redundancy cannot be equal or greater than the number + * of fragments. + */ +#define EC_MAX_NODES min(EC_MAX_FRAGMENTS * 2 - 1, EC_METHOD_MAX_NODES) + +#endif /* __EC_H__ */ |