diff options
author | Yaniv Kaul <ykaul@redhat.com> | 2018-07-21 12:05:30 +0300 |
---|---|---|
committer | Kotresh HR <khiremat@redhat.com> | 2018-08-16 04:06:13 +0000 |
commit | 4fe662be4143629e405b1718836be6c990fcebbb (patch) | |
tree | 0f6268b4b9b84c92f8b648fe2fa54ffafb32d9d9 | |
parent | 4d1ea415f5346b24c13810aab18a24256fb2b41a (diff) |
contrib/xxhash: update to latest xxHash (0.6.5)
Update to latest xxHash, which is supposed to faster with small keys.
Specifically, updated to 3064d42e7d74b0921bdd1818395d9cb37bb8976a,
which is a bit higher than 0.6.5.
Compiled hopefully with namespace (XXH_NAMESPACE=GF_),
which allows to use XXH() funcs with no fear they'll 'leak'
from our library.
Only compile tested!
xxhsum is modified to display messages which was conflicting
with regression tests (TAP harness). So modified the
gfid2path_fuse.t and gfid2path_nfs.t to adhere to that.
updates: bz#1193929
Signed-off-by: Yaniv Kaul <ykaul@redhat.com>
Change-Id: I35cea5cc93f338c1023ac2c9bc6d7d13225a967b
-rw-r--r-- | contrib/xxhash/xxhash.c | 501 | ||||
-rw-r--r-- | contrib/xxhash/xxhash.h | 317 | ||||
-rw-r--r-- | contrib/xxhash/xxhsum.c | 353 | ||||
-rw-r--r-- | libglusterfs/src/Makefile.am | 1 | ||||
-rw-r--r-- | libglusterfs/src/common-utils.c | 15 | ||||
-rw-r--r-- | libglusterfs/src/common-utils.h | 4 | ||||
-rw-r--r-- | tests/gfid2path/gfid2path_fuse.t | 20 | ||||
-rw-r--r-- | tests/gfid2path/gfid2path_nfs.t | 18 |
8 files changed, 730 insertions, 499 deletions
diff --git a/contrib/xxhash/xxhash.c b/contrib/xxhash/xxhash.c index d5592c2d7ee..56f80f8811d 100644 --- a/contrib/xxhash/xxhash.c +++ b/contrib/xxhash/xxhash.c @@ -50,26 +50,32 @@ * Prefer these methods in priority order (0 > 1 > 2) */ #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ + || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) \ + || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define XXH_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \ + || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \ + || defined(__ARM_ARCH_7S__) )) # define XXH_FORCE_MEMORY_ACCESS 1 # endif #endif /*!XXH_ACCEPT_NULL_INPUT_POINTER : - * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. - * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. - * By default, this option is disabled. To enable it, uncomment below define : + * If input pointer is NULL, xxHash default behavior is to dereference it, triggering a segfault. + * When this macro is enabled, xxHash actively checks input for null pointer. + * It it is, result for null input pointers is the same as a null-length input. */ -/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ +#ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ +# define XXH_ACCEPT_NULL_INPUT_POINTER 0 +#endif /*!XXH_FORCE_NATIVE_FORMAT : - * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. * Results are therefore identical for little-endian and big-endian CPU. * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. - * Should endian-independance be of no importance for your application, you may set the #define below to 1, + * Should endian-independence be of no importance for your application, you may set the #define below to 1, * to improve speed for Big-endian CPU. * This option has no impact on Little_Endian CPU. */ @@ -80,8 +86,9 @@ /*!XXH_FORCE_ALIGN_CHECK : * This is a minor performance trick, only useful with lots of very small keys. * It means : check for aligned/unaligned input. - * The check costs one initial branch per hash; set to 0 when the input data - * is guaranteed to be aligned. + * The check costs one initial branch per hash; + * set it to 0 when the input is guaranteed to be aligned, + * or when alignment doesn't matter for performance. */ #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ # if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) @@ -95,15 +102,17 @@ /* ************************************* * Includes & Memory related functions ***************************************/ -/* Modify the local functions below should you wish to use some other memory routines */ -/* for malloc(), free() */ +/*! Modify the local functions below should you wish to use some other memory routines +* for malloc(), free() */ #include <stdlib.h> static void* XXH_malloc(size_t s) { return malloc(s); } static void XXH_free (void* p) { free(p); } -/* for memcpy() */ +/*! and for memcpy() */ #include <string.h> static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } +#include <assert.h> /* assert */ + #define XXH_STATIC_LINKING_ONLY #include "xxhash.h" @@ -115,7 +124,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # define FORCE_INLINE static __forceinline #else -# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # ifdef __GNUC__ # define FORCE_INLINE static inline __attribute__((always_inline)) # else @@ -131,17 +140,17 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp * Basic Types ***************************************/ #ifndef MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include <stdint.h> typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; - typedef int32_t S32; # else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; - typedef signed int S32; # endif #endif @@ -208,8 +217,12 @@ typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ #ifndef XXH_CPU_LITTLE_ENDIAN - static const int g_one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +static int XXH_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() #endif @@ -240,12 +253,12 @@ static U32 XXH_readBE32(const void* ptr) /* ************************************* * Macros ***************************************/ -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ -XXH_PUBLIC_API unsigned GF_XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } +#define XXH_STATIC_ASSERT(c) { enum { XXH_sa = 1/(int)(!!(c)) }; } /* use after variable declarations */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } /* ******************************************************************* -* 32-bits hash functions +* 32-bit hash functions *********************************************************************/ static const U32 PRIME32_1 = 2654435761U; static const U32 PRIME32_2 = 2246822519U; @@ -261,14 +274,89 @@ static U32 XXH32_round(U32 seed, U32 input) return seed; } -FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +/* mix all bits */ +static U32 XXH32_avalanche(U32 h32) +{ + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +static U32 +XXH32_finalize(U32 h32, const void* ptr, size_t len, + XXH_endianess endian, XXH_alignment align) + +{ + const BYTE* p = (const BYTE*)ptr; +#define PROCESS1 \ + h32 += (*p) * PRIME32_5; \ + p++; \ + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + +#define PROCESS4 \ + h32 += XXH_get32bits(p) * PRIME32_3; \ + p+=4; \ + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + + switch(len&15) /* or switch(bEnd - p) */ + { + case 12: PROCESS4; + /* fallthrough */ + case 8: PROCESS4; + /* fallthrough */ + case 4: PROCESS4; + return XXH32_avalanche(h32); + + case 13: PROCESS4; + /* fallthrough */ + case 9: PROCESS4; + /* fallthrough */ + case 5: PROCESS4; + PROCESS1; + return XXH32_avalanche(h32); + + case 14: PROCESS4; + /* fallthrough */ + case 10: PROCESS4; + /* fallthrough */ + case 6: PROCESS4; + PROCESS1; + PROCESS1; + return XXH32_avalanche(h32); + + case 15: PROCESS4; + /* fallthrough */ + case 11: PROCESS4; + /* fallthrough */ + case 7: PROCESS4; + /* fallthrough */ + case 3: PROCESS1; + /* fallthrough */ + case 2: PROCESS1; + /* fallthrough */ + case 1: PROCESS1; + /* fallthrough */ + case 0: return XXH32_avalanche(h32); + } + assert(0); + return h32; /* reaching this point is deemed impossible */ +} + + +FORCE_INLINE U32 +XXH32_endian_align(const void* input, size_t len, U32 seed, + XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; const BYTE* bEnd = p + len; U32 h32; -#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)16; @@ -276,7 +364,7 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH #endif if (len>=16) { - const BYTE* const limit = bEnd - 16; + const BYTE* const limit = bEnd - 15; U32 v1 = seed + PRIME32_1 + PRIME32_2; U32 v2 = seed + PRIME32_2; U32 v3 = seed + 0; @@ -287,45 +375,28 @@ FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; - } while (p<=limit); + } while (p < limit); - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); } else { h32 = seed + PRIME32_5; } - h32 += (U32) len; - - while (p+4<=bEnd) { - h32 += XXH_get32bits(p) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } + h32 += (U32)len; - while (p<bEnd) { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; + return XXH32_finalize(h32, p, len&15, endian, align); } -XXH_PUBLIC_API unsigned int GF_XXH32 (const void* input, size_t len, unsigned int seed) +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH32_CREATESTATE_STATIC(state); - GF_XXH32_reset(state, seed); - GF_XXH32_update(state, input, len); - return GF_XXH32_digest(state); + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); #else XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -348,41 +419,46 @@ XXH_PUBLIC_API unsigned int GF_XXH32 (const void* input, size_t len, unsigned in /*====== Hash streaming ======*/ -XXH_PUBLIC_API GF_XXH32_state_t* GF_XXH32_createState(void) +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) { - return (GF_XXH32_state_t*)XXH_malloc(sizeof(GF_XXH32_state_t)); + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); } -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH32_freeState(GF_XXH32_state_t* statePtr) +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } -XXH_PUBLIC_API void GF_XXH32_copyState(GF_XXH32_state_t* restrict dstState, const GF_XXH32_state_t* restrict srcState) +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) { memcpy(dstState, srcState, sizeof(*dstState)); } -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH32_reset(GF_XXH32_state_t* statePtr, unsigned int seed) +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) { - GF_XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME32_1 + PRIME32_2; state.v2 = seed + PRIME32_2; state.v3 = seed + 0; state.v4 = seed - PRIME32_1; - memcpy(statePtr, &state, sizeof(state)); + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); return XXH_OK; } -FORCE_INLINE GF_XXH_errorcode XXH32_update_endian (GF_XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +FORCE_INLINE +XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; #endif state->total_len_32 += (unsigned)len; @@ -400,7 +476,7 @@ FORCE_INLINE GF_XXH_errorcode XXH32_update_endian (GF_XXH32_state_t* state, cons state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; - state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); } p += 16-state->memsize; state->memsize = 0; @@ -434,7 +510,8 @@ FORCE_INLINE GF_XXH_errorcode XXH32_update_endian (GF_XXH32_state_t* state, cons return XXH_OK; } -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH32_update (GF_XXH32_state_t* state_in, const void* input, size_t len) + +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -445,44 +522,27 @@ XXH_PUBLIC_API GF_XXH_errorcode GF_XXH32_update (GF_XXH32_state_t* state_in, con } - -FORCE_INLINE U32 XXH32_digest_endian (const GF_XXH32_state_t* state, XXH_endianess endian) +FORCE_INLINE U32 +XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) { - const BYTE * p = (const BYTE*)state->mem32; - const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; U32 h32; if (state->large_len) { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + h32 = XXH_rotl32(state->v1, 1) + + XXH_rotl32(state->v2, 7) + + XXH_rotl32(state->v3, 12) + + XXH_rotl32(state->v4, 18); } else { h32 = state->v3 /* == seed */ + PRIME32_5; } h32 += state->total_len_32; - while (p+4<=bEnd) { - h32 += XXH_readLE32(p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p<bEnd) { - h32 += (*p) * PRIME32_5; - h32 = XXH_rotl32(h32, 11) * PRIME32_1; - p++; - } - - h32 ^= h32 >> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; + return XXH32_finalize(h32, state->mem32, state->memsize, endian, XXH_aligned); } -XXH_PUBLIC_API unsigned int GF_XXH32_digest (const GF_XXH32_state_t* state_in) +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -498,17 +558,17 @@ XXH_PUBLIC_API unsigned int GF_XXH32_digest (const GF_XXH32_state_t* state_in) /*! Default XXH result types are basic unsigned 32 and 64 bits. * The canonical representation follows human-readable write convention, aka big-endian (large digits first). * These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. +* This way, hash values can be written into a file or buffer, remaining comparable across different systems. */ -XXH_PUBLIC_API void GF_XXH32_canonicalFromHash(GF_XXH32_canonical_t* dst, GF_XXH32_hash_t hash) +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) { - XXH_STATIC_ASSERT(sizeof(GF_XXH32_canonical_t) == sizeof(GF_XXH32_hash_t)); + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); memcpy(dst, &hash, sizeof(*dst)); } -XXH_PUBLIC_API GF_XXH32_hash_t GF_XXH32_hashFromCanonical(const GF_XXH32_canonical_t* src) +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) { return XXH_readBE32(src); } @@ -517,18 +577,21 @@ XXH_PUBLIC_API GF_XXH32_hash_t GF_XXH32_hashFromCanonical(const GF_XXH32_canonic #ifndef XXH_NO_LONG_LONG /* ******************************************************************* -* 64-bits hash functions +* 64-bit hash functions *********************************************************************/ /*====== Memory access ======*/ #ifndef MEM_MODULE # define MEM_MODULE -# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) # include <stdint.h> typedef uint64_t U64; # else - typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ + /* if compiler doesn't support unsigned long long, replace by another 64-bit type */ + typedef unsigned long long U64; # endif #endif @@ -543,7 +606,6 @@ static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ /* currently only defined for gcc and icc */ typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64; - static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } #else @@ -622,14 +684,138 @@ static U64 XXH64_mergeRound(U64 acc, U64 val) return acc; } -FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +static U64 XXH64_avalanche(U64 h64) +{ + h64 ^= h64 >> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +static U64 +XXH64_finalize(U64 h64, const void* ptr, size_t len, + XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)ptr; + +#define PROCESS1_64 \ + h64 ^= (*p) * PRIME64_5; \ + p++; \ + h64 = XXH_rotl64(h64, 11) * PRIME64_1; + +#define PROCESS4_64 \ + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; \ + p+=4; \ + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + +#define PROCESS8_64 { \ + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); \ + p+=8; \ + h64 ^= k1; \ + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; \ +} + + switch(len&31) { + case 24: PROCESS8_64; + /* fallthrough */ + case 16: PROCESS8_64; + /* fallthrough */ + case 8: PROCESS8_64; + return XXH64_avalanche(h64); + + case 28: PROCESS8_64; + /* fallthrough */ + case 20: PROCESS8_64; + /* fallthrough */ + case 12: PROCESS8_64; + /* fallthrough */ + case 4: PROCESS4_64; + return XXH64_avalanche(h64); + + case 25: PROCESS8_64; + /* fallthrough */ + case 17: PROCESS8_64; + /* fallthrough */ + case 9: PROCESS8_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 29: PROCESS8_64; + /* fallthrough */ + case 21: PROCESS8_64; + /* fallthrough */ + case 13: PROCESS8_64; + /* fallthrough */ + case 5: PROCESS4_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 26: PROCESS8_64; + /* fallthrough */ + case 18: PROCESS8_64; + /* fallthrough */ + case 10: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 30: PROCESS8_64; + /* fallthrough */ + case 22: PROCESS8_64; + /* fallthrough */ + case 14: PROCESS8_64; + /* fallthrough */ + case 6: PROCESS4_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 27: PROCESS8_64; + /* fallthrough */ + case 19: PROCESS8_64; + /* fallthrough */ + case 11: PROCESS8_64; + PROCESS1_64; + PROCESS1_64; + PROCESS1_64; + return XXH64_avalanche(h64); + + case 31: PROCESS8_64; + /* fallthrough */ + case 23: PROCESS8_64; + /* fallthrough */ + case 15: PROCESS8_64; + /* fallthrough */ + case 7: PROCESS4_64; + /* fallthrough */ + case 3: PROCESS1_64; + /* fallthrough */ + case 2: PROCESS1_64; + /* fallthrough */ + case 1: PROCESS1_64; + /* fallthrough */ + case 0: return XXH64_avalanche(h64); + } + + /* impossible to reach */ + assert(0); + return 0; /* unreachable, but some compilers complain without it */ +} + +FORCE_INLINE U64 +XXH64_endian_align(const void* input, size_t len, U64 seed, + XXH_endianess endian, XXH_alignment align) { const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; + const BYTE* bEnd = p + len; U64 h64; -#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)32; @@ -662,43 +848,18 @@ FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH h64 += (U64) len; - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_get64bits(p)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; + return XXH64_finalize(h64, p, len, endian, align); } -XXH_PUBLIC_API unsigned long long GF_XXH64 (const void* input, size_t len, unsigned long long seed) +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) { #if 0 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ - XXH64_CREATESTATE_STATIC(state); - GF_XXH64_reset(state, seed); - GF_XXH64_update(state, input, len); - return GF_XXH64_digest(state); + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, input, len); + return XXH64_digest(&state); #else XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -719,40 +880,45 @@ XXH_PUBLIC_API unsigned long long GF_XXH64 (const void* input, size_t len, unsig /*====== Hash Streaming ======*/ -XXH_PUBLIC_API GF_XXH64_state_t* GF_XXH64_createState(void) +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) { - return (GF_XXH64_state_t*)XXH_malloc(sizeof(GF_XXH64_state_t)); + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); } -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH64_freeState(GF_XXH64_state_t* statePtr) +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) { XXH_free(statePtr); return XXH_OK; } -XXH_PUBLIC_API void GF_XXH64_copyState(GF_XXH64_state_t* restrict dstState, const GF_XXH64_state_t* restrict srcState) +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) { memcpy(dstState, srcState, sizeof(*dstState)); } -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH64_reset(GF_XXH64_state_t* statePtr, unsigned long long seed) +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) { - GF_XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ - memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); state.v1 = seed + PRIME64_1 + PRIME64_2; state.v2 = seed + PRIME64_2; state.v3 = seed + 0; state.v4 = seed - PRIME64_1; - memcpy(statePtr, &state, sizeof(state)); + /* do not write into reserved, planned to be removed in a future version */ + memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); return XXH_OK; } -FORCE_INLINE GF_XXH_errorcode XXH64_update_endian (GF_XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +FORCE_INLINE +XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) { const BYTE* p = (const BYTE*)input; const BYTE* const bEnd = p + len; -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; + if (input==NULL) +#if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && (XXH_ACCEPT_NULL_INPUT_POINTER>=1) + return XXH_OK; +#else + return XXH_ERROR; #endif state->total_len += len; @@ -801,7 +967,7 @@ FORCE_INLINE GF_XXH_errorcode XXH64_update_endian (GF_XXH64_state_t* state, cons return XXH_OK; } -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH64_update (GF_XXH64_state_t* state_in, const void* input, size_t len) +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -811,10 +977,8 @@ XXH_PUBLIC_API GF_XXH_errorcode GF_XXH64_update (GF_XXH64_state_t* state_in, con return XXH64_update_endian(state_in, input, len, XXH_bigEndian); } -FORCE_INLINE U64 XXH64_digest_endian (const GF_XXH64_state_t* state, XXH_endianess endian) +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) { - const BYTE * p = (const BYTE*)state->mem64; - const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; U64 h64; if (state->total_len >= 32) { @@ -829,40 +993,15 @@ FORCE_INLINE U64 XXH64_digest_endian (const GF_XXH64_state_t* state, XXH_endiane h64 = XXH64_mergeRound(h64, v3); h64 = XXH64_mergeRound(h64, v4); } else { - h64 = state->v3 + PRIME64_5; + h64 = state->v3 /*seed*/ + PRIME64_5; } h64 += (U64) state->total_len; - while (p+8<=bEnd) { - U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); - h64 ^= k1; - h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; - p+=8; - } - - if (p+4<=bEnd) { - h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; - h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; - p+=4; - } - - while (p<bEnd) { - h64 ^= (*p) * PRIME64_5; - h64 = XXH_rotl64(h64, 11) * PRIME64_1; - p++; - } - - h64 ^= h64 >> 33; - h64 *= PRIME64_2; - h64 ^= h64 >> 29; - h64 *= PRIME64_3; - h64 ^= h64 >> 32; - - return h64; + return XXH64_finalize(h64, state->mem64, (size_t)state->total_len, endian, XXH_aligned); } -XXH_PUBLIC_API unsigned long long GF_XXH64_digest (const GF_XXH64_state_t* state_in) +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) { XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; @@ -875,14 +1014,14 @@ XXH_PUBLIC_API unsigned long long GF_XXH64_digest (const GF_XXH64_state_t* state /*====== Canonical representation ======*/ -XXH_PUBLIC_API void GF_XXH64_canonicalFromHash(GF_XXH64_canonical_t* dst, GF_XXH64_hash_t hash) +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) { - XXH_STATIC_ASSERT(sizeof(GF_XXH64_canonical_t) == sizeof(GF_XXH64_hash_t)); + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); memcpy(dst, &hash, sizeof(*dst)); } -XXH_PUBLIC_API GF_XXH64_hash_t GF_XXH64_hashFromCanonical(const GF_XXH64_canonical_t* src) +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) { return XXH_readBE64(src); } diff --git a/contrib/xxhash/xxhash.h b/contrib/xxhash/xxhash.h index 98352b9018e..d6bad943358 100644 --- a/contrib/xxhash/xxhash.h +++ b/contrib/xxhash/xxhash.h @@ -57,8 +57,8 @@ Q.Score is a measure of quality of the hash function. It depends on successfully passing SMHasher test set. 10 is a perfect score. -A 64-bits version, named XXH64, is available since r35. -It offers much better speed, but for 64-bits applications only. +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. Name Speed on 64 bits Speed on 32 bits XXH64 13.8 GB/s 1.9 GB/s XXH32 6.8 GB/s 6.0 GB/s @@ -73,33 +73,26 @@ extern "C" { /* **************************** -* Compiler specifics -******************************/ -#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ -# define restrict /* disable restrict */ -#endif - - -/* **************************** * Definitions ******************************/ #include <stddef.h> /* size_t */ -typedef enum { XXH_OK=0, XXH_ERROR } GF_XXH_errorcode; +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; /* **************************** -* API modifier -******************************/ -/** XXH_PRIVATE_API -* This is useful to include xxhash functions in `static` mode -* in order to inline them, and remove their symbol from the public list. -* Methodology : -* #define XXH_PRIVATE_API -* #include "xxhash.h" -* `xxhash.c` is automatically included. -* It's not useful to compile and link it as a separate module. -*/ -#ifdef XXH_PRIVATE_API + * API modifier + ******************************/ +/** XXH_INLINE_ALL (and XXH_PRIVATE_API) + * This is useful to include xxhash functions in `static` mode + * in order to inline them, and remove their symbol from the public list. + * Inlining can offer dramatic performance improvement on small keys. + * Methodology : + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * `xxhash.c` is automatically included. + * It's not useful to compile and link it as a separate module. + */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) # ifndef XXH_STATIC_LINKING_ONLY # define XXH_STATIC_LINKING_ONLY # endif @@ -110,45 +103,46 @@ typedef enum { XXH_OK=0, XXH_ERROR } GF_XXH_errorcode; # elif defined(_MSC_VER) # define XXH_PUBLIC_API static __inline # else -# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ + /* this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static # endif #else # define XXH_PUBLIC_API /* do nothing */ -#endif /* XXH_PRIVATE_API */ - -/*!XXH_NAMESPACE, aka Namespace Emulation : - -If you want to include _and expose_ xxHash functions from within your own library, -but also want to avoid symbol collisions with other libraries which may also include xxHash, - -you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library -with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). - -Note that no change is required within the calling program as long as it includes `xxhash.h` : -regular symbol name will be automatically translated by this header. -*/ +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/*! XXH_NAMESPACE, aka Namespace Emulation : + * + * If you want to include _and expose_ xxHash functions from within your own library, + * but also want to avoid symbol collisions with other libraries which may also include xxHash, + * + * you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library + * with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). + * + * Note that no change is required within the calling program as long as it includes `xxhash.h` : + * regular symbol name will be automatically translated by this header. + */ #ifdef XXH_NAMESPACE # define XXH_CAT(A,B) A##B # define XXH_NAME2(A,B) XXH_CAT(A,B) -# define GF_XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, GF_XXH_versionNumber) -# define GF_XXH32 XXH_NAME2(XXH_NAMESPACE, GF_XXH32) -# define GF_XXH32_createState XXH_NAME2(XXH_NAMESPACE, GF_XXH32_createState) -# define GF_XXH32_freeState XXH_NAME2(XXH_NAMESPACE, GF_XXH32_freeState) -# define GF_XXH32_reset XXH_NAME2(XXH_NAMESPACE, GF_XXH32_reset) -# define GF_XXH32_update XXH_NAME2(XXH_NAMESPACE, GF_XXH32_update) -# define GF_XXH32_digest XXH_NAME2(XXH_NAMESPACE, GF_XXH32_digest) -# define GF_XXH32_copyState XXH_NAME2(XXH_NAMESPACE, GF_XXH32_copyState) -# define GF_XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, GF_XXH32_canonicalFromHash) -# define GF_XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, GF_XXH32_hashFromCanonical) -# define GF_XXH64 XXH_NAME2(XXH_NAMESPACE, GF_XXH64) -# define GF_XXH64_createState XXH_NAME2(XXH_NAMESPACE, GF_XXH64_createState) -# define GF_XXH64_freeState XXH_NAME2(XXH_NAMESPACE, GF_XXH64_freeState) -# define GF_XXH64_reset XXH_NAME2(XXH_NAMESPACE, GF_XXH64_reset) -# define GF_XXH64_update XXH_NAME2(XXH_NAMESPACE, GF_XXH64_update) -# define GF_XXH64_digest XXH_NAME2(XXH_NAMESPACE, GF_XXH64_digest) -# define GF_XXH64_copyState XXH_NAME2(XXH_NAMESPACE, GF_XXH64_copyState) -# define GF_XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, GF_XXH64_canonicalFromHash) -# define GF_XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, GF_XXH64_hashFromCanonical) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) #endif @@ -157,140 +151,173 @@ regular symbol name will be automatically translated by this header. ***************************************/ #define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MINOR 6 -#define XXH_VERSION_RELEASE 2 +#define XXH_VERSION_RELEASE 5 #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) -XXH_PUBLIC_API unsigned GF_XXH_versionNumber (void); +XXH_PUBLIC_API unsigned XXH_versionNumber (void); /*-********************************************************************** -* 32-bits hash +* 32-bit hash ************************************************************************/ -typedef unsigned int GF_XXH32_hash_t; +typedef unsigned int XXH32_hash_t; /*! XXH32() : - Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + Calculate the 32-bit hash of sequence "length" bytes stored at memory address "input". The memory between input & input+length must be valid (allocated and read-accessible). "seed" can be used to alter the result predictably. Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ -XXH_PUBLIC_API GF_XXH32_hash_t GF_XXH32 (const void* input, size_t length, unsigned int seed); +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); /*====== Streaming ======*/ -typedef struct XXH32_state_s GF_XXH32_state_t; /* incomplete type */ -XXH_PUBLIC_API GF_XXH32_state_t* GF_XXH32_createState(void); -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH32_freeState(GF_XXH32_state_t* statePtr); -XXH_PUBLIC_API void GF_XXH32_copyState(GF_XXH32_state_t* restrict dst_state, const GF_XXH32_state_t* restrict src_state); +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH32_reset (GF_XXH32_state_t* statePtr, unsigned int seed); -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH32_update (GF_XXH32_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API GF_XXH32_hash_t GF_XXH32_digest (const GF_XXH32_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); /* -These functions generate the xxHash of an input provided in multiple segments. -Note that, for small input, they are slower than single-call functions, due to state management. -For small input, prefer `XXH32()` and `XXH64()` . - -XXH state must first be allocated, using XXH*_createState() . - -Start a new hash by initializing state with a seed, using XXH*_reset(). - -Then, feed the hash state by calling XXH*_update() as many times as necessary. -Obviously, input must be allocated and read accessible. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. - -Finally, a hash value can be produced anytime, by using XXH*_digest(). -This function returns the nn-bits hash as an int or long long. - -It's still possible to continue inserting input into the hash state after a digest, -and generate some new hashes later on, by calling again XXH*_digest(). - -When done, free XXH state space if it was allocated dynamically. -*/ + * Streaming functions generate the xxHash of an input provided in multiple segments. + * Note that, for small input, they are slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * XXH state must first be allocated, using XXH*_createState() . + * + * Start a new hash by initializing state with a seed, using XXH*_reset(). + * + * Then, feed the hash state by calling XXH*_update() as many times as necessary. + * The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using XXH*_digest(). + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a digest, + * and generate some new hashes later on, by calling again XXH*_digest(). + * + * When done, free XXH state space if it was allocated dynamically. + */ /*====== Canonical representation ======*/ -typedef struct { unsigned char digest[4]; } GF_XXH32_canonical_t; -XXH_PUBLIC_API void GF_XXH32_canonicalFromHash(GF_XXH32_canonical_t* dst, GF_XXH32_hash_t hash); -XXH_PUBLIC_API GF_XXH32_hash_t GF_XXH32_hashFromCanonical(const GF_XXH32_canonical_t* src); +typedef struct { unsigned char digest[4]; } XXH32_canonical_t; +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); /* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ + * The canonical representation uses human-readable write convention, aka big-endian (large digits first). + * These functions allow transformation of hash result into and from its canonical format. + * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. + */ #ifndef XXH_NO_LONG_LONG /*-********************************************************************** -* 64-bits hash +* 64-bit hash ************************************************************************/ -typedef unsigned long long GF_XXH64_hash_t; +typedef unsigned long long XXH64_hash_t; /*! XXH64() : - Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + Calculate the 64-bit hash of sequence of length "len" stored at memory address "input". "seed" can be used to alter the result predictably. - This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). + This function runs faster on 64-bit systems, but slower on 32-bit systems (see benchmark). */ -XXH_PUBLIC_API GF_XXH64_hash_t GF_XXH64 (const void* input, size_t length, unsigned long long seed); +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); /*====== Streaming ======*/ -typedef struct XXH64_state_s GF_XXH64_state_t; /* incomplete type */ -XXH_PUBLIC_API GF_XXH64_state_t* GF_XXH64_createState(void); -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH64_freeState(GF_XXH64_state_t* statePtr); -XXH_PUBLIC_API void GF_XXH64_copyState(GF_XXH64_state_t* restrict dst_state, const GF_XXH64_state_t* restrict src_state); +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH64_reset (GF_XXH64_state_t* statePtr, unsigned long long seed); -XXH_PUBLIC_API GF_XXH_errorcode GF_XXH64_update (GF_XXH64_state_t* statePtr, const void* input, size_t length); -XXH_PUBLIC_API GF_XXH64_hash_t GF_XXH64_digest (const GF_XXH64_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); /*====== Canonical representation ======*/ -typedef struct { unsigned char digest[8]; } GF_XXH64_canonical_t; -XXH_PUBLIC_API void GF_XXH64_canonicalFromHash(GF_XXH64_canonical_t* dst, GF_XXH64_hash_t hash); -XXH_PUBLIC_API GF_XXH64_hash_t GF_XXH64_hashFromCanonical(const GF_XXH64_canonical_t* src); +typedef struct { unsigned char digest[8]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); #endif /* XXH_NO_LONG_LONG */ + #ifdef XXH_STATIC_LINKING_ONLY /* ================================================================================================ - This section contains definitions which are not guaranteed to remain stable. + This section contains declarations which are not guaranteed to remain stable. They may change in future versions, becoming incompatible with a different version of the library. - They shall only be used with static linking. - Never use these definitions in association with dynamic linking ! + These declarations should only be used with static linking. + Never use them in association with dynamic linking ! =================================================================================================== */ -/* These definitions are only meant to allow allocation of XXH state - statically, on stack, or in a struct for example. - Do not use members directly. */ - - struct XXH32_state_s { - unsigned total_len_32; - unsigned large_len; - unsigned v1; - unsigned v2; - unsigned v3; - unsigned v4; - unsigned mem32[4]; /* buffer defined as U32 for alignment */ - unsigned memsize; - unsigned reserved; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH32_state_t */ - -#ifndef XXH_NO_LONG_LONG - struct XXH64_state_s { - unsigned long long total_len; - unsigned long long v1; - unsigned long long v2; - unsigned long long v3; - unsigned long long v4; - unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ - unsigned memsize; - unsigned reserved[2]; /* never read nor write, will be removed in a future version */ - }; /* typedef'd to XXH64_state_t */ +/* These definitions are only present to allow + * static allocation of XXH state, on stack or in a struct for example. + * Never **ever** use members directly. */ + +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include <stdint.h> + +struct XXH32_state_s { + uint32_t total_len_32; + uint32_t large_len; + uint32_t v1; + uint32_t v2; + uint32_t v3; + uint32_t v4; + uint32_t mem32[4]; + uint32_t memsize; + uint32_t reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +struct XXH64_state_s { + uint64_t total_len; + uint64_t v1; + uint64_t v2; + uint64_t v3; + uint64_t v4; + uint64_t mem64[4]; + uint32_t memsize; + uint32_t reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ + +# else + +struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; + unsigned memsize; + unsigned reserved; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH32_state_t */ + +# ifndef XXH_NO_LONG_LONG /* remove 64-bit support */ +struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; + unsigned memsize; + unsigned reserved[2]; /* never read nor write, might be removed in a future version */ +}; /* typedef'd to XXH64_state_t */ +# endif + +# endif + + +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ #endif -# ifdef XXH_PRIVATE_API -# include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ -# endif - #endif /* XXH_STATIC_LINKING_ONLY */ diff --git a/contrib/xxhash/xxhsum.c b/contrib/xxhash/xxhsum.c index 6cb54386093..69931f727f0 100644 --- a/contrib/xxhash/xxhsum.c +++ b/contrib/xxhash/xxhsum.c @@ -32,8 +32,8 @@ #define XXHASH_C_2097394837 /* ************************************ -* Compiler Options -**************************************/ + * Compiler Options + **************************************/ /* MS Visual */ #if defined(_MSC_VER) || defined(_WIN32) # define _CRT_SECURE_NO_WARNINGS /* removes visual warnings */ @@ -46,28 +46,26 @@ /* ************************************ -* Includes -**************************************/ -#include <stdlib.h> /* malloc */ -#include <stdio.h> /* fprintf, fopen, ftello64, fread, stdin, stdout; when present : _fileno */ + * Includes + **************************************/ +#include <stdlib.h> /* malloc, calloc, free, exit */ +#include <stdio.h> /* fprintf, fopen, ftello64, fread, stdin, stdout, _fileno (when present) */ #include <string.h> /* strcmp */ -#include <sys/types.h> /* stat64 */ -#include <sys/stat.h> /* stat64 */ +#include <sys/types.h> /* stat, stat64, _stat64 */ +#include <sys/stat.h> /* stat, stat64, _stat64 */ #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ +#include <assert.h> /* assert */ #define XXH_STATIC_LINKING_ONLY /* *_state_t */ #include "xxhash.h" -/*-************************************ -* OS-Specific Includes -**************************************/ +/* ************************************ + * OS-Specific Includes + **************************************/ #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) # include <fcntl.h> /* _O_BINARY */ # include <io.h> /* _setmode, _isatty */ -# ifdef __MINGW32__ - int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */ -# endif # define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY) # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else @@ -110,8 +108,8 @@ static unsigned BMK_isLittleEndian(void) /* ************************************* -* Constants -***************************************/ + * Constants + ***************************************/ #define LIB_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE #define QUOTE(str) #str #define EXPAND_AND_QUOTE(str) QUOTE(str) @@ -121,18 +119,20 @@ static const char g_lename[] = "little endian"; static const char g_bename[] = "big endian"; #define ENDIAN_NAME (BMK_isLittleEndian() ? g_lename : g_bename) static const char author[] = "Yann Collet"; -#define WELCOME_MESSAGE(exename) "%s %s (%i-bits %s), by %s \n", exename, PROGRAM_VERSION, g_nbBits, ENDIAN_NAME, author +#define WELCOME_MESSAGE(exename) "%s %s (%i-bits %s), by %s \n", \ + exename, PROGRAM_VERSION, g_nbBits, ENDIAN_NAME, author + +#define KB *( 1<<10) +#define MB *( 1<<20) +#define GB *(1U<<30) +static size_t XXH_DEFAULT_SAMPLE_SIZE = 100 KB; #define NBLOOPS 3 /* Default number of benchmark iterations */ #define TIMELOOP_S 1 #define TIMELOOP (TIMELOOP_S * CLOCKS_PER_SEC) /* Minimum timing per iteration */ #define XXHSUM32_DEFAULT_SEED 0 /* Default seed for algo_xxh32 */ #define XXHSUM64_DEFAULT_SEED 0 /* Default seed for algo_xxh64 */ -#define KB *( 1<<10) -#define MB *( 1<<20) -#define GB *(1U<<30) - #define MAX_MEM (2 GB - 64 MB) static const char stdinName[] = "-"; @@ -141,31 +141,30 @@ static const algoType g_defaultAlgo = algo_xxh64; /* required within main() & /* <16 hex char> <SPC> <SPC> <filename> <'\0'> * '4096' is typical Linux PATH_MAX configuration. */ -#define DEFAULT_LINE_LENGTH (sizeof(GF_XXH64_hash_t) * 2 + 2 + 4096 + 1) +#define DEFAULT_LINE_LENGTH (sizeof(XXH64_hash_t) * 2 + 2 + 4096 + 1) /* Maximum acceptable line length. */ #define MAX_LINE_LENGTH (32 KB) /* ************************************ -* Display macros -**************************************/ -#define DISPLAY(...) /*fprintf(stderr, __VA_ARGS__)*/ + * Display macros + **************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYRESULT(...) fprintf(stdout, __VA_ARGS__) -#define DISPLAYLEVEL(l, ...) /*if (g_displayLevel>=l) DISPLAY(__VA_ARGS__);*/ -static U32 g_displayLevel = 1; +#define DISPLAYLEVEL(l, ...) do { if (g_displayLevel>=l) DISPLAY(__VA_ARGS__); } while (0) +static int g_displayLevel = 2; /* ************************************ -* Local variables -**************************************/ -static size_t g_sampleSize = 100 KB; + * Local variables + **************************************/ static U32 g_nbIterations = NBLOOPS; /* ************************************ -* Benchmark Functions -**************************************/ + * Benchmark Functions + **************************************/ static clock_t BMK_clockSpan( clock_t start ) { return clock() - start; /* works even if overflow; Typical max span ~ 30 mn */ @@ -212,59 +211,80 @@ static U64 BMK_GetFileSize(const char* infilename) typedef U32 (*hashFunction)(const void* buffer, size_t bufferSize, U32 seed); -static U32 localXXH32(const void* buffer, size_t bufferSize, U32 seed) { return GF_XXH32(buffer, bufferSize, seed); } +static U32 localXXH32(const void* buffer, size_t bufferSize, U32 seed) { return XXH32(buffer, bufferSize, seed); } -static U32 localXXH64(const void* buffer, size_t bufferSize, U32 seed) { return (U32)GF_XXH64(buffer, bufferSize, seed); } +static U32 localXXH64(const void* buffer, size_t bufferSize, U32 seed) { return (U32)XXH64(buffer, bufferSize, seed); } static void BMK_benchHash(hashFunction h, const char* hName, const void* buffer, size_t bufferSize) { - static const U32 nbh_perloop = 100; + U32 nbh_perIteration = ((300 MB) / (bufferSize+1)) + 1; /* first loop conservatively aims for 300 MB/s */ U32 iterationNb; double fastestH = 100000000.; - DISPLAY("\r%79s\r", ""); /* Clean display line */ + DISPLAYLEVEL(2, "\r%70s\r", ""); /* Clean display line */ if (g_nbIterations<1) g_nbIterations=1; for (iterationNb = 1; iterationNb <= g_nbIterations; iterationNb++) { - U32 nbHashes = 0, r=0; + U32 r=0; clock_t cStart; - DISPLAY("%1i-%-17.17s : %10u ->\r", iterationNb, hName, (U32)bufferSize); + DISPLAYLEVEL(2, "%1i-%-17.17s : %10u ->\r", iterationNb, hName, (U32)bufferSize); cStart = clock(); while (clock() == cStart); /* starts clock() at its exact beginning */ cStart = clock(); - while (BMK_clockSpan(cStart) < TIMELOOP) { - U32 i; - for (i=0; i<nbh_perloop; i++) + { U32 i; + for (i=0; i<nbh_perIteration; i++) r += h(buffer, bufferSize, i); - nbHashes += nbh_perloop; } - if (r==0) DISPLAY(".\r"); /* need to do something with r to avoid compiler optimizing away the hash function */ - { double const timeS = ((double)BMK_clockSpan(cStart) / CLOCKS_PER_SEC) / nbHashes; + if (r==0) DISPLAYLEVEL(3,".\r"); /* do something with r to avoid compiler "optimizing" away hash function */ + { double const timeS = ((double)BMK_clockSpan(cStart) / CLOCKS_PER_SEC) / nbh_perIteration; if (timeS < fastestH) fastestH = timeS; - DISPLAY("%1i-%-17.17s : %10u -> %7.1f MB/s\r", iterationNb, hName, (U32)bufferSize, ((double)bufferSize / (1<<20)) / fastestH ); + DISPLAYLEVEL(2, "%1i-%-17.17s : %10u -> %8.0f it/s (%7.1f MB/s) \r", + iterationNb, hName, (U32)bufferSize, + (double)1 / fastestH, + ((double)bufferSize / (1<<20)) / fastestH ); } + assert(fastestH > 1./2000000000); /* avoid U32 overflow */ + nbh_perIteration = (U32)(1 / fastestH) + 1; /* adjust nbh_perIteration to last roughtly one second */ } - DISPLAY("%-19.19s : %10u -> %7.1f MB/s \n", hName, (U32)bufferSize, ((double)bufferSize / (1<<20)) / fastestH); + DISPLAYLEVEL(1, "%-19.19s : %10u -> %8.0f it/s (%7.1f MB/s) \n", hName, (U32)bufferSize, + (double)1 / fastestH, + ((double)bufferSize / (1<<20)) / fastestH); + if (g_displayLevel<1) + DISPLAYLEVEL(0, "%u, ", (U32)((double)1 / fastestH)); } -/* Note : buffer is supposed malloc'ed, hence aligned */ -static void BMK_benchMem(const void* buffer, size_t bufferSize) +/* BMK_benchMem(): + * specificTest : 0 == run all tests, 1+ run only specific test + * buffer : is supposed 8-bytes aligned (if malloc'ed, it should be) + * the real allocated size of buffer is supposed to be >= (bufferSize+3). + * @return : 0 on success, 1 if error (invalid mode selected) */ +static int BMK_benchMem(const void* buffer, size_t bufferSize, U32 specificTest) { + assert((((size_t)buffer) & 8) == 0); /* ensure alignment */ + /* XXH32 bench */ - BMK_benchHash(localXXH32, "XXH32", buffer, bufferSize); + if ((specificTest==0) | (specificTest==1)) + BMK_benchHash(localXXH32, "XXH32", buffer, bufferSize); /* Bench XXH32 on Unaligned input */ - if (bufferSize>1) - BMK_benchHash(localXXH32, "XXH32 unaligned", ((const char*)buffer)+1, bufferSize-1); + if ((specificTest==0) | (specificTest==2)) + BMK_benchHash(localXXH32, "XXH32 unaligned", ((const char*)buffer)+1, bufferSize); /* Bench XXH64 */ - BMK_benchHash(localXXH64, "XXH64", buffer, bufferSize); + if ((specificTest==0) | (specificTest==3)) + BMK_benchHash(localXXH64, "XXH64", buffer, bufferSize); /* Bench XXH64 on Unaligned input */ - if (bufferSize>1) - BMK_benchHash(localXXH64, "XXH64 unaligned", ((const char*)buffer)+1, bufferSize-1); + if ((specificTest==0) | (specificTest==4)) + BMK_benchHash(localXXH64, "XXH64 unaligned", ((const char*)buffer)+3, bufferSize); + + if (specificTest > 4) { + DISPLAY("benchmark mode invalid \n"); + return 1; + } + return 0; } @@ -279,19 +299,21 @@ static size_t BMK_selectBenchedSize(const char* fileName) } -static int BMK_benchFiles(const char** fileNamesTable, int nbFiles) +static int BMK_benchFiles(const char** fileNamesTable, int nbFiles, U32 specificTest) { + int result = 0; int fileIdx; + for (fileIdx=0; fileIdx<nbFiles; fileIdx++) { const char* const inFileName = fileNamesTable[fileIdx]; FILE* const inFile = fopen( inFileName, "rb" ); size_t const benchedSize = BMK_selectBenchedSize(inFileName); - char* const buffer = (char*)malloc(benchedSize+16); - void* const alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); /* align on next 16 bytes boundaries */ + char* const buffer = (char*)calloc(benchedSize+16+3, 1); + void* const alignedBuffer = (buffer+15) - (((size_t)(buffer+15)) & 0xF); /* align on next 16 bytes */ /* Checks */ if ((inFile==NULL) || (inFileName==NULL)) { - DISPLAY( "Pb opening %s\n", inFileName); + DISPLAY("Pb opening %s\n", inFileName); free(buffer); return 11; } @@ -302,7 +324,7 @@ static int BMK_benchFiles(const char** fileNamesTable, int nbFiles) } /* Fill input buffer */ - DISPLAY("\nLoading %s... \n", inFileName); + DISPLAYLEVEL(1, "\rLoading %s... \n", inFileName); { size_t const readSize = fread(alignedBuffer, 1, benchedSize, inFile); fclose(inFile); if(readSize != benchedSize) { @@ -312,39 +334,47 @@ static int BMK_benchFiles(const char** fileNamesTable, int nbFiles) } } /* bench */ - BMK_benchMem(alignedBuffer, benchedSize); + result |= BMK_benchMem(alignedBuffer, benchedSize, specificTest); free(buffer); } - return 0; + return result; } -static int BMK_benchInternal(void) +static int BMK_benchInternal(size_t keySize, int specificTest) { - size_t const benchedSize = g_sampleSize; - void* const buffer = malloc(benchedSize); + void* const buffer = calloc(keySize+16+3, 1); + void* const alignedBuffer = ((char*)buffer+15) - (((size_t)((char*)buffer+15)) & 0xF); /* align on next 16 bytes */ if(!buffer) { DISPLAY("\nError: not enough memory!\n"); return 12; } /* bench */ - DISPLAY("\rSample of %u KB... \n", (U32)(benchedSize >> 10)); - BMK_benchMem(buffer, benchedSize); + DISPLAYLEVEL(1, "Sample of "); + if (keySize > 10 KB) { + DISPLAYLEVEL(1, "%u KB", (U32)(keySize >> 10)); + } else { + DISPLAYLEVEL(1, "%u bytes", (U32)keySize); + } + DISPLAYLEVEL(1, "... \n"); - free(buffer); - return 0; + { int const result = BMK_benchMem(alignedBuffer, keySize, specificTest); + free(buffer); + return result; + } } static void BMK_checkResult(U32 r1, U32 r2) { static int nbTests = 1; - if (r1==r2) DISPLAY("\rTest%3i : %08X == %08X ok ", nbTests, r1, r2); - else { + if (r1==r2) { + DISPLAYLEVEL(3, "\rTest%3i : %08X == %08X ok ", nbTests, r1, r2); + } else { DISPLAY("\rERROR : Test%3i : %08X <> %08X !!!!! \n", nbTests, r1, r2); exit(1); } @@ -356,7 +386,7 @@ static void BMK_checkResult64(U64 r1, U64 r2) { static int nbTests = 1; if (r1!=r2) { - DISPLAY("\rERROR : Test%3i : 64-bits values non equals !!!!! \n", nbTests); + DISPLAY("\rERROR : Test%3i : 64-bit values non equals !!!!! \n", nbTests); DISPLAY("\r %08X%08X != %08X%08X \n", (U32)(r1>>32), (U32)r1, (U32)(r2>>32), (U32)r2); exit(1); } @@ -366,42 +396,44 @@ static void BMK_checkResult64(U64 r1, U64 r2) static void BMK_testSequence64(void* sentence, size_t len, U64 seed, U64 Nresult) { - GF_XXH64_state_t state; + XXH64_state_t state; U64 Dresult; size_t pos; - Dresult = GF_XXH64(sentence, len, seed); + Dresult = XXH64(sentence, len, seed); BMK_checkResult64(Dresult, Nresult); - GF_XXH64_reset(&state, seed); - GF_XXH64_update(&state, sentence, len); - Dresult = GF_XXH64_digest(&state); + XXH64_reset(&state, seed); + XXH64_update(&state, sentence, len); + Dresult = XXH64_digest(&state); BMK_checkResult64(Dresult, Nresult); - GF_XXH64_reset(&state, seed); - for (pos=0; pos<len; pos++) GF_XXH64_update(&state, ((char*)sentence)+pos, 1); - Dresult = GF_XXH64_digest(&state); + XXH64_reset(&state, seed); + for (pos=0; pos<len; pos++) + XXH64_update(&state, ((char*)sentence)+pos, 1); + Dresult = XXH64_digest(&state); BMK_checkResult64(Dresult, Nresult); } static void BMK_testSequence(const void* sequence, size_t len, U32 seed, U32 Nresult) { - GF_XXH32_state_t state; + XXH32_state_t state; U32 Dresult; size_t pos; - Dresult = GF_XXH32(sequence, len, seed); + Dresult = XXH32(sequence, len, seed); BMK_checkResult(Dresult, Nresult); - GF_XXH32_reset(&state, seed); - GF_XXH32_update(&state, sequence, len); - Dresult = GF_XXH32_digest(&state); + XXH32_reset(&state, seed); + XXH32_update(&state, sequence, len); + Dresult = XXH32_digest(&state); BMK_checkResult(Dresult, Nresult); - GF_XXH32_reset(&state, seed); - for (pos=0; pos<len; pos++) GF_XXH32_update(&state, ((const char*)sequence)+pos, 1); - Dresult = GF_XXH32_digest(&state); + XXH32_reset(&state, seed); + for (pos=0; pos<len; pos++) + XXH32_update(&state, ((const char*)sequence)+pos, 1); + Dresult = XXH32_digest(&state); BMK_checkResult(Dresult, Nresult); } @@ -437,8 +469,8 @@ static void BMK_sanityCheck(void) BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, 0, 0x0EAB543384F878ADULL); BMK_testSequence64(sanityBuffer, SANITY_BUFFER_SIZE, prime, 0xCAA65939306F1E21ULL); - DISPLAY("\r%79s\r", ""); /* Clean display line */ - DISPLAYLEVEL(2, "Sanity check -- all tests ok\n"); + DISPLAYLEVEL(3, "\r%70s\r", ""); /* Clean display line */ + DISPLAYLEVEL(3, "Sanity check -- all tests ok\n"); } @@ -464,13 +496,13 @@ static void BMK_display_BigEndian(const void* ptr, size_t length) static void BMK_hashStream(void* xxhHashValue, const algoType hashType, FILE* inFile, void* buffer, size_t blockSize) { - GF_XXH64_state_t state64; - GF_XXH32_state_t state32; + XXH64_state_t state64; + XXH32_state_t state32; size_t readSize; /* Init */ - GF_XXH32_reset(&state32, XXHSUM32_DEFAULT_SEED); - GF_XXH64_reset(&state64, XXHSUM64_DEFAULT_SEED); + XXH32_reset(&state32, XXHSUM32_DEFAULT_SEED); + XXH64_reset(&state64, XXHSUM64_DEFAULT_SEED); /* Load file & update hash */ readSize = 1; @@ -479,10 +511,10 @@ static void BMK_hashStream(void* xxhHashValue, const algoType hashType, FILE* in switch(hashType) { case algo_xxh32: - GF_XXH32_update(&state32, buffer, readSize); + XXH32_update(&state32, buffer, readSize); break; case algo_xxh64: - GF_XXH64_update(&state64, buffer, readSize); + XXH64_update(&state64, buffer, readSize); break; default: break; @@ -492,12 +524,12 @@ static void BMK_hashStream(void* xxhHashValue, const algoType hashType, FILE* in switch(hashType) { case algo_xxh32: - { U32 const h32 = GF_XXH32_digest(&state32); + { U32 const h32 = XXH32_digest(&state32); memcpy(xxhHashValue, &h32, sizeof(h32)); break; } case algo_xxh64: - { U64 const h64 = GF_XXH64_digest(&state64); + { U64 const h64 = XXH64_digest(&state64); memcpy(xxhHashValue, &h64, sizeof(h64)); break; } @@ -542,45 +574,46 @@ static int BMK_hash(const char* fileName, /* loading notification */ { const size_t fileNameSize = strlen(fileName); const char* const fileNameEnd = fileName + fileNameSize; - const size_t maxInfoFilenameSize = fileNameSize > 30 ? 30 : fileNameSize; - size_t infoFilenameSize = 1; - while ( (infoFilenameSize < maxInfoFilenameSize) - &&(fileNameEnd[-1-infoFilenameSize] != '/') - &&(fileNameEnd[-1-infoFilenameSize] != '\\') ) + const int maxInfoFilenameSize = (int)(fileNameSize > 30 ? 30 : fileNameSize); + int infoFilenameSize = 1; + while ((infoFilenameSize < maxInfoFilenameSize) + && (fileNameEnd[-1-infoFilenameSize] != '/') + && (fileNameEnd[-1-infoFilenameSize] != '\\') ) infoFilenameSize++; - DISPLAY("\nLoading %s... \n", fileNameEnd - infoFilenameSize); - } + DISPLAY("\rLoading %s... \r", fileNameEnd - infoFilenameSize); - /* Load file & update hash */ - switch(hashType) - { - case algo_xxh32: - BMK_hashStream(&h32, algo_xxh32, inFile, buffer, blockSize); - break; - case algo_xxh64: - BMK_hashStream(&h64, algo_xxh64, inFile, buffer, blockSize); - break; - default: - break; - } + /* Load file & update hash */ + switch(hashType) + { + case algo_xxh32: + BMK_hashStream(&h32, algo_xxh32, inFile, buffer, blockSize); + break; + case algo_xxh64: + BMK_hashStream(&h64, algo_xxh64, inFile, buffer, blockSize); + break; + default: + break; + } - fclose(inFile); - free(buffer); + fclose(inFile); + free(buffer); + DISPLAY("%s \r", fileNameEnd - infoFilenameSize); /* erase line */ + } /* display Hash */ switch(hashType) { case algo_xxh32: - { GF_XXH32_canonical_t hcbe32; - GF_XXH32_canonicalFromHash(&hcbe32, h32); + { XXH32_canonical_t hcbe32; + XXH32_canonicalFromHash(&hcbe32, h32); displayEndianess==big_endian ? BMK_display_BigEndian(&hcbe32, sizeof(hcbe32)) : BMK_display_LittleEndian(&hcbe32, sizeof(hcbe32)); DISPLAYRESULT(" %s\n", fileName); break; } case algo_xxh64: - { GF_XXH64_canonical_t hcbe64; - GF_XXH64_canonicalFromHash(&hcbe64, h64); + { XXH64_canonical_t hcbe64; + XXH64_canonicalFromHash(&hcbe64, h64); displayEndianess==big_endian ? BMK_display_BigEndian(&hcbe64, sizeof(hcbe64)) : BMK_display_LittleEndian(&hcbe64, sizeof(hcbe64)); DISPLAYRESULT(" %s\n", fileName); @@ -634,8 +667,8 @@ typedef enum { } LineStatus; typedef union { - GF_XXH32_canonical_t xxh32; - GF_XXH64_canonical_t xxh64; + XXH32_canonical_t xxh32; + XXH64_canonical_t xxh64; } Canonical; typedef struct { @@ -680,10 +713,12 @@ static GetLineResult getLine(char** lineBuf, int* lineMax, FILE* inFile) GetLineResult result = GetLine_ok; int len = 0; - if (*lineBuf == NULL || *lineMax < 1) { - *lineMax = DEFAULT_LINE_LENGTH; - *lineBuf = (char*) realloc(*lineBuf, *lineMax); + if ((*lineBuf == NULL) || (*lineMax<1)) { + free(*lineBuf); /* in case it's != NULL */ + *lineMax = 0; + *lineBuf = (char*)malloc(DEFAULT_LINE_LENGTH); if(*lineBuf == NULL) return GetLine_outOfMemory; + *lineMax = DEFAULT_LINE_LENGTH; } for (;;) { @@ -788,7 +823,7 @@ static ParseLineResult parseLine(ParsedLine* parsedLine, const char* line) switch (firstSpace - line) { case 8: - { GF_XXH32_canonical_t* xxh32c = &parsedLine->canonical.xxh32; + { XXH32_canonical_t* xxh32c = &parsedLine->canonical.xxh32; if (canonicalFromString(xxh32c->digest, sizeof(xxh32c->digest), line) != CanonicalFromString_ok) { return ParseLine_invalidFormat; @@ -798,7 +833,7 @@ static ParseLineResult parseLine(ParsedLine* parsedLine, const char* line) } case 16: - { GF_XXH64_canonical_t* xxh64c = &parsedLine->canonical.xxh64; + { XXH64_canonical_t* xxh64c = &parsedLine->canonical.xxh64; if (canonicalFromString(xxh64c->digest, sizeof(xxh64c->digest), line) != CanonicalFromString_ok) { return ParseLine_invalidFormat; @@ -905,17 +940,17 @@ static void parseFile1(ParseFileArg* parseFileArg) switch (parsedLine.xxhBits) { case 32: - { GF_XXH32_hash_t xxh; + { XXH32_hash_t xxh; BMK_hashStream(&xxh, algo_xxh32, fp, parseFileArg->blockBuf, parseFileArg->blockSize); - if (xxh == GF_XXH32_hashFromCanonical(&parsedLine.canonical.xxh32)) { + if (xxh == XXH32_hashFromCanonical(&parsedLine.canonical.xxh32)) { lineStatus = LineStatus_hashOk; } } break; case 64: - { GF_XXH64_hash_t xxh; + { XXH64_hash_t xxh; BMK_hashStream(&xxh, algo_xxh64, fp, parseFileArg->blockBuf, parseFileArg->blockSize); - if (xxh == GF_XXH64_hashFromCanonical(&parsedLine.canonical.xxh64)) { + if (xxh == XXH64_hashFromCanonical(&parsedLine.canonical.xxh64)) { lineStatus = LineStatus_hashOk; } } break; @@ -1122,10 +1157,30 @@ static int badusage(const char* exename) return 1; } +/*! readU32FromChar() : + @return : unsigned integer value read from input in `char` format, + 0 is no figure at *stringPtr position. + Interprets K, KB, KiB, M, MB and MiB suffix. + Modifies `*stringPtr`, advancing it to position where reading stopped. + Note : function result can overflow if digit string > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} int main(int argc, const char** argv) { - int i, filenamesStart=0; + int i, filenamesStart = 0; const char* const exename = argv[0]; U32 benchmarkMode = 0; U32 fileCheckMode = 0; @@ -1133,7 +1188,9 @@ int main(int argc, const char** argv) U32 statusOnly = 0; U32 warn = 0; U32 quiet = 0; - algoType algo = g_defaultAlgo; + U32 specificTest = 0; + size_t keySize = XXH_DEFAULT_SAMPLE_SIZE; + algoType algo = g_defaultAlgo; endianess displayEndianess = big_endian; /* special case : xxh32sum default to 32 bits checksum */ @@ -1193,21 +1250,26 @@ int main(int argc, const char** argv) /* Trigger benchmark mode */ case 'b': argument++; - benchmarkMode=1; + benchmarkMode = 1; + specificTest = readU32FromChar(&argument); /* select one specific test (hidden option) */ break; /* Modify Nb Iterations (benchmark only) */ case 'i': - g_nbIterations = argument[1] - '0'; - argument+=2; + argument++; + g_nbIterations = readU32FromChar(&argument); break; /* Modify Block size (benchmark only) */ case 'B': argument++; - g_sampleSize = 0; - while (argument[0]>='0' && argument[0]<='9') - g_sampleSize *= 10, g_sampleSize += argument[0]-'0', argument++; + keySize = readU32FromChar(&argument); + break; + + /* Modify verbosity of benchmark output (hidden option) */ + case 'q': + argument++; + g_displayLevel--; break; default: @@ -1218,10 +1280,10 @@ int main(int argc, const char** argv) /* Check benchmark mode */ if (benchmarkMode) { - DISPLAY( WELCOME_MESSAGE(exename) ); + DISPLAYLEVEL(2, WELCOME_MESSAGE(exename) ); BMK_sanityCheck(); - if (filenamesStart==0) return BMK_benchInternal(); - return BMK_benchFiles(argv+filenamesStart, argc-filenamesStart); + if (filenamesStart==0) return BMK_benchInternal(keySize, specificTest); + return BMK_benchFiles(argv+filenamesStart, argc-filenamesStart, specificTest); } /* Check if input is defined as console; trigger an error in this case */ @@ -1229,7 +1291,8 @@ int main(int argc, const char** argv) if (filenamesStart==0) filenamesStart = argc; if (fileCheckMode) { - return checkFiles(argv+filenamesStart, argc-filenamesStart, displayEndianess, strictMode, statusOnly, warn, quiet); + return checkFiles(argv+filenamesStart, argc-filenamesStart, + displayEndianess, strictMode, statusOnly, warn, quiet); } else { return BMK_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianess); } diff --git a/libglusterfs/src/Makefile.am b/libglusterfs/src/Makefile.am index c7cd3d5e6fc..92521cc44d1 100644 --- a/libglusterfs/src/Makefile.am +++ b/libglusterfs/src/Makefile.am @@ -6,6 +6,7 @@ libglusterfs_la_CFLAGS = $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \ libglusterfs_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 \ -DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \ -DXLATORPARENTDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)\" \ + -DXXH_NAMESPACE=GF_ \ -I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/ \ -I$(top_srcdir)/rpc/rpc-lib/src/ -I$(CONTRIBDIR)/rbtree \ -I$(CONTRIBDIR)/libexecinfo ${ARGP_STANDALONE_CPPFLAGS} \ diff --git a/libglusterfs/src/common-utils.c b/libglusterfs/src/common-utils.c index 7414691c2c7..543f8601ebe 100644 --- a/libglusterfs/src/common-utils.c +++ b/libglusterfs/src/common-utils.c @@ -49,6 +49,7 @@ #include "lkowner.h" #include "syscall.h" #include "cli1-xdr.h" +#define XXH_INLINE_ALL #include "xxhash.h" #include <ifaddrs.h> #include "libglusterfs-messages.h" @@ -78,17 +79,17 @@ typedef int32_t (*rw_op_t)(int32_t fd, char *buf, int32_t size); typedef int32_t (*rwv_op_t)(int32_t fd, const struct iovec *buf, int32_t size); void -gf_xxh64_wrapper(const unsigned char *data, size_t len, unsigned long long seed, +gf_xxh64_wrapper(const unsigned char *data, size_t const len, unsigned long long const seed, char *xxh64) { unsigned short i = 0; - unsigned short lim = GF_XXH64_DIGEST_LENGTH*2+1; - GF_XXH64_hash_t hash = 0; - GF_XXH64_canonical_t c_hash = {{0,},}; - const uint8_t *p = (const uint8_t *) &c_hash; + const unsigned short lim = GF_XXH64_DIGEST_LENGTH*2+1; + XXH64_hash_t hash = 0; + XXH64_canonical_t c_hash = {{0,},}; + const uint8_t *p = (const uint8_t *) &c_hash; - hash = GF_XXH64(data, len, seed); - GF_XXH64_canonicalFromHash(&c_hash, hash); + hash = XXH64(data, len, seed); + XXH64_canonicalFromHash(&c_hash, hash); for (i = 0; i < GF_XXH64_DIGEST_LENGTH; i++) snprintf(xxh64 + i * 2, lim-i*2, "%02x", p[i]); diff --git a/libglusterfs/src/common-utils.h b/libglusterfs/src/common-utils.h index 3217c16a82e..55ef916ddf7 100644 --- a/libglusterfs/src/common-utils.h +++ b/libglusterfs/src/common-utils.h @@ -857,8 +857,8 @@ gf_ports_reserved (char *blocked_port, unsigned char *ports, uint32_t ceiling); int gf_get_hostname_from_ip (char *client_ip, char **hostname); gf_boolean_t gf_is_local_addr (char *hostname); gf_boolean_t gf_is_same_address (char *host1, char *host2); -void gf_xxh64_wrapper(const unsigned char *data, size_t len, - unsigned long long seed, char *xxh64); +void gf_xxh64_wrapper(const unsigned char *data, size_t const len, + unsigned long long const seed, char *xxh64); int gf_set_timestamp (const char *src, const char* dest); int gf_thread_create (pthread_t *thread, const pthread_attr_t *attr, diff --git a/tests/gfid2path/gfid2path_fuse.t b/tests/gfid2path/gfid2path_fuse.t index c7e79466673..d0fe1fc16ae 100644 --- a/tests/gfid2path/gfid2path_fuse.t +++ b/tests/gfid2path/gfid2path_fuse.t @@ -47,7 +47,7 @@ TEST $CLI_SETGFID2PATH $backpath #Check for the presence of xattr pgfid_bname=$pgfid/before_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath @@ -62,7 +62,7 @@ backpath=$B0/${V0}1/file1 #Check for the presence of xattr pgfid_bname=$pgfid/file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath @@ -74,7 +74,7 @@ backpath=$B0/${V0}1/mknod_file1 #Check for the presence of xattr pgfid_bname=$pgfid/mknod_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath @@ -88,13 +88,13 @@ backpath2=$B0/${V0}1/hl_file1 #Check for the presence of two xattrs pgfid_bname=$pgfid/file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath1 pgfid_bname=$pgfid/hl_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath2 @@ -107,13 +107,13 @@ backpath=$B0/${V0}1/rn_file1 #Check for the presence of new xattr pgfid_bname=$pgfid/file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT_NOT $pgfid_bname get_text_xattr $key $backpath pgfid_bname=$pgfid/rn_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath @@ -126,13 +126,13 @@ backpath=$B0/${V0}1/rn_file1 #Check removal of xattr pgfid_bname=$pgfid/hl_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT_NOT $pgfid_bname get_text_xattr $key $backpath pgfid_bname=$pgfid/rn_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath @@ -145,7 +145,7 @@ backpath=$B0/${V0}1/sym_file1 #Check for the presence of xattr pgfid_bname=$pgfid/sym_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath diff --git a/tests/gfid2path/gfid2path_nfs.t b/tests/gfid2path/gfid2path_nfs.t index 239dafd46fb..969aa484b82 100644 --- a/tests/gfid2path/gfid2path_nfs.t +++ b/tests/gfid2path/gfid2path_nfs.t @@ -46,7 +46,7 @@ backpath=$B0/${V0}1/file1 #Check for the presence of xattr pgfid_bname=$pgfid/file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath @@ -58,7 +58,7 @@ backpath=$B0/${V0}1/mknod_file1 #Check for the presence of xattr pgfid_bname=$pgfid/mknod_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath @@ -72,13 +72,13 @@ backpath2=$B0/${V0}1/hl_file1 #Check for the presence of two xattrs pgfid_bname=$pgfid/file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath1 pgfid_bname=$pgfid/hl_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath2 @@ -91,13 +91,13 @@ backpath=$B0/${V0}1/rn_file1 #Check for the presence of new xattr pgfid_bname=$pgfid/file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT_NOT $pgfid_bname get_text_xattr $key $backpath pgfid_bname=$pgfid/rn_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath @@ -110,13 +110,13 @@ backpath=$B0/${V0}1/rn_file1 #Check removal of xattr pgfid_bname=$pgfid/hl_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT_NOT $pgfid_bname get_text_xattr $key $backpath pgfid_bname=$pgfid/rn_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath @@ -129,7 +129,7 @@ backpath=$B0/${V0}1/sym_file1 #Check for the presence of xattr pgfid_bname=$pgfid/sym_file1 echo -n $pgfid_bname > $xxh64_file -xxh64sum=$($XXHSUM_EXEC $xxh64_file | awk '{print $1}') +xxh64sum=$(($XXHSUM_EXEC $xxh64_file) 2>/dev/null | awk '{print $1}') key="trusted.gfid2path.$xxh64sum" EXPECT $pgfid_bname get_text_xattr $key $backpath |