1/*
2 * xxHash - Extremely Fast Hash algorithm
3 * Header File
4 * Copyright (C) 2012-2023 Yann Collet
5 *
6 * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above
15 * copyright notice, this list of conditions and the following disclaimer
16 * in the documentation and/or other materials provided with the
17 * distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * You can contact the author at:
32 * - xxHash homepage: https://www.xxhash.com
33 * - xxHash source repository: https://github.com/Cyan4973/xxHash
34 */
35
36/*!
37 * @mainpage xxHash
38 *
39 * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed
40 * limits.
41 *
42 * It is proposed in four flavors, in three families:
43 * 1. @ref XXH32_family
44 * - Classic 32-bit hash function. Simple, compact, and runs on almost all
45 * 32-bit and 64-bit systems.
46 * 2. @ref XXH64_family
47 * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most
48 * 64-bit systems (but _not_ 32-bit systems).
49 * 3. @ref XXH3_family
50 * - Modern 64-bit and 128-bit hash function family which features improved
51 * strength and performance across the board, especially on smaller data.
52 * It benefits greatly from SIMD and 64-bit without requiring it.
53 *
54 * Benchmarks
55 * ---
56 * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04.
57 * The open source benchmark program is compiled with clang v10.0 using -O3 flag.
58 *
59 * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity |
60 * | -------------------- | ------- | ----: | ---------------: | ------------------: |
61 * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 |
62 * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 |
63 * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 |
64 * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 |
65 * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 |
66 * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 |
67 * | RAM sequential read | | N/A | 28.0 GB/s | N/A |
68 * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 |
69 * | City64 | | 64 | 22.0 GB/s | 76.6 |
70 * | T1ha2 | | 64 | 22.0 GB/s | 99.0 |
71 * | City128 | | 128 | 21.7 GB/s | 57.7 |
72 * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 |
73 * | XXH64() | | 64 | 19.4 GB/s | 71.0 |
74 * | SpookyHash | | 64 | 19.3 GB/s | 53.2 |
75 * | Mum | | 64 | 18.0 GB/s | 67.0 |
76 * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 |
77 * | XXH32() | | 32 | 9.7 GB/s | 71.9 |
78 * | City32 | | 32 | 9.1 GB/s | 66.0 |
79 * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 |
80 * | Murmur3 | | 32 | 3.9 GB/s | 56.1 |
81 * | SipHash* | | 64 | 3.0 GB/s | 43.2 |
82 * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 |
83 * | HighwayHash | | 64 | 1.4 GB/s | 6.0 |
84 * | FNV64 | | 64 | 1.2 GB/s | 62.7 |
85 * | Blake2* | | 256 | 1.1 GB/s | 5.1 |
86 * | SHA1* | | 160 | 0.8 GB/s | 5.6 |
87 * | MD5* | | 128 | 0.6 GB/s | 7.8 |
88 * @note
89 * - Hashes which require a specific ISA extension are noted. SSE2 is also noted,
90 * even though it is mandatory on x64.
91 * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic
92 * by modern standards.
93 * - Small data velocity is a rough average of algorithm's efficiency for small
94 * data. For more accurate information, see the wiki.
95 * - More benchmarks and strength tests are found on the wiki:
96 * https://github.com/Cyan4973/xxHash/wiki
97 *
98 * Usage
99 * ------
100 * All xxHash variants use a similar API. Changing the algorithm is a trivial
101 * substitution.
102 *
103 * @pre
104 * For functions which take an input and length parameter, the following
105 * requirements are assumed:
106 * - The range from [`input`, `input + length`) is valid, readable memory.
107 * - The only exception is if the `length` is `0`, `input` may be `NULL`.
108 * - For C++, the objects must have the *TriviallyCopyable* property, as the
109 * functions access bytes directly as if it was an array of `unsigned char`.
110 *
111 * @anchor single_shot_example
112 * **Single Shot**
113 *
114 * These functions are stateless functions which hash a contiguous block of memory,
115 * immediately returning the result. They are the easiest and usually the fastest
116 * option.
117 *
118 * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits()
119 *
120 * @code{.c}
121 * #include <string.h>
122 * #include "xxhash.h"
123 *
124 * // Example for a function which hashes a null terminated string with XXH32().
125 * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed)
126 * {
127 * // NULL pointers are only valid if the length is zero
128 * size_t length = (string == NULL) ? 0 : strlen(string);
129 * return XXH32(string, length, seed);
130 * }
131 * @endcode
132 *
133 *
134 * @anchor streaming_example
135 * **Streaming**
136 *
137 * These groups of functions allow incremental hashing of unknown size, even
138 * more than what would fit in a size_t.
139 *
140 * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset()
141 *
142 * @code{.c}
143 * #include <stdio.h>
144 * #include <assert.h>
145 * #include "xxhash.h"
146 * // Example for a function which hashes a FILE incrementally with XXH3_64bits().
147 * XXH64_hash_t hashFile(FILE* f)
148 * {
149 * // Allocate a state struct. Do not just use malloc() or new.
150 * XXH3_state_t* state = XXH3_createState();
151 * assert(state != NULL && "Out of memory!");
152 * // Reset the state to start a new hashing session.
153 * XXH3_64bits_reset(state);
154 * char buffer[4096];
155 * size_t count;
156 * // Read the file in chunks
157 * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) {
158 * // Run update() as many times as necessary to process the data
159 * XXH3_64bits_update(state, buffer, count);
160 * }
161 * // Retrieve the finalized hash. This will not change the state.
162 * XXH64_hash_t result = XXH3_64bits_digest(state);
163 * // Free the state. Do not use free().
164 * XXH3_freeState(state);
165 * return result;
166 * }
167 * @endcode
168 *
169 * Streaming functions generate the xxHash value from an incremental input.
170 * This method is slower than single-call functions, due to state management.
171 * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
172 *
173 * An XXH state must first be allocated using `XXH*_createState()`.
174 *
175 * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
176 *
177 * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
178 *
179 * The function returns an error code, with 0 meaning OK, and any other value
180 * meaning there is an error.
181 *
182 * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
183 * This function returns the nn-bits hash as an int or long long.
184 *
185 * It's still possible to continue inserting input into the hash state after a
186 * digest, and generate new hash values later on by invoking `XXH*_digest()`.
187 *
188 * When done, release the state using `XXH*_freeState()`.
189 *
190 *
191 * @anchor canonical_representation_example
192 * **Canonical Representation**
193 *
194 * The default return values from XXH functions are unsigned 32, 64 and 128 bit
195 * integers.
196 * This the simplest and fastest format for further post-processing.
197 *
198 * However, this leaves open the question of what is the order on the byte level,
199 * since little and big endian conventions will store the same number differently.
200 *
201 * The canonical representation settles this issue by mandating big-endian
202 * convention, the same convention as human-readable numbers (large digits first).
203 *
204 * When writing hash values to storage, sending them over a network, or printing
205 * them, it's highly recommended to use the canonical representation to ensure
206 * portability across a wider range of systems, present and future.
207 *
208 * The following functions allow transformation of hash values to and from
209 * canonical format.
210 *
211 * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(),
212 * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(),
213 * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(),
214 *
215 * @code{.c}
216 * #include <stdio.h>
217 * #include "xxhash.h"
218 *
219 * // Example for a function which prints XXH32_hash_t in human readable format
220 * void printXxh32(XXH32_hash_t hash)
221 * {
222 * XXH32_canonical_t cano;
223 * XXH32_canonicalFromHash(&cano, hash);
224 * size_t i;
225 * for(i = 0; i < sizeof(cano.digest); ++i) {
226 * printf("%02x", cano.digest[i]);
227 * }
228 * printf("\n");
229 * }
230 *
231 * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t
232 * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano)
233 * {
234 * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano);
235 * return hash;
236 * }
237 * @endcode
238 *
239 *
240 * @file xxhash.h
241 * xxHash prototypes and implementation
242 */
243
244#if defined (__cplusplus)
245extern "C" {
246#endif
247
248/* ****************************
249 * INLINE mode
250 ******************************/
251/*!
252 * @defgroup public Public API
253 * Contains details on the public xxHash functions.
254 * @{
255 */
256#ifdef XXH_DOXYGEN
257/*!
258 * @brief Gives access to internal state declaration, required for static allocation.
259 *
260 * Incompatible with dynamic linking, due to risks of ABI changes.
261 *
262 * Usage:
263 * @code{.c}
264 * #define XXH_STATIC_LINKING_ONLY
265 * #include "xxhash.h"
266 * @endcode
267 */
268# define XXH_STATIC_LINKING_ONLY
269/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */
270
271/*!
272 * @brief Gives access to internal definitions.
273 *
274 * Usage:
275 * @code{.c}
276 * #define XXH_STATIC_LINKING_ONLY
277 * #define XXH_IMPLEMENTATION
278 * #include "xxhash.h"
279 * @endcode
280 */
281# define XXH_IMPLEMENTATION
282/* Do not undef XXH_IMPLEMENTATION for Doxygen */
283
284/*!
285 * @brief Exposes the implementation and marks all functions as `inline`.
286 *
287 * Use these build macros to inline xxhash into the target unit.
288 * Inlining improves performance on small inputs, especially when the length is
289 * expressed as a compile-time constant:
290 *
291 * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
292 *
293 * It also keeps xxHash symbols private to the unit, so they are not exported.
294 *
295 * Usage:
296 * @code{.c}
297 * #define XXH_INLINE_ALL
298 * #include "xxhash.h"
299 * @endcode
300 * Do not compile and link xxhash.o as a separate object, as it is not useful.
301 */
302# define XXH_INLINE_ALL
303# undef XXH_INLINE_ALL
304/*!
305 * @brief Exposes the implementation without marking functions as inline.
306 */
307# define XXH_PRIVATE_API
308# undef XXH_PRIVATE_API
309/*!
310 * @brief Emulate a namespace by transparently prefixing all symbols.
311 *
312 * If you want to include _and expose_ xxHash functions from within your own
313 * library, but also want to avoid symbol collisions with other libraries which
314 * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix
315 * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE
316 * (therefore, avoid empty or numeric values).
317 *
318 * Note that no change is required within the calling program as long as it
319 * includes `xxhash.h`: Regular symbol names will be automatically translated
320 * by this header.
321 */
322# define XXH_NAMESPACE /* YOUR NAME HERE */
323# undef XXH_NAMESPACE
324#endif
325
326#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
327 && !defined(XXH_INLINE_ALL_31684351384)
328 /* this section should be traversed only once */
329# define XXH_INLINE_ALL_31684351384
330 /* give access to the advanced API, required to compile implementations */
331# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */
332# define XXH_STATIC_LINKING_ONLY
333 /* make all functions private */
334# undef XXH_PUBLIC_API
335# if defined(__GNUC__)
336# define XXH_PUBLIC_API static __inline __attribute__((__unused__))
337# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
338# define XXH_PUBLIC_API static inline
339# elif defined(_MSC_VER)
340# define XXH_PUBLIC_API static __inline
341# else
342 /* note: this version may generate warnings for unused static functions */
343# define XXH_PUBLIC_API static
344# endif
345
346 /*
347 * This part deals with the special case where a unit wants to inline xxHash,
348 * but "xxhash.h" has previously been included without XXH_INLINE_ALL,
349 * such as part of some previously included *.h header file.
350 * Without further action, the new include would just be ignored,
351 * and functions would effectively _not_ be inlined (silent failure).
352 * The following macros solve this situation by prefixing all inlined names,
353 * avoiding naming collision with previous inclusions.
354 */
355 /* Before that, we unconditionally #undef all symbols,
356 * in case they were already defined with XXH_NAMESPACE.
357 * They will then be redefined for XXH_INLINE_ALL
358 */
359# undef XXH_versionNumber
360 /* XXH32 */
361# undef XXH32
362# undef XXH32_createState
363# undef XXH32_freeState
364# undef XXH32_reset
365# undef XXH32_update
366# undef XXH32_digest
367# undef XXH32_copyState
368# undef XXH32_canonicalFromHash
369# undef XXH32_hashFromCanonical
370 /* XXH64 */
371# undef XXH64
372# undef XXH64_createState
373# undef XXH64_freeState
374# undef XXH64_reset
375# undef XXH64_update
376# undef XXH64_digest
377# undef XXH64_copyState
378# undef XXH64_canonicalFromHash
379# undef XXH64_hashFromCanonical
380 /* XXH3_64bits */
381# undef XXH3_64bits
382# undef XXH3_64bits_withSecret
383# undef XXH3_64bits_withSeed
384# undef XXH3_64bits_withSecretandSeed
385# undef XXH3_createState
386# undef XXH3_freeState
387# undef XXH3_copyState
388# undef XXH3_64bits_reset
389# undef XXH3_64bits_reset_withSeed
390# undef XXH3_64bits_reset_withSecret
391# undef XXH3_64bits_update
392# undef XXH3_64bits_digest
393# undef XXH3_generateSecret
394 /* XXH3_128bits */
395# undef XXH128
396# undef XXH3_128bits
397# undef XXH3_128bits_withSeed
398# undef XXH3_128bits_withSecret
399# undef XXH3_128bits_reset
400# undef XXH3_128bits_reset_withSeed
401# undef XXH3_128bits_reset_withSecret
402# undef XXH3_128bits_reset_withSecretandSeed
403# undef XXH3_128bits_update
404# undef XXH3_128bits_digest
405# undef XXH128_isEqual
406# undef XXH128_cmp
407# undef XXH128_canonicalFromHash
408# undef XXH128_hashFromCanonical
409 /* Finally, free the namespace itself */
410# undef XXH_NAMESPACE
411
412 /* employ the namespace for XXH_INLINE_ALL */
413# define XXH_NAMESPACE XXH_INLINE_
414 /*
415 * Some identifiers (enums, type names) are not symbols,
416 * but they must nonetheless be renamed to avoid redeclaration.
417 * Alternative solution: do not redeclare them.
418 * However, this requires some #ifdefs, and has a more dispersed impact.
419 * Meanwhile, renaming can be achieved in a single place.
420 */
421# define XXH_IPREF(Id) XXH_NAMESPACE ## Id
422# define XXH_OK XXH_IPREF(XXH_OK)
423# define XXH_ERROR XXH_IPREF(XXH_ERROR)
424# define XXH_errorcode XXH_IPREF(XXH_errorcode)
425# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t)
426# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t)
427# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
428# define XXH32_state_s XXH_IPREF(XXH32_state_s)
429# define XXH32_state_t XXH_IPREF(XXH32_state_t)
430# define XXH64_state_s XXH_IPREF(XXH64_state_s)
431# define XXH64_state_t XXH_IPREF(XXH64_state_t)
432# define XXH3_state_s XXH_IPREF(XXH3_state_s)
433# define XXH3_state_t XXH_IPREF(XXH3_state_t)
434# define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
435 /* Ensure the header is parsed again, even if it was previously included */
436# undef XXHASH_H_5627135585666179
437# undef XXHASH_H_STATIC_13879238742
438#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
439
440/* ****************************************************************
441 * Stable API
442 *****************************************************************/
443#ifndef XXHASH_H_5627135585666179
444#define XXHASH_H_5627135585666179 1
445
446/*! @brief Marks a global symbol. */
447#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
448# if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
449# ifdef XXH_EXPORT
450# define XXH_PUBLIC_API __declspec(dllexport)
451# elif XXH_IMPORT
452# define XXH_PUBLIC_API __declspec(dllimport)
453# endif
454# else
455# define XXH_PUBLIC_API /* do nothing */
456# endif
457#endif
458
459#ifdef XXH_NAMESPACE
460# define XXH_CAT(A,B) A##B
461# define XXH_NAME2(A,B) XXH_CAT(A,B)
462# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
463/* XXH32 */
464# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
465# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
466# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
467# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
468# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
469# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
470# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
471# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
472# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
473/* XXH64 */
474# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
475# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
476# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
477# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
478# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
479# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
480# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
481# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
482# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
483/* XXH3_64bits */
484# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
485# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
486# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
487# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
488# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
489# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
490# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
491# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
492# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
493# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
494# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
495# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
496# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
497# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
498# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
499/* XXH3_128bits */
500# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
501# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
502# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
503# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
504# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
505# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
506# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
507# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
508# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
509# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
510# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
511# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
512# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
513# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
514# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
515#endif
516
517
518/* *************************************
519* Compiler specifics
520***************************************/
521
522/* specific declaration modes for Windows */
523#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
524# if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
525# ifdef XXH_EXPORT
526# define XXH_PUBLIC_API __declspec(dllexport)
527# elif XXH_IMPORT
528# define XXH_PUBLIC_API __declspec(dllimport)
529# endif
530# else
531# define XXH_PUBLIC_API /* do nothing */
532# endif
533#endif
534
535#if defined (__GNUC__)
536# define XXH_CONSTF __attribute__((__const__))
537# define XXH_PUREF __attribute__((__pure__))
538# define XXH_MALLOCF __attribute__((__malloc__))
539#else
540# define XXH_CONSTF /* disable */
541# define XXH_PUREF
542# define XXH_MALLOCF
543#endif
544
545/* *************************************
546* Version
547***************************************/
548#define XXH_VERSION_MAJOR 0
549#define XXH_VERSION_MINOR 8
550#define XXH_VERSION_RELEASE 3
551/*! @brief Version number, encoded as two digits each */
552#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
553
554/*!
555 * @brief Obtains the xxHash version.
556 *
557 * This is mostly useful when xxHash is compiled as a shared library,
558 * since the returned value comes from the library, as opposed to header file.
559 *
560 * @return @ref XXH_VERSION_NUMBER of the invoked library.
561 */
562XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void);
563
564
565/* ****************************
566* Common basic types
567******************************/
568#include <stddef.h> /* size_t */
569/*!
570 * @brief Exit code for the streaming API.
571 */
572typedef enum {
573 XXH_OK = 0, /*!< OK */
574 XXH_ERROR /*!< Error */
575} XXH_errorcode;
576
577
578/*-**********************************************************************
579* 32-bit hash
580************************************************************************/
581#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
582/*!
583 * @brief An unsigned 32-bit integer.
584 *
585 * Not necessarily defined to `uint32_t` but functionally equivalent.
586 */
587typedef uint32_t XXH32_hash_t;
588
589#elif !defined (__VMS) \
590 && (defined (__cplusplus) \
591 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
592# ifdef _AIX
593# include <inttypes.h>
594# else
595# include <stdint.h>
596# endif
597 typedef uint32_t XXH32_hash_t;
598
599#else
600# include <limits.h>
601# if UINT_MAX == 0xFFFFFFFFUL
602 typedef unsigned int XXH32_hash_t;
603# elif ULONG_MAX == 0xFFFFFFFFUL
604 typedef unsigned long XXH32_hash_t;
605# else
606# error "unsupported platform: need a 32-bit type"
607# endif
608#endif
609
610/*!
611 * @}
612 *
613 * @defgroup XXH32_family XXH32 family
614 * @ingroup public
615 * Contains functions used in the classic 32-bit xxHash algorithm.
616 *
617 * @note
618 * XXH32 is useful for older platforms, with no or poor 64-bit performance.
619 * Note that the @ref XXH3_family provides competitive speed for both 32-bit
620 * and 64-bit systems, and offers true 64/128 bit hash results.
621 *
622 * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families
623 * @see @ref XXH32_impl for implementation details
624 * @{
625 */
626
627/*!
628 * @brief Calculates the 32-bit hash of @p input using xxHash32.
629 *
630 * @param input The block of data to be hashed, at least @p length bytes in size.
631 * @param length The length of @p input, in bytes.
632 * @param seed The 32-bit seed to alter the hash's output predictably.
633 *
634 * @pre
635 * The memory between @p input and @p input + @p length must be valid,
636 * readable, contiguous memory. However, if @p length is `0`, @p input may be
637 * `NULL`. In C++, this also must be *TriviallyCopyable*.
638 *
639 * @return The calculated 32-bit xxHash32 value.
640 *
641 * @see @ref single_shot_example "Single Shot Example" for an example.
642 */
643XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
644
645#ifndef XXH_NO_STREAM
646/*!
647 * @typedef struct XXH32_state_s XXH32_state_t
648 * @brief The opaque state struct for the XXH32 streaming API.
649 *
650 * @see XXH32_state_s for details.
651 * @see @ref streaming_example "Streaming Example"
652 */
653typedef struct XXH32_state_s XXH32_state_t;
654
655/*!
656 * @brief Allocates an @ref XXH32_state_t.
657 *
658 * @return An allocated pointer of @ref XXH32_state_t on success.
659 * @return `NULL` on failure.
660 *
661 * @note Must be freed with XXH32_freeState().
662 *
663 * @see @ref streaming_example "Streaming Example"
664 */
665XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void);
666/*!
667 * @brief Frees an @ref XXH32_state_t.
668 *
669 * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
670 *
671 * @return @ref XXH_OK.
672 *
673 * @note @p statePtr must be allocated with XXH32_createState().
674 *
675 * @see @ref streaming_example "Streaming Example"
676 *
677 */
678XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
679/*!
680 * @brief Copies one @ref XXH32_state_t to another.
681 *
682 * @param dst_state The state to copy to.
683 * @param src_state The state to copy from.
684 * @pre
685 * @p dst_state and @p src_state must not be `NULL` and must not overlap.
686 */
687XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
688
689/*!
690 * @brief Resets an @ref XXH32_state_t to begin a new hash.
691 *
692 * @param statePtr The state struct to reset.
693 * @param seed The 32-bit seed to alter the hash result predictably.
694 *
695 * @pre
696 * @p statePtr must not be `NULL`.
697 *
698 * @return @ref XXH_OK on success.
699 * @return @ref XXH_ERROR on failure.
700 *
701 * @note This function resets and seeds a state. Call it before @ref XXH32_update().
702 *
703 * @see @ref streaming_example "Streaming Example"
704 */
705XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed);
706
707/*!
708 * @brief Consumes a block of @p input to an @ref XXH32_state_t.
709 *
710 * @param statePtr The state struct to update.
711 * @param input The block of data to be hashed, at least @p length bytes in size.
712 * @param length The length of @p input, in bytes.
713 *
714 * @pre
715 * @p statePtr must not be `NULL`.
716 * @pre
717 * The memory between @p input and @p input + @p length must be valid,
718 * readable, contiguous memory. However, if @p length is `0`, @p input may be
719 * `NULL`. In C++, this also must be *TriviallyCopyable*.
720 *
721 * @return @ref XXH_OK on success.
722 * @return @ref XXH_ERROR on failure.
723 *
724 * @note Call this to incrementally consume blocks of data.
725 *
726 * @see @ref streaming_example "Streaming Example"
727 */
728XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
729
730/*!
731 * @brief Returns the calculated hash value from an @ref XXH32_state_t.
732 *
733 * @param statePtr The state struct to calculate the hash from.
734 *
735 * @pre
736 * @p statePtr must not be `NULL`.
737 *
738 * @return The calculated 32-bit xxHash32 value from that state.
739 *
740 * @note
741 * Calling XXH32_digest() will not affect @p statePtr, so you can update,
742 * digest, and update again.
743 *
744 * @see @ref streaming_example "Streaming Example"
745 */
746XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
747#endif /* !XXH_NO_STREAM */
748
749/******* Canonical representation *******/
750
751/*!
752 * @brief Canonical (big endian) representation of @ref XXH32_hash_t.
753 */
754typedef struct {
755 unsigned char digest[4]; /*!< Hash bytes, big endian */
756} XXH32_canonical_t;
757
758/*!
759 * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
760 *
761 * @param dst The @ref XXH32_canonical_t pointer to be stored to.
762 * @param hash The @ref XXH32_hash_t to be converted.
763 *
764 * @pre
765 * @p dst must not be `NULL`.
766 *
767 * @see @ref canonical_representation_example "Canonical Representation Example"
768 */
769XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
770
771/*!
772 * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
773 *
774 * @param src The @ref XXH32_canonical_t to convert.
775 *
776 * @pre
777 * @p src must not be `NULL`.
778 *
779 * @return The converted hash.
780 *
781 * @see @ref canonical_representation_example "Canonical Representation Example"
782 */
783XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
784
785
786/*! @cond Doxygen ignores this part */
787#ifdef __has_attribute
788# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
789#else
790# define XXH_HAS_ATTRIBUTE(x) 0
791#endif
792/*! @endcond */
793
794/*! @cond Doxygen ignores this part */
795/*
796 * C23 __STDC_VERSION__ number hasn't been specified yet. For now
797 * leave as `201711L` (C17 + 1).
798 * TODO: Update to correct value when its been specified.
799 */
800#define XXH_C23_VN 201711L
801/*! @endcond */
802
803/*! @cond Doxygen ignores this part */
804/* C-language Attributes are added in C23. */
805#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute)
806# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
807#else
808# define XXH_HAS_C_ATTRIBUTE(x) 0
809#endif
810/*! @endcond */
811
812/*! @cond Doxygen ignores this part */
813#if defined(__cplusplus) && defined(__has_cpp_attribute)
814# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
815#else
816# define XXH_HAS_CPP_ATTRIBUTE(x) 0
817#endif
818/*! @endcond */
819
820/*! @cond Doxygen ignores this part */
821/*
822 * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
823 * introduced in CPP17 and C23.
824 * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
825 * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough
826 */
827#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough)
828# define XXH_FALLTHROUGH [[fallthrough]]
829#elif XXH_HAS_ATTRIBUTE(__fallthrough__)
830# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__))
831#else
832# define XXH_FALLTHROUGH /* fallthrough */
833#endif
834/*! @endcond */
835
836/*! @cond Doxygen ignores this part */
837/*
838 * Define XXH_NOESCAPE for annotated pointers in public API.
839 * https://clang.llvm.org/docs/AttributeReference.html#noescape
840 * As of writing this, only supported by clang.
841 */
842#if XXH_HAS_ATTRIBUTE(noescape)
843# define XXH_NOESCAPE __attribute__((__noescape__))
844#else
845# define XXH_NOESCAPE
846#endif
847/*! @endcond */
848
849
850/*!
851 * @}
852 * @ingroup public
853 * @{
854 */
855
856#ifndef XXH_NO_LONG_LONG
857/*-**********************************************************************
858* 64-bit hash
859************************************************************************/
860#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
861/*!
862 * @brief An unsigned 64-bit integer.
863 *
864 * Not necessarily defined to `uint64_t` but functionally equivalent.
865 */
866typedef uint64_t XXH64_hash_t;
867#elif !defined (__VMS) \
868 && (defined (__cplusplus) \
869 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
870# ifdef _AIX
871# include <inttypes.h>
872# else
873# include <stdint.h>
874# endif
875 typedef uint64_t XXH64_hash_t;
876#else
877# include <limits.h>
878# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
879 /* LP64 ABI says uint64_t is unsigned long */
880 typedef unsigned long XXH64_hash_t;
881# else
882 /* the following type must have a width of 64-bit */
883 typedef unsigned long long XXH64_hash_t;
884# endif
885#endif
886
887/*!
888 * @}
889 *
890 * @defgroup XXH64_family XXH64 family
891 * @ingroup public
892 * @{
893 * Contains functions used in the classic 64-bit xxHash algorithm.
894 *
895 * @note
896 * XXH3 provides competitive speed for both 32-bit and 64-bit systems,
897 * and offers true 64/128 bit hash results.
898 * It provides better speed for systems with vector processing capabilities.
899 */
900
901/*!
902 * @brief Calculates the 64-bit hash of @p input using xxHash64.
903 *
904 * @param input The block of data to be hashed, at least @p length bytes in size.
905 * @param length The length of @p input, in bytes.
906 * @param seed The 64-bit seed to alter the hash's output predictably.
907 *
908 * @pre
909 * The memory between @p input and @p input + @p length must be valid,
910 * readable, contiguous memory. However, if @p length is `0`, @p input may be
911 * `NULL`. In C++, this also must be *TriviallyCopyable*.
912 *
913 * @return The calculated 64-bit xxHash64 value.
914 *
915 * @see @ref single_shot_example "Single Shot Example" for an example.
916 */
917XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
918
919/******* Streaming *******/
920#ifndef XXH_NO_STREAM
921/*!
922 * @brief The opaque state struct for the XXH64 streaming API.
923 *
924 * @see XXH64_state_s for details.
925 * @see @ref streaming_example "Streaming Example"
926 */
927typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
928
929/*!
930 * @brief Allocates an @ref XXH64_state_t.
931 *
932 * @return An allocated pointer of @ref XXH64_state_t on success.
933 * @return `NULL` on failure.
934 *
935 * @note Must be freed with XXH64_freeState().
936 *
937 * @see @ref streaming_example "Streaming Example"
938 */
939XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void);
940
941/*!
942 * @brief Frees an @ref XXH64_state_t.
943 *
944 * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState().
945 *
946 * @return @ref XXH_OK.
947 *
948 * @note @p statePtr must be allocated with XXH64_createState().
949 *
950 * @see @ref streaming_example "Streaming Example"
951 */
952XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
953
954/*!
955 * @brief Copies one @ref XXH64_state_t to another.
956 *
957 * @param dst_state The state to copy to.
958 * @param src_state The state to copy from.
959 * @pre
960 * @p dst_state and @p src_state must not be `NULL` and must not overlap.
961 */
962XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state);
963
964/*!
965 * @brief Resets an @ref XXH64_state_t to begin a new hash.
966 *
967 * @param statePtr The state struct to reset.
968 * @param seed The 64-bit seed to alter the hash result predictably.
969 *
970 * @pre
971 * @p statePtr must not be `NULL`.
972 *
973 * @return @ref XXH_OK on success.
974 * @return @ref XXH_ERROR on failure.
975 *
976 * @note This function resets and seeds a state. Call it before @ref XXH64_update().
977 *
978 * @see @ref streaming_example "Streaming Example"
979 */
980XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed);
981
982/*!
983 * @brief Consumes a block of @p input to an @ref XXH64_state_t.
984 *
985 * @param statePtr The state struct to update.
986 * @param input The block of data to be hashed, at least @p length bytes in size.
987 * @param length The length of @p input, in bytes.
988 *
989 * @pre
990 * @p statePtr must not be `NULL`.
991 * @pre
992 * The memory between @p input and @p input + @p length must be valid,
993 * readable, contiguous memory. However, if @p length is `0`, @p input may be
994 * `NULL`. In C++, this also must be *TriviallyCopyable*.
995 *
996 * @return @ref XXH_OK on success.
997 * @return @ref XXH_ERROR on failure.
998 *
999 * @note Call this to incrementally consume blocks of data.
1000 *
1001 * @see @ref streaming_example "Streaming Example"
1002 */
1003XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
1004
1005/*!
1006 * @brief Returns the calculated hash value from an @ref XXH64_state_t.
1007 *
1008 * @param statePtr The state struct to calculate the hash from.
1009 *
1010 * @pre
1011 * @p statePtr must not be `NULL`.
1012 *
1013 * @return The calculated 64-bit xxHash64 value from that state.
1014 *
1015 * @note
1016 * Calling XXH64_digest() will not affect @p statePtr, so you can update,
1017 * digest, and update again.
1018 *
1019 * @see @ref streaming_example "Streaming Example"
1020 */
1021XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr);
1022#endif /* !XXH_NO_STREAM */
1023/******* Canonical representation *******/
1024
1025/*!
1026 * @brief Canonical (big endian) representation of @ref XXH64_hash_t.
1027 */
1028typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
1029
1030/*!
1031 * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t.
1032 *
1033 * @param dst The @ref XXH64_canonical_t pointer to be stored to.
1034 * @param hash The @ref XXH64_hash_t to be converted.
1035 *
1036 * @pre
1037 * @p dst must not be `NULL`.
1038 *
1039 * @see @ref canonical_representation_example "Canonical Representation Example"
1040 */
1041XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash);
1042
1043/*!
1044 * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t.
1045 *
1046 * @param src The @ref XXH64_canonical_t to convert.
1047 *
1048 * @pre
1049 * @p src must not be `NULL`.
1050 *
1051 * @return The converted hash.
1052 *
1053 * @see @ref canonical_representation_example "Canonical Representation Example"
1054 */
1055XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src);
1056
1057#ifndef XXH_NO_XXH3
1058
1059/*!
1060 * @}
1061 * ************************************************************************
1062 * @defgroup XXH3_family XXH3 family
1063 * @ingroup public
1064 * @{
1065 *
1066 * XXH3 is a more recent hash algorithm featuring:
1067 * - Improved speed for both small and large inputs
1068 * - True 64-bit and 128-bit outputs
1069 * - SIMD acceleration
1070 * - Improved 32-bit viability
1071 *
1072 * Speed analysis methodology is explained here:
1073 *
1074 * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
1075 *
1076 * Compared to XXH64, expect XXH3 to run approximately
1077 * ~2x faster on large inputs and >3x faster on small ones,
1078 * exact differences vary depending on platform.
1079 *
1080 * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
1081 * but does not require it.
1082 * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3
1083 * at competitive speeds, even without vector support. Further details are
1084 * explained in the implementation.
1085 *
1086 * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD
1087 * implementations for many common platforms:
1088 * - AVX512
1089 * - AVX2
1090 * - SSE2
1091 * - ARM NEON
1092 * - WebAssembly SIMD128
1093 * - POWER8 VSX
1094 * - s390x ZVector
1095 * This can be controlled via the @ref XXH_VECTOR macro, but it automatically
1096 * selects the best version according to predefined macros. For the x86 family, an
1097 * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c.
1098 *
1099 * XXH3 implementation is portable:
1100 * it has a generic C90 formulation that can be compiled on any platform,
1101 * all implementations generate exactly the same hash value on all platforms.
1102 * Starting from v0.8.0, it's also labelled "stable", meaning that
1103 * any future version will also generate the same hash value.
1104 *
1105 * XXH3 offers 2 variants, _64bits and _128bits.
1106 *
1107 * When only 64 bits are needed, prefer invoking the _64bits variant, as it
1108 * reduces the amount of mixing, resulting in faster speed on small inputs.
1109 * It's also generally simpler to manipulate a scalar return type than a struct.
1110 *
1111 * The API supports one-shot hashing, streaming mode, and custom secrets.
1112 */
1113/*-**********************************************************************
1114* XXH3 64-bit variant
1115************************************************************************/
1116
1117/*!
1118 * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input.
1119 *
1120 * @param input The block of data to be hashed, at least @p length bytes in size.
1121 * @param length The length of @p input, in bytes.
1122 *
1123 * @pre
1124 * The memory between @p input and @p input + @p length must be valid,
1125 * readable, contiguous memory. However, if @p length is `0`, @p input may be
1126 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1127 *
1128 * @return The calculated 64-bit XXH3 hash value.
1129 *
1130 * @note
1131 * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however
1132 * it may have slightly better performance due to constant propagation of the
1133 * defaults.
1134 *
1135 * @see
1136 * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants
1137 * @see @ref single_shot_example "Single Shot Example" for an example.
1138 */
1139XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length);
1140
1141/*!
1142 * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input.
1143 *
1144 * @param input The block of data to be hashed, at least @p length bytes in size.
1145 * @param length The length of @p input, in bytes.
1146 * @param seed The 64-bit seed to alter the hash result predictably.
1147 *
1148 * @pre
1149 * The memory between @p input and @p input + @p length must be valid,
1150 * readable, contiguous memory. However, if @p length is `0`, @p input may be
1151 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1152 *
1153 * @return The calculated 64-bit XXH3 hash value.
1154 *
1155 * @note
1156 * seed == 0 produces the same results as @ref XXH3_64bits().
1157 *
1158 * This variant generates a custom secret on the fly based on default secret
1159 * altered using the @p seed value.
1160 *
1161 * While this operation is decently fast, note that it's not completely free.
1162 *
1163 * @see @ref single_shot_example "Single Shot Example" for an example.
1164 */
1165XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
1166
1167/*!
1168 * The bare minimum size for a custom secret.
1169 *
1170 * @see
1171 * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
1172 * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
1173 */
1174#define XXH3_SECRET_SIZE_MIN 136
1175
1176/*!
1177 * @brief Calculates 64-bit variant of XXH3 with a custom "secret".
1178 *
1179 * @param data The block of data to be hashed, at least @p len bytes in size.
1180 * @param len The length of @p data, in bytes.
1181 * @param secret The secret data.
1182 * @param secretSize The length of @p secret, in bytes.
1183 *
1184 * @return The calculated 64-bit XXH3 hash value.
1185 *
1186 * @pre
1187 * The memory between @p data and @p data + @p len must be valid,
1188 * readable, contiguous memory. However, if @p length is `0`, @p data may be
1189 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1190 *
1191 * It's possible to provide any blob of bytes as a "secret" to generate the hash.
1192 * This makes it more difficult for an external actor to prepare an intentional collision.
1193 * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
1194 * However, the quality of the secret impacts the dispersion of the hash algorithm.
1195 * Therefore, the secret _must_ look like a bunch of random bytes.
1196 * Avoid "trivial" or structured data such as repeated sequences or a text document.
1197 * Whenever in doubt about the "randomness" of the blob of bytes,
1198 * consider employing @ref XXH3_generateSecret() instead (see below).
1199 * It will generate a proper high entropy secret derived from the blob of bytes.
1200 * Another advantage of using XXH3_generateSecret() is that
1201 * it guarantees that all bits within the initial blob of bytes
1202 * will impact every bit of the output.
1203 * This is not necessarily the case when using the blob of bytes directly
1204 * because, when hashing _small_ inputs, only a portion of the secret is employed.
1205 *
1206 * @see @ref single_shot_example "Single Shot Example" for an example.
1207 */
1208XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
1209
1210
1211/******* Streaming *******/
1212#ifndef XXH_NO_STREAM
1213/*
1214 * Streaming requires state maintenance.
1215 * This operation costs memory and CPU.
1216 * As a consequence, streaming is slower than one-shot hashing.
1217 * For better performance, prefer one-shot functions whenever applicable.
1218 */
1219
1220/*!
1221 * @brief The opaque state struct for the XXH3 streaming API.
1222 *
1223 * @see XXH3_state_s for details.
1224 * @see @ref streaming_example "Streaming Example"
1225 */
1226typedef struct XXH3_state_s XXH3_state_t;
1227XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void);
1228XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
1229
1230/*!
1231 * @brief Copies one @ref XXH3_state_t to another.
1232 *
1233 * @param dst_state The state to copy to.
1234 * @param src_state The state to copy from.
1235 * @pre
1236 * @p dst_state and @p src_state must not be `NULL` and must not overlap.
1237 */
1238XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state);
1239
1240/*!
1241 * @brief Resets an @ref XXH3_state_t to begin a new hash.
1242 *
1243 * @param statePtr The state struct to reset.
1244 *
1245 * @pre
1246 * @p statePtr must not be `NULL`.
1247 *
1248 * @return @ref XXH_OK on success.
1249 * @return @ref XXH_ERROR on failure.
1250 *
1251 * @note
1252 * - This function resets `statePtr` and generate a secret with default parameters.
1253 * - Call this function before @ref XXH3_64bits_update().
1254 * - Digest will be equivalent to `XXH3_64bits()`.
1255 *
1256 * @see @ref streaming_example "Streaming Example"
1257 *
1258 */
1259XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
1260
1261/*!
1262 * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
1263 *
1264 * @param statePtr The state struct to reset.
1265 * @param seed The 64-bit seed to alter the hash result predictably.
1266 *
1267 * @pre
1268 * @p statePtr must not be `NULL`.
1269 *
1270 * @return @ref XXH_OK on success.
1271 * @return @ref XXH_ERROR on failure.
1272 *
1273 * @note
1274 * - This function resets `statePtr` and generate a secret from `seed`.
1275 * - Call this function before @ref XXH3_64bits_update().
1276 * - Digest will be equivalent to `XXH3_64bits_withSeed()`.
1277 *
1278 * @see @ref streaming_example "Streaming Example"
1279 *
1280 */
1281XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
1282
1283/*!
1284 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
1285 *
1286 * @param statePtr The state struct to reset.
1287 * @param secret The secret data.
1288 * @param secretSize The length of @p secret, in bytes.
1289 *
1290 * @pre
1291 * @p statePtr must not be `NULL`.
1292 *
1293 * @return @ref XXH_OK on success.
1294 * @return @ref XXH_ERROR on failure.
1295 *
1296 * @note
1297 * `secret` is referenced, it _must outlive_ the hash streaming session.
1298 *
1299 * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
1300 * and the quality of produced hash values depends on secret's entropy
1301 * (secret's content should look like a bunch of random bytes).
1302 * When in doubt about the randomness of a candidate `secret`,
1303 * consider employing `XXH3_generateSecret()` instead (see below).
1304 *
1305 * @see @ref streaming_example "Streaming Example"
1306 */
1307XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
1308
1309/*!
1310 * @brief Consumes a block of @p input to an @ref XXH3_state_t.
1311 *
1312 * @param statePtr The state struct to update.
1313 * @param input The block of data to be hashed, at least @p length bytes in size.
1314 * @param length The length of @p input, in bytes.
1315 *
1316 * @pre
1317 * @p statePtr must not be `NULL`.
1318 * @pre
1319 * The memory between @p input and @p input + @p length must be valid,
1320 * readable, contiguous memory. However, if @p length is `0`, @p input may be
1321 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1322 *
1323 * @return @ref XXH_OK on success.
1324 * @return @ref XXH_ERROR on failure.
1325 *
1326 * @note Call this to incrementally consume blocks of data.
1327 *
1328 * @see @ref streaming_example "Streaming Example"
1329 */
1330XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
1331
1332/*!
1333 * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t.
1334 *
1335 * @param statePtr The state struct to calculate the hash from.
1336 *
1337 * @pre
1338 * @p statePtr must not be `NULL`.
1339 *
1340 * @return The calculated XXH3 64-bit hash value from that state.
1341 *
1342 * @note
1343 * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update,
1344 * digest, and update again.
1345 *
1346 * @see @ref streaming_example "Streaming Example"
1347 */
1348XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
1349#endif /* !XXH_NO_STREAM */
1350
1351/* note : canonical representation of XXH3 is the same as XXH64
1352 * since they both produce XXH64_hash_t values */
1353
1354
1355/*-**********************************************************************
1356* XXH3 128-bit variant
1357************************************************************************/
1358
1359/*!
1360 * @brief The return value from 128-bit hashes.
1361 *
1362 * Stored in little endian order, although the fields themselves are in native
1363 * endianness.
1364 */
1365typedef struct {
1366 XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */
1367 XXH64_hash_t high64; /*!< `value >> 64` */
1368} XXH128_hash_t;
1369
1370/*!
1371 * @brief Calculates 128-bit unseeded variant of XXH3 of @p data.
1372 *
1373 * @param data The block of data to be hashed, at least @p length bytes in size.
1374 * @param len The length of @p data, in bytes.
1375 *
1376 * @return The calculated 128-bit variant of XXH3 value.
1377 *
1378 * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead
1379 * for shorter inputs.
1380 *
1381 * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however
1382 * it may have slightly better performance due to constant propagation of the
1383 * defaults.
1384 *
1385 * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants
1386 * @see @ref single_shot_example "Single Shot Example" for an example.
1387 */
1388XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len);
1389/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
1390 *
1391 * @param data The block of data to be hashed, at least @p length bytes in size.
1392 * @param len The length of @p data, in bytes.
1393 * @param seed The 64-bit seed to alter the hash result predictably.
1394 *
1395 * @return The calculated 128-bit variant of XXH3 value.
1396 *
1397 * @note
1398 * seed == 0 produces the same results as @ref XXH3_64bits().
1399 *
1400 * This variant generates a custom secret on the fly based on default secret
1401 * altered using the @p seed value.
1402 *
1403 * While this operation is decently fast, note that it's not completely free.
1404 *
1405 * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants
1406 * @see @ref single_shot_example "Single Shot Example" for an example.
1407 */
1408XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
1409/*!
1410 * @brief Calculates 128-bit variant of XXH3 with a custom "secret".
1411 *
1412 * @param data The block of data to be hashed, at least @p len bytes in size.
1413 * @param len The length of @p data, in bytes.
1414 * @param secret The secret data.
1415 * @param secretSize The length of @p secret, in bytes.
1416 *
1417 * @return The calculated 128-bit variant of XXH3 value.
1418 *
1419 * It's possible to provide any blob of bytes as a "secret" to generate the hash.
1420 * This makes it more difficult for an external actor to prepare an intentional collision.
1421 * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
1422 * However, the quality of the secret impacts the dispersion of the hash algorithm.
1423 * Therefore, the secret _must_ look like a bunch of random bytes.
1424 * Avoid "trivial" or structured data such as repeated sequences or a text document.
1425 * Whenever in doubt about the "randomness" of the blob of bytes,
1426 * consider employing @ref XXH3_generateSecret() instead (see below).
1427 * It will generate a proper high entropy secret derived from the blob of bytes.
1428 * Another advantage of using XXH3_generateSecret() is that
1429 * it guarantees that all bits within the initial blob of bytes
1430 * will impact every bit of the output.
1431 * This is not necessarily the case when using the blob of bytes directly
1432 * because, when hashing _small_ inputs, only a portion of the secret is employed.
1433 *
1434 * @see @ref single_shot_example "Single Shot Example" for an example.
1435 */
1436XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
1437
1438/******* Streaming *******/
1439#ifndef XXH_NO_STREAM
1440/*
1441 * Streaming requires state maintenance.
1442 * This operation costs memory and CPU.
1443 * As a consequence, streaming is slower than one-shot hashing.
1444 * For better performance, prefer one-shot functions whenever applicable.
1445 *
1446 * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
1447 * Use already declared XXH3_createState() and XXH3_freeState().
1448 *
1449 * All reset and streaming functions have same meaning as their 64-bit counterpart.
1450 */
1451
1452/*!
1453 * @brief Resets an @ref XXH3_state_t to begin a new hash.
1454 *
1455 * @param statePtr The state struct to reset.
1456 *
1457 * @pre
1458 * @p statePtr must not be `NULL`.
1459 *
1460 * @return @ref XXH_OK on success.
1461 * @return @ref XXH_ERROR on failure.
1462 *
1463 * @note
1464 * - This function resets `statePtr` and generate a secret with default parameters.
1465 * - Call it before @ref XXH3_128bits_update().
1466 * - Digest will be equivalent to `XXH3_128bits()`.
1467 *
1468 * @see @ref streaming_example "Streaming Example"
1469 */
1470XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
1471
1472/*!
1473 * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
1474 *
1475 * @param statePtr The state struct to reset.
1476 * @param seed The 64-bit seed to alter the hash result predictably.
1477 *
1478 * @pre
1479 * @p statePtr must not be `NULL`.
1480 *
1481 * @return @ref XXH_OK on success.
1482 * @return @ref XXH_ERROR on failure.
1483 *
1484 * @note
1485 * - This function resets `statePtr` and generate a secret from `seed`.
1486 * - Call it before @ref XXH3_128bits_update().
1487 * - Digest will be equivalent to `XXH3_128bits_withSeed()`.
1488 *
1489 * @see @ref streaming_example "Streaming Example"
1490 */
1491XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
1492/*!
1493 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
1494 *
1495 * @param statePtr The state struct to reset.
1496 * @param secret The secret data.
1497 * @param secretSize The length of @p secret, in bytes.
1498 *
1499 * @pre
1500 * @p statePtr must not be `NULL`.
1501 *
1502 * @return @ref XXH_OK on success.
1503 * @return @ref XXH_ERROR on failure.
1504 *
1505 * `secret` is referenced, it _must outlive_ the hash streaming session.
1506 * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
1507 * and the quality of produced hash values depends on secret's entropy
1508 * (secret's content should look like a bunch of random bytes).
1509 * When in doubt about the randomness of a candidate `secret`,
1510 * consider employing `XXH3_generateSecret()` instead (see below).
1511 *
1512 * @see @ref streaming_example "Streaming Example"
1513 */
1514XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
1515
1516/*!
1517 * @brief Consumes a block of @p input to an @ref XXH3_state_t.
1518 *
1519 * Call this to incrementally consume blocks of data.
1520 *
1521 * @param statePtr The state struct to update.
1522 * @param input The block of data to be hashed, at least @p length bytes in size.
1523 * @param length The length of @p input, in bytes.
1524 *
1525 * @pre
1526 * @p statePtr must not be `NULL`.
1527 *
1528 * @return @ref XXH_OK on success.
1529 * @return @ref XXH_ERROR on failure.
1530 *
1531 * @note
1532 * The memory between @p input and @p input + @p length must be valid,
1533 * readable, contiguous memory. However, if @p length is `0`, @p input may be
1534 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1535 *
1536 */
1537XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
1538
1539/*!
1540 * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t.
1541 *
1542 * @param statePtr The state struct to calculate the hash from.
1543 *
1544 * @pre
1545 * @p statePtr must not be `NULL`.
1546 *
1547 * @return The calculated XXH3 128-bit hash value from that state.
1548 *
1549 * @note
1550 * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update,
1551 * digest, and update again.
1552 *
1553 */
1554XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
1555#endif /* !XXH_NO_STREAM */
1556
1557/* Following helper functions make it possible to compare XXH128_hast_t values.
1558 * Since XXH128_hash_t is a structure, this capability is not offered by the language.
1559 * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
1560
1561/*!
1562 * @brief Check equality of two XXH128_hash_t values
1563 *
1564 * @param h1 The 128-bit hash value.
1565 * @param h2 Another 128-bit hash value.
1566 *
1567 * @return `1` if `h1` and `h2` are equal.
1568 * @return `0` if they are not.
1569 */
1570XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
1571
1572/*!
1573 * @brief Compares two @ref XXH128_hash_t
1574 *
1575 * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
1576 *
1577 * @param h128_1 Left-hand side value
1578 * @param h128_2 Right-hand side value
1579 *
1580 * @return >0 if @p h128_1 > @p h128_2
1581 * @return =0 if @p h128_1 == @p h128_2
1582 * @return <0 if @p h128_1 < @p h128_2
1583 */
1584XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2);
1585
1586
1587/******* Canonical representation *******/
1588typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
1589
1590
1591/*!
1592 * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t.
1593 *
1594 * @param dst The @ref XXH128_canonical_t pointer to be stored to.
1595 * @param hash The @ref XXH128_hash_t to be converted.
1596 *
1597 * @pre
1598 * @p dst must not be `NULL`.
1599 * @see @ref canonical_representation_example "Canonical Representation Example"
1600 */
1601XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash);
1602
1603/*!
1604 * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t.
1605 *
1606 * @param src The @ref XXH128_canonical_t to convert.
1607 *
1608 * @pre
1609 * @p src must not be `NULL`.
1610 *
1611 * @return The converted hash.
1612 * @see @ref canonical_representation_example "Canonical Representation Example"
1613 */
1614XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src);
1615
1616
1617#endif /* !XXH_NO_XXH3 */
1618#endif /* XXH_NO_LONG_LONG */
1619
1620/*!
1621 * @}
1622 */
1623#endif /* XXHASH_H_5627135585666179 */
1624
1625
1626
1627#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
1628#define XXHASH_H_STATIC_13879238742
1629/* ****************************************************************************
1630 * This section contains declarations which are not guaranteed to remain stable.
1631 * They may change in future versions, becoming incompatible with a different
1632 * version of the library.
1633 * These declarations should only be used with static linking.
1634 * Never use them in association with dynamic linking!
1635 ***************************************************************************** */
1636
1637/*
1638 * These definitions are only present to allow static allocation
1639 * of XXH states, on stack or in a struct, for example.
1640 * Never **ever** access their members directly.
1641 */
1642
1643/*!
1644 * @internal
1645 * @brief Structure for XXH32 streaming API.
1646 *
1647 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1648 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
1649 * an opaque type. This allows fields to safely be changed.
1650 *
1651 * Typedef'd to @ref XXH32_state_t.
1652 * Do not access the members of this struct directly.
1653 * @see XXH64_state_s, XXH3_state_s
1654 */
1655struct XXH32_state_s {
1656 XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
1657 XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
1658 XXH32_hash_t v[4]; /*!< Accumulator lanes */
1659 XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */
1660 XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */
1661 XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */
1662}; /* typedef'd to XXH32_state_t */
1663
1664
1665#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */
1666
1667/*!
1668 * @internal
1669 * @brief Structure for XXH64 streaming API.
1670 *
1671 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1672 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
1673 * an opaque type. This allows fields to safely be changed.
1674 *
1675 * Typedef'd to @ref XXH64_state_t.
1676 * Do not access the members of this struct directly.
1677 * @see XXH32_state_s, XXH3_state_s
1678 */
1679struct XXH64_state_s {
1680 XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */
1681 XXH64_hash_t v[4]; /*!< Accumulator lanes */
1682 XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */
1683 XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */
1684 XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/
1685 XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */
1686}; /* typedef'd to XXH64_state_t */
1687
1688#ifndef XXH_NO_XXH3
1689
1690/* Windows SDK under 10.0.22000 is missing stdalign.h so we add a check
1691 before allowing the windows compiler to use the C11 form.
1692 Reference: https://github.com/Cyan4973/xxHash/issues/955 */
1693#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) \
1694 && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */
1695# include <stdalign.h>
1696# define XXH_ALIGN(n) alignas(n)
1697#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
1698/* In C++ alignas() is a keyword */
1699# define XXH_ALIGN(n) alignas(n)
1700#elif defined(__GNUC__)
1701# define XXH_ALIGN(n) __attribute__ ((aligned(n)))
1702#elif defined(_MSC_VER)
1703# define XXH_ALIGN(n) __declspec(align(n))
1704#else
1705# define XXH_ALIGN(n) /* disabled */
1706#endif
1707
1708/* Old GCC versions only accept the attribute after the type in structures. */
1709#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \
1710 && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
1711 && defined(__GNUC__)
1712# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
1713#else
1714# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
1715#endif
1716
1717/*!
1718 * @brief The size of the internal XXH3 buffer.
1719 *
1720 * This is the optimal update size for incremental hashing.
1721 *
1722 * @see XXH3_64b_update(), XXH3_128b_update().
1723 */
1724#define XXH3_INTERNALBUFFER_SIZE 256
1725
1726/*!
1727 * @internal
1728 * @brief Default size of the secret buffer (and @ref XXH3_kSecret).
1729 *
1730 * This is the size used in @ref XXH3_kSecret and the seeded functions.
1731 *
1732 * Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
1733 */
1734#define XXH3_SECRET_DEFAULT_SIZE 192
1735
1736/*!
1737 * @internal
1738 * @brief Structure for XXH3 streaming API.
1739 *
1740 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1741 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
1742 * Otherwise it is an opaque type.
1743 * Never use this definition in combination with dynamic library.
1744 * This allows fields to safely be changed in the future.
1745 *
1746 * @note ** This structure has a strict alignment requirement of 64 bytes!! **
1747 * Do not allocate this with `malloc()` or `new`,
1748 * it will not be sufficiently aligned.
1749 * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
1750 *
1751 * Typedef'd to @ref XXH3_state_t.
1752 * Do never access the members of this struct directly.
1753 *
1754 * @see XXH3_INITSTATE() for stack initialization.
1755 * @see XXH3_createState(), XXH3_freeState().
1756 * @see XXH32_state_s, XXH64_state_s
1757 */
1758struct XXH3_state_s {
1759 XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
1760 /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */
1761 XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
1762 /*!< Used to store a custom secret generated from a seed. */
1763 XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
1764 /*!< The internal buffer. @see XXH32_state_s::mem32 */
1765 XXH32_hash_t bufferedSize;
1766 /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */
1767 XXH32_hash_t useSeed;
1768 /*!< Reserved field. Needed for padding on 64-bit. */
1769 size_t nbStripesSoFar;
1770 /*!< Number or stripes processed. */
1771 XXH64_hash_t totalLen;
1772 /*!< Total length hashed. 64-bit even on 32-bit targets. */
1773 size_t nbStripesPerBlock;
1774 /*!< Number of stripes per block. */
1775 size_t secretLimit;
1776 /*!< Size of @ref customSecret or @ref extSecret */
1777 XXH64_hash_t seed;
1778 /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */
1779 XXH64_hash_t reserved64;
1780 /*!< Reserved field. */
1781 const unsigned char* extSecret;
1782 /*!< Reference to an external secret for the _withSecret variants, NULL
1783 * for other variants. */
1784 /* note: there may be some padding at the end due to alignment on 64 bytes */
1785}; /* typedef'd to XXH3_state_t */
1786
1787#undef XXH_ALIGN_MEMBER
1788
1789/*!
1790 * @brief Initializes a stack-allocated `XXH3_state_s`.
1791 *
1792 * When the @ref XXH3_state_t structure is merely emplaced on stack,
1793 * it should be initialized with XXH3_INITSTATE() or a memset()
1794 * in case its first reset uses XXH3_NNbits_reset_withSeed().
1795 * This init can be omitted if the first reset uses default or _withSecret mode.
1796 * This operation isn't necessary when the state is created with XXH3_createState().
1797 * Note that this doesn't prepare the state for a streaming operation,
1798 * it's still necessary to use XXH3_NNbits_reset*() afterwards.
1799 */
1800#define XXH3_INITSTATE(XXH3_state_ptr) \
1801 do { \
1802 XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \
1803 tmp_xxh3_state_ptr->seed = 0; \
1804 tmp_xxh3_state_ptr->extSecret = NULL; \
1805 } while(0)
1806
1807
1808/*!
1809 * @brief Calculates the 128-bit hash of @p data using XXH3.
1810 *
1811 * @param data The block of data to be hashed, at least @p len bytes in size.
1812 * @param len The length of @p data, in bytes.
1813 * @param seed The 64-bit seed to alter the hash's output predictably.
1814 *
1815 * @pre
1816 * The memory between @p data and @p data + @p len must be valid,
1817 * readable, contiguous memory. However, if @p len is `0`, @p data may be
1818 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1819 *
1820 * @return The calculated 128-bit XXH3 value.
1821 *
1822 * @see @ref single_shot_example "Single Shot Example" for an example.
1823 */
1824XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
1825
1826
1827/* === Experimental API === */
1828/* Symbols defined below must be considered tied to a specific library version. */
1829
1830/*!
1831 * @brief Derive a high-entropy secret from any user-defined content, named customSeed.
1832 *
1833 * @param secretBuffer A writable buffer for derived high-entropy secret data.
1834 * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_SIZE_MIN.
1835 * @param customSeed A user-defined content.
1836 * @param customSeedSize Size of customSeed, in bytes.
1837 *
1838 * @return @ref XXH_OK on success.
1839 * @return @ref XXH_ERROR on failure.
1840 *
1841 * The generated secret can be used in combination with `*_withSecret()` functions.
1842 * The `_withSecret()` variants are useful to provide a higher level of protection
1843 * than 64-bit seed, as it becomes much more difficult for an external actor to
1844 * guess how to impact the calculation logic.
1845 *
1846 * The function accepts as input a custom seed of any length and any content,
1847 * and derives from it a high-entropy secret of length @p secretSize into an
1848 * already allocated buffer @p secretBuffer.
1849 *
1850 * The generated secret can then be used with any `*_withSecret()` variant.
1851 * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(),
1852 * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret()
1853 * are part of this list. They all accept a `secret` parameter
1854 * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN)
1855 * _and_ feature very high entropy (consist of random-looking bytes).
1856 * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can
1857 * be employed to ensure proper quality.
1858 *
1859 * @p customSeed can be anything. It can have any size, even small ones,
1860 * and its content can be anything, even "poor entropy" sources such as a bunch
1861 * of zeroes. The resulting `secret` will nonetheless provide all required qualities.
1862 *
1863 * @pre
1864 * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN
1865 * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
1866 *
1867 * Example code:
1868 * @code{.c}
1869 * #include <stdio.h>
1870 * #include <stdlib.h>
1871 * #include <string.h>
1872 * #define XXH_STATIC_LINKING_ONLY // expose unstable API
1873 * #include "xxhash.h"
1874 * // Hashes argv[2] using the entropy from argv[1].
1875 * int main(int argc, char* argv[])
1876 * {
1877 * char secret[XXH3_SECRET_SIZE_MIN];
1878 * if (argv != 3) { return 1; }
1879 * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1]));
1880 * XXH64_hash_t h = XXH3_64bits_withSecret(
1881 * argv[2], strlen(argv[2]),
1882 * secret, sizeof(secret)
1883 * );
1884 * printf("%016llx\n", (unsigned long long) h);
1885 * }
1886 * @endcode
1887 */
1888XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize);
1889
1890/*!
1891 * @brief Generate the same secret as the _withSeed() variants.
1892 *
1893 * @param secretBuffer A writable buffer of @ref XXH3_SECRET_DEFAULT_SIZE bytes
1894 * @param seed The 64-bit seed to alter the hash result predictably.
1895 *
1896 * The generated secret can be used in combination with
1897 *`*_withSecret()` and `_withSecretandSeed()` variants.
1898 *
1899 * Example C++ `std::string` hash class:
1900 * @code{.cpp}
1901 * #include <string>
1902 * #define XXH_STATIC_LINKING_ONLY // expose unstable API
1903 * #include "xxhash.h"
1904 * // Slow, seeds each time
1905 * class HashSlow {
1906 * XXH64_hash_t seed;
1907 * public:
1908 * HashSlow(XXH64_hash_t s) : seed{s} {}
1909 * size_t operator()(const std::string& x) const {
1910 * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)};
1911 * }
1912 * };
1913 * // Fast, caches the seeded secret for future uses.
1914 * class HashFast {
1915 * unsigned char secret[XXH3_SECRET_DEFAULT_SIZE];
1916 * public:
1917 * HashFast(XXH64_hash_t s) {
1918 * XXH3_generateSecret_fromSeed(secret, seed);
1919 * }
1920 * size_t operator()(const std::string& x) const {
1921 * return size_t{
1922 * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret))
1923 * };
1924 * }
1925 * };
1926 * @endcode
1927 */
1928XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed);
1929
1930/*!
1931 * @brief Maximum size of "short" key in bytes.
1932 */
1933#define XXH3_MIDSIZE_MAX 240
1934
1935/*!
1936 * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data.
1937 *
1938 * @param data The block of data to be hashed, at least @p len bytes in size.
1939 * @param len The length of @p data, in bytes.
1940 * @param secret The secret data.
1941 * @param secretSize The length of @p secret, in bytes.
1942 * @param seed The 64-bit seed to alter the hash result predictably.
1943 *
1944 * These variants generate hash values using either:
1945 * - @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes)
1946 * - @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX).
1947 *
1948 * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
1949 * `_withSeed()` has to generate the secret on the fly for "large" keys.
1950 * It's fast, but can be perceptible for "not so large" keys (< 1 KB).
1951 * `_withSecret()` has to generate the masks on the fly for "small" keys,
1952 * which requires more instructions than _withSeed() variants.
1953 * Therefore, _withSecretandSeed variant combines the best of both worlds.
1954 *
1955 * When @p secret has been generated by XXH3_generateSecret_fromSeed(),
1956 * this variant produces *exactly* the same results as `_withSeed()` variant,
1957 * hence offering only a pure speed benefit on "large" input,
1958 * by skipping the need to regenerate the secret for every large input.
1959 *
1960 * Another usage scenario is to hash the secret to a 64-bit hash value,
1961 * for example with XXH3_64bits(), which then becomes the seed,
1962 * and then employ both the seed and the secret in _withSecretandSeed().
1963 * On top of speed, an added benefit is that each bit in the secret
1964 * has a 50% chance to swap each bit in the output, via its impact to the seed.
1965 *
1966 * This is not guaranteed when using the secret directly in "small data" scenarios,
1967 * because only portions of the secret are employed for small data.
1968 */
1969XXH_PUBLIC_API XXH_PUREF XXH64_hash_t
1970XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len,
1971 XXH_NOESCAPE const void* secret, size_t secretSize,
1972 XXH64_hash_t seed);
1973
1974/*!
1975 * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
1976 *
1977 * @param data The memory segment to be hashed, at least @p len bytes in size.
1978 * @param length The length of @p data, in bytes.
1979 * @param secret The secret used to alter hash result predictably.
1980 * @param secretSize The length of @p secret, in bytes (must be >= XXH3_SECRET_SIZE_MIN)
1981 * @param seed64 The 64-bit seed to alter the hash result predictably.
1982 *
1983 * @return @ref XXH_OK on success.
1984 * @return @ref XXH_ERROR on failure.
1985 *
1986 * @see XXH3_64bits_withSecretandSeed(): contract is the same.
1987 */
1988XXH_PUBLIC_API XXH_PUREF XXH128_hash_t
1989XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length,
1990 XXH_NOESCAPE const void* secret, size_t secretSize,
1991 XXH64_hash_t seed64);
1992
1993#ifndef XXH_NO_STREAM
1994/*!
1995 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
1996 *
1997 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
1998 * @param secret The secret data.
1999 * @param secretSize The length of @p secret, in bytes.
2000 * @param seed64 The 64-bit seed to alter the hash result predictably.
2001 *
2002 * @return @ref XXH_OK on success.
2003 * @return @ref XXH_ERROR on failure.
2004 *
2005 * @see XXH3_64bits_withSecretandSeed(). Contract is identical.
2006 */
2007XXH_PUBLIC_API XXH_errorcode
2008XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
2009 XXH_NOESCAPE const void* secret, size_t secretSize,
2010 XXH64_hash_t seed64);
2011
2012/*!
2013 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
2014 *
2015 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
2016 * @param secret The secret data.
2017 * @param secretSize The length of @p secret, in bytes.
2018 * @param seed64 The 64-bit seed to alter the hash result predictably.
2019 *
2020 * @return @ref XXH_OK on success.
2021 * @return @ref XXH_ERROR on failure.
2022 *
2023 * @see XXH3_64bits_withSecretandSeed(). Contract is identical.
2024 *
2025 * Note: there was a bug in an earlier version of this function (<= v0.8.2)
2026 * that would make it generate an incorrect hash value
2027 * when @p seed == 0 and @p length < XXH3_MIDSIZE_MAX
2028 * and @p secret is different from XXH3_generateSecret_fromSeed().
2029 * As stated in the contract, the correct hash result must be
2030 * the same as XXH3_128bits_withSeed() when @p length <= XXH3_MIDSIZE_MAX.
2031 * Results generated by this older version are wrong, hence not comparable.
2032 */
2033XXH_PUBLIC_API XXH_errorcode
2034XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
2035 XXH_NOESCAPE const void* secret, size_t secretSize,
2036 XXH64_hash_t seed64);
2037
2038#endif /* !XXH_NO_STREAM */
2039
2040#endif /* !XXH_NO_XXH3 */
2041#endif /* XXH_NO_LONG_LONG */
2042#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
2043# define XXH_IMPLEMENTATION
2044#endif
2045
2046#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
2047
2048
2049/* ======================================================================== */
2050/* ======================================================================== */
2051/* ======================================================================== */
2052
2053
2054/*-**********************************************************************
2055 * xxHash implementation
2056 *-**********************************************************************
2057 * xxHash's implementation used to be hosted inside xxhash.c.
2058 *
2059 * However, inlining requires implementation to be visible to the compiler,
2060 * hence be included alongside the header.
2061 * Previously, implementation was hosted inside xxhash.c,
2062 * which was then #included when inlining was activated.
2063 * This construction created issues with a few build and install systems,
2064 * as it required xxhash.c to be stored in /include directory.
2065 *
2066 * xxHash implementation is now directly integrated within xxhash.h.
2067 * As a consequence, xxhash.c is no longer needed in /include.
2068 *
2069 * xxhash.c is still available and is still useful.
2070 * In a "normal" setup, when xxhash is not inlined,
2071 * xxhash.h only exposes the prototypes and public symbols,
2072 * while xxhash.c can be built into an object file xxhash.o
2073 * which can then be linked into the final binary.
2074 ************************************************************************/
2075
2076#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
2077 || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
2078# define XXH_IMPLEM_13a8737387
2079
2080/* *************************************
2081* Tuning parameters
2082***************************************/
2083
2084/*!
2085 * @defgroup tuning Tuning parameters
2086 * @{
2087 *
2088 * Various macros to control xxHash's behavior.
2089 */
2090#ifdef XXH_DOXYGEN
2091/*!
2092 * @brief Define this to disable 64-bit code.
2093 *
2094 * Useful if only using the @ref XXH32_family and you have a strict C90 compiler.
2095 */
2096# define XXH_NO_LONG_LONG
2097# undef XXH_NO_LONG_LONG /* don't actually */
2098/*!
2099 * @brief Controls how unaligned memory is accessed.
2100 *
2101 * By default, access to unaligned memory is controlled by `memcpy()`, which is
2102 * safe and portable.
2103 *
2104 * Unfortunately, on some target/compiler combinations, the generated assembly
2105 * is sub-optimal.
2106 *
2107 * The below switch allow selection of a different access method
2108 * in the search for improved performance.
2109 *
2110 * @par Possible options:
2111 *
2112 * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
2113 * @par
2114 * Use `memcpy()`. Safe and portable. Note that most modern compilers will
2115 * eliminate the function call and treat it as an unaligned access.
2116 *
2117 * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))`
2118 * @par
2119 * Depends on compiler extensions and is therefore not portable.
2120 * This method is safe _if_ your compiler supports it,
2121 * and *generally* as fast or faster than `memcpy`.
2122 *
2123 * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
2124 * @par
2125 * Casts directly and dereferences. This method doesn't depend on the
2126 * compiler, but it violates the C standard as it directly dereferences an
2127 * unaligned pointer. It can generate buggy code on targets which do not
2128 * support unaligned memory accesses, but in some circumstances, it's the
2129 * only known way to get the most performance.
2130 *
2131 * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
2132 * @par
2133 * Also portable. This can generate the best code on old compilers which don't
2134 * inline small `memcpy()` calls, and it might also be faster on big-endian
2135 * systems which lack a native byteswap instruction. However, some compilers
2136 * will emit literal byteshifts even if the target supports unaligned access.
2137 *
2138 *
2139 * @warning
2140 * Methods 1 and 2 rely on implementation-defined behavior. Use these with
2141 * care, as what works on one compiler/platform/optimization level may cause
2142 * another to read garbage data or even crash.
2143 *
2144 * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
2145 *
2146 * Prefer these methods in priority order (0 > 3 > 1 > 2)
2147 */
2148# define XXH_FORCE_MEMORY_ACCESS 0
2149
2150/*!
2151 * @def XXH_SIZE_OPT
2152 * @brief Controls how much xxHash optimizes for size.
2153 *
2154 * xxHash, when compiled, tends to result in a rather large binary size. This
2155 * is mostly due to heavy usage to forced inlining and constant folding of the
2156 * @ref XXH3_family to increase performance.
2157 *
2158 * However, some developers prefer size over speed. This option can
2159 * significantly reduce the size of the generated code. When using the `-Os`
2160 * or `-Oz` options on GCC or Clang, this is defined to 1 by default,
2161 * otherwise it is defined to 0.
2162 *
2163 * Most of these size optimizations can be controlled manually.
2164 *
2165 * This is a number from 0-2.
2166 * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed
2167 * comes first.
2168 * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more
2169 * conservative and disables hacks that increase code size. It implies the
2170 * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0,
2171 * and @ref XXH3_NEON_LANES == 8 if they are not already defined.
2172 * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible.
2173 * Performance may cry. For example, the single shot functions just use the
2174 * streaming API.
2175 */
2176# define XXH_SIZE_OPT 0
2177
2178/*!
2179 * @def XXH_FORCE_ALIGN_CHECK
2180 * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
2181 * and XXH64() only).
2182 *
2183 * This is an important performance trick for architectures without decent
2184 * unaligned memory access performance.
2185 *
2186 * It checks for input alignment, and when conditions are met, uses a "fast
2187 * path" employing direct 32-bit/64-bit reads, resulting in _dramatically
2188 * faster_ read speed.
2189 *
2190 * The check costs one initial branch per hash, which is generally negligible,
2191 * but not zero.
2192 *
2193 * Moreover, it's not useful to generate an additional code path if memory
2194 * access uses the same instruction for both aligned and unaligned
2195 * addresses (e.g. x86 and aarch64).
2196 *
2197 * In these cases, the alignment check can be removed by setting this macro to 0.
2198 * Then the code will always use unaligned memory access.
2199 * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips
2200 * which are platforms known to offer good unaligned memory accesses performance.
2201 *
2202 * It is also disabled by default when @ref XXH_SIZE_OPT >= 1.
2203 *
2204 * This option does not affect XXH3 (only XXH32 and XXH64).
2205 */
2206# define XXH_FORCE_ALIGN_CHECK 0
2207
2208/*!
2209 * @def XXH_NO_INLINE_HINTS
2210 * @brief When non-zero, sets all functions to `static`.
2211 *
2212 * By default, xxHash tries to force the compiler to inline almost all internal
2213 * functions.
2214 *
2215 * This can usually improve performance due to reduced jumping and improved
2216 * constant folding, but significantly increases the size of the binary which
2217 * might not be favorable.
2218 *
2219 * Additionally, sometimes the forced inlining can be detrimental to performance,
2220 * depending on the architecture.
2221 *
2222 * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
2223 * compiler full control on whether to inline or not.
2224 *
2225 * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if
2226 * @ref XXH_SIZE_OPT >= 1, this will automatically be defined.
2227 */
2228# define XXH_NO_INLINE_HINTS 0
2229
2230/*!
2231 * @def XXH3_INLINE_SECRET
2232 * @brief Determines whether to inline the XXH3 withSecret code.
2233 *
2234 * When the secret size is known, the compiler can improve the performance
2235 * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret().
2236 *
2237 * However, if the secret size is not known, it doesn't have any benefit. This
2238 * happens when xxHash is compiled into a global symbol. Therefore, if
2239 * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0.
2240 *
2241 * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers
2242 * that are *sometimes* force inline on -Og, and it is impossible to automatically
2243 * detect this optimization level.
2244 */
2245# define XXH3_INLINE_SECRET 0
2246
2247/*!
2248 * @def XXH32_ENDJMP
2249 * @brief Whether to use a jump for `XXH32_finalize`.
2250 *
2251 * For performance, `XXH32_finalize` uses multiple branches in the finalizer.
2252 * This is generally preferable for performance,
2253 * but depending on exact architecture, a jmp may be preferable.
2254 *
2255 * This setting is only possibly making a difference for very small inputs.
2256 */
2257# define XXH32_ENDJMP 0
2258
2259/*!
2260 * @internal
2261 * @brief Redefines old internal names.
2262 *
2263 * For compatibility with code that uses xxHash's internals before the names
2264 * were changed to improve namespacing. There is no other reason to use this.
2265 */
2266# define XXH_OLD_NAMES
2267# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */
2268
2269/*!
2270 * @def XXH_NO_STREAM
2271 * @brief Disables the streaming API.
2272 *
2273 * When xxHash is not inlined and the streaming functions are not used, disabling
2274 * the streaming functions can improve code size significantly, especially with
2275 * the @ref XXH3_family which tends to make constant folded copies of itself.
2276 */
2277# define XXH_NO_STREAM
2278# undef XXH_NO_STREAM /* don't actually */
2279#endif /* XXH_DOXYGEN */
2280/*!
2281 * @}
2282 */
2283
2284#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
2285 /* prefer __packed__ structures (method 1) for GCC
2286 * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy
2287 * which for some reason does unaligned loads. */
2288# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED))
2289# define XXH_FORCE_MEMORY_ACCESS 1
2290# endif
2291#endif
2292
2293#ifndef XXH_SIZE_OPT
2294 /* default to 1 for -Os or -Oz */
2295# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__)
2296# define XXH_SIZE_OPT 1
2297# else
2298# define XXH_SIZE_OPT 0
2299# endif
2300#endif
2301
2302#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
2303 /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */
2304# if XXH_SIZE_OPT >= 1 || \
2305 defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \
2306 || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */
2307# define XXH_FORCE_ALIGN_CHECK 0
2308# else
2309# define XXH_FORCE_ALIGN_CHECK 1
2310# endif
2311#endif
2312
2313#ifndef XXH_NO_INLINE_HINTS
2314# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */
2315# define XXH_NO_INLINE_HINTS 1
2316# else
2317# define XXH_NO_INLINE_HINTS 0
2318# endif
2319#endif
2320
2321#ifndef XXH3_INLINE_SECRET
2322# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \
2323 || !defined(XXH_INLINE_ALL)
2324# define XXH3_INLINE_SECRET 0
2325# else
2326# define XXH3_INLINE_SECRET 1
2327# endif
2328#endif
2329
2330#ifndef XXH32_ENDJMP
2331/* generally preferable for performance */
2332# define XXH32_ENDJMP 0
2333#endif
2334
2335/*!
2336 * @defgroup impl Implementation
2337 * @{
2338 */
2339
2340
2341/* *************************************
2342* Includes & Memory related functions
2343***************************************/
2344#if defined(XXH_NO_STREAM)
2345/* nothing */
2346#elif defined(XXH_NO_STDLIB)
2347
2348/* When requesting to disable any mention of stdlib,
2349 * the library loses the ability to invoked malloc / free.
2350 * In practice, it means that functions like `XXH*_createState()`
2351 * will always fail, and return NULL.
2352 * This flag is useful in situations where
2353 * xxhash.h is integrated into some kernel, embedded or limited environment
2354 * without access to dynamic allocation.
2355 */
2356
2357static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; }
2358static void XXH_free(void* p) { (void)p; }
2359
2360#else
2361
2362/*
2363 * Modify the local functions below should you wish to use
2364 * different memory routines for malloc() and free()
2365 */
2366#include <stdlib.h>
2367
2368/*!
2369 * @internal
2370 * @brief Modify this function to use a different routine than malloc().
2371 */
2372static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); }
2373
2374/*!
2375 * @internal
2376 * @brief Modify this function to use a different routine than free().
2377 */
2378static void XXH_free(void* p) { free(p); }
2379
2380#endif /* XXH_NO_STDLIB */
2381
2382#include <string.h>
2383
2384/*!
2385 * @internal
2386 * @brief Modify this function to use a different routine than memcpy().
2387 */
2388static void* XXH_memcpy(void* dest, const void* src, size_t size)
2389{
2390 return memcpy(dest,src,size);
2391}
2392
2393#include <limits.h> /* ULLONG_MAX */
2394
2395
2396/* *************************************
2397* Compiler Specific Options
2398***************************************/
2399#ifdef _MSC_VER /* Visual Studio warning fix */
2400# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
2401#endif
2402
2403#if XXH_NO_INLINE_HINTS /* disable inlining hints */
2404# if defined(__GNUC__) || defined(__clang__)
2405# define XXH_FORCE_INLINE static __attribute__((__unused__))
2406# else
2407# define XXH_FORCE_INLINE static
2408# endif
2409# define XXH_NO_INLINE static
2410/* enable inlining hints */
2411#elif defined(__GNUC__) || defined(__clang__)
2412# define XXH_FORCE_INLINE static __inline__ __attribute__((__always_inline__, __unused__))
2413# define XXH_NO_INLINE static __attribute__((__noinline__))
2414#elif defined(_MSC_VER) /* Visual Studio */
2415# define XXH_FORCE_INLINE static __forceinline
2416# define XXH_NO_INLINE static __declspec(noinline)
2417#elif defined (__cplusplus) \
2418 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */
2419# define XXH_FORCE_INLINE static inline
2420# define XXH_NO_INLINE static
2421#else
2422# define XXH_FORCE_INLINE static
2423# define XXH_NO_INLINE static
2424#endif
2425
2426#if XXH3_INLINE_SECRET
2427# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE
2428#else
2429# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE
2430#endif
2431
2432
2433/* *************************************
2434* Debug
2435***************************************/
2436/*!
2437 * @ingroup tuning
2438 * @def XXH_DEBUGLEVEL
2439 * @brief Sets the debugging level.
2440 *
2441 * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
2442 * compiler's command line options. The value must be a number.
2443 */
2444#ifndef XXH_DEBUGLEVEL
2445# ifdef DEBUGLEVEL /* backwards compat */
2446# define XXH_DEBUGLEVEL DEBUGLEVEL
2447# else
2448# define XXH_DEBUGLEVEL 0
2449# endif
2450#endif
2451
2452#if (XXH_DEBUGLEVEL>=1)
2453# include <assert.h> /* note: can still be disabled with NDEBUG */
2454# define XXH_ASSERT(c) assert(c)
2455#else
2456# if defined(__INTEL_COMPILER)
2457# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c))
2458# else
2459# define XXH_ASSERT(c) XXH_ASSUME(c)
2460# endif
2461#endif
2462
2463/* note: use after variable declarations */
2464#ifndef XXH_STATIC_ASSERT
2465# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
2466# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0)
2467# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */
2468# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
2469# else
2470# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
2471# endif
2472# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
2473#endif
2474
2475/*!
2476 * @internal
2477 * @def XXH_COMPILER_GUARD(var)
2478 * @brief Used to prevent unwanted optimizations for @p var.
2479 *
2480 * It uses an empty GCC inline assembly statement with a register constraint
2481 * which forces @p var into a general purpose register (eg eax, ebx, ecx
2482 * on x86) and marks it as modified.
2483 *
2484 * This is used in a few places to avoid unwanted autovectorization (e.g.
2485 * XXH32_round()). All vectorization we want is explicit via intrinsics,
2486 * and _usually_ isn't wanted elsewhere.
2487 *
2488 * We also use it to prevent unwanted constant folding for AArch64 in
2489 * XXH3_initCustomSecret_scalar().
2490 */
2491#if defined(__GNUC__) || defined(__clang__)
2492# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var))
2493#else
2494# define XXH_COMPILER_GUARD(var) ((void)0)
2495#endif
2496
2497/* Specifically for NEON vectors which use the "w" constraint, on
2498 * Clang. */
2499#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__)
2500# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var))
2501#else
2502# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0)
2503#endif
2504
2505/* *************************************
2506* Basic Types
2507***************************************/
2508#if !defined (__VMS) \
2509 && (defined (__cplusplus) \
2510 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
2511# ifdef _AIX
2512# include <inttypes.h>
2513# else
2514# include <stdint.h>
2515# endif
2516 typedef uint8_t xxh_u8;
2517#else
2518 typedef unsigned char xxh_u8;
2519#endif
2520typedef XXH32_hash_t xxh_u32;
2521
2522#ifdef XXH_OLD_NAMES
2523# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly"
2524# define BYTE xxh_u8
2525# define U8 xxh_u8
2526# define U32 xxh_u32
2527#endif
2528
2529/* *** Memory access *** */
2530
2531/*!
2532 * @internal
2533 * @fn xxh_u32 XXH_read32(const void* ptr)
2534 * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
2535 *
2536 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2537 *
2538 * @param ptr The pointer to read from.
2539 * @return The 32-bit native endian integer from the bytes at @p ptr.
2540 */
2541
2542/*!
2543 * @internal
2544 * @fn xxh_u32 XXH_readLE32(const void* ptr)
2545 * @brief Reads an unaligned 32-bit little endian integer from @p ptr.
2546 *
2547 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2548 *
2549 * @param ptr The pointer to read from.
2550 * @return The 32-bit little endian integer from the bytes at @p ptr.
2551 */
2552
2553/*!
2554 * @internal
2555 * @fn xxh_u32 XXH_readBE32(const void* ptr)
2556 * @brief Reads an unaligned 32-bit big endian integer from @p ptr.
2557 *
2558 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2559 *
2560 * @param ptr The pointer to read from.
2561 * @return The 32-bit big endian integer from the bytes at @p ptr.
2562 */
2563
2564/*!
2565 * @internal
2566 * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
2567 * @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
2568 *
2569 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2570 * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
2571 * always @ref XXH_alignment::XXH_unaligned.
2572 *
2573 * @param ptr The pointer to read from.
2574 * @param align Whether @p ptr is aligned.
2575 * @pre
2576 * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
2577 * aligned.
2578 * @return The 32-bit little endian integer from the bytes at @p ptr.
2579 */
2580
2581#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2582/*
2583 * Manual byteshift. Best for old compilers which don't inline memcpy.
2584 * We actually directly use XXH_readLE32 and XXH_readBE32.
2585 */
2586#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
2587
2588/*
2589 * Force direct memory access. Only works on CPU which support unaligned memory
2590 * access in hardware.
2591 */
2592static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }
2593
2594#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
2595
2596/*
2597 * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
2598 * documentation claimed that it only increased the alignment, but actually it
2599 * can decrease it on gcc, clang, and icc:
2600 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
2601 * https://gcc.godbolt.org/z/xYez1j67Y.
2602 */
2603#ifdef XXH_OLD_NAMES
2604typedef union { xxh_u32 u32; } __attribute__((__packed__)) unalign;
2605#endif
2606static xxh_u32 XXH_read32(const void* ptr)
2607{
2608 typedef __attribute__((__aligned__(1))) xxh_u32 xxh_unalign32;
2609 return *((const xxh_unalign32*)ptr);
2610}
2611
2612#else
2613
2614/*
2615 * Portable and safe solution. Generally efficient.
2616 * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
2617 */
2618static xxh_u32 XXH_read32(const void* memPtr)
2619{
2620 xxh_u32 val;
2621 XXH_memcpy(&val, memPtr, sizeof(val));
2622 return val;
2623}
2624
2625#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
2626
2627
2628/* *** Endianness *** */
2629
2630/*!
2631 * @ingroup tuning
2632 * @def XXH_CPU_LITTLE_ENDIAN
2633 * @brief Whether the target is little endian.
2634 *
2635 * Defined to 1 if the target is little endian, or 0 if it is big endian.
2636 * It can be defined externally, for example on the compiler command line.
2637 *
2638 * If it is not defined,
2639 * a runtime check (which is usually constant folded) is used instead.
2640 *
2641 * @note
2642 * This is not necessarily defined to an integer constant.
2643 *
2644 * @see XXH_isLittleEndian() for the runtime check.
2645 */
2646#ifndef XXH_CPU_LITTLE_ENDIAN
2647/*
2648 * Try to detect endianness automatically, to avoid the nonstandard behavior
2649 * in `XXH_isLittleEndian()`
2650 */
2651# if defined(_WIN32) /* Windows is always little endian */ \
2652 || defined(__LITTLE_ENDIAN__) \
2653 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
2654# define XXH_CPU_LITTLE_ENDIAN 1
2655# elif defined(__BIG_ENDIAN__) \
2656 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2657# define XXH_CPU_LITTLE_ENDIAN 0
2658# else
2659/*!
2660 * @internal
2661 * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
2662 *
2663 * Most compilers will constant fold this.
2664 */
2665static int XXH_isLittleEndian(void)
2666{
2667 /*
2668 * Portable and well-defined behavior.
2669 * Don't use static: it is detrimental to performance.
2670 */
2671 const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
2672 return one.c[0];
2673}
2674# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian()
2675# endif
2676#endif
2677
2678
2679
2680
2681/* ****************************************
2682* Compiler-specific Functions and Macros
2683******************************************/
2684#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
2685
2686#ifdef __has_builtin
2687# define XXH_HAS_BUILTIN(x) __has_builtin(x)
2688#else
2689# define XXH_HAS_BUILTIN(x) 0
2690#endif
2691
2692
2693
2694/*
2695 * C23 and future versions have standard "unreachable()".
2696 * Once it has been implemented reliably we can add it as an
2697 * additional case:
2698 *
2699 * ```
2700 * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN)
2701 * # include <stddef.h>
2702 * # ifdef unreachable
2703 * # define XXH_UNREACHABLE() unreachable()
2704 * # endif
2705 * #endif
2706 * ```
2707 *
2708 * Note C++23 also has std::unreachable() which can be detected
2709 * as follows:
2710 * ```
2711 * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L)
2712 * # include <utility>
2713 * # define XXH_UNREACHABLE() std::unreachable()
2714 * #endif
2715 * ```
2716 * NB: `__cpp_lib_unreachable` is defined in the `<version>` header.
2717 * We don't use that as including `<utility>` in `extern "C"` blocks
2718 * doesn't work on GCC12
2719 */
2720
2721#if XXH_HAS_BUILTIN(__builtin_unreachable)
2722# define XXH_UNREACHABLE() __builtin_unreachable()
2723
2724#elif defined(_MSC_VER)
2725# define XXH_UNREACHABLE() __assume(0)
2726
2727#else
2728# define XXH_UNREACHABLE()
2729#endif
2730
2731#if XXH_HAS_BUILTIN(__builtin_assume)
2732# define XXH_ASSUME(c) __builtin_assume(c)
2733#else
2734# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); }
2735#endif
2736
2737/*!
2738 * @internal
2739 * @def XXH_rotl32(x,r)
2740 * @brief 32-bit rotate left.
2741 *
2742 * @param x The 32-bit integer to be rotated.
2743 * @param r The number of bits to rotate.
2744 * @pre
2745 * @p r > 0 && @p r < 32
2746 * @note
2747 * @p x and @p r may be evaluated multiple times.
2748 * @return The rotated result.
2749 */
2750#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
2751 && XXH_HAS_BUILTIN(__builtin_rotateleft64)
2752# define XXH_rotl32 __builtin_rotateleft32
2753# define XXH_rotl64 __builtin_rotateleft64
2754/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
2755#elif defined(_MSC_VER)
2756# define XXH_rotl32(x,r) _rotl(x,r)
2757# define XXH_rotl64(x,r) _rotl64(x,r)
2758#else
2759# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
2760# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
2761#endif
2762
2763/*!
2764 * @internal
2765 * @fn xxh_u32 XXH_swap32(xxh_u32 x)
2766 * @brief A 32-bit byteswap.
2767 *
2768 * @param x The 32-bit integer to byteswap.
2769 * @return @p x, byteswapped.
2770 */
2771#if defined(_MSC_VER) /* Visual Studio */
2772# define XXH_swap32 _byteswap_ulong
2773#elif XXH_GCC_VERSION >= 403
2774# define XXH_swap32 __builtin_bswap32
2775#else
2776static xxh_u32 XXH_swap32 (xxh_u32 x)
2777{
2778 return ((x << 24) & 0xff000000 ) |
2779 ((x << 8) & 0x00ff0000 ) |
2780 ((x >> 8) & 0x0000ff00 ) |
2781 ((x >> 24) & 0x000000ff );
2782}
2783#endif
2784
2785
2786/* ***************************
2787* Memory reads
2788*****************************/
2789
2790/*!
2791 * @internal
2792 * @brief Enum to indicate whether a pointer is aligned.
2793 */
2794typedef enum {
2795 XXH_aligned, /*!< Aligned */
2796 XXH_unaligned /*!< Possibly unaligned */
2797} XXH_alignment;
2798
2799/*
2800 * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
2801 *
2802 * This is ideal for older compilers which don't inline memcpy.
2803 */
2804#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2805
2806XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
2807{
2808 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
2809 return bytePtr[0]
2810 | ((xxh_u32)bytePtr[1] << 8)
2811 | ((xxh_u32)bytePtr[2] << 16)
2812 | ((xxh_u32)bytePtr[3] << 24);
2813}
2814
2815XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
2816{
2817 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
2818 return bytePtr[3]
2819 | ((xxh_u32)bytePtr[2] << 8)
2820 | ((xxh_u32)bytePtr[1] << 16)
2821 | ((xxh_u32)bytePtr[0] << 24);
2822}
2823
2824#else
2825XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
2826{
2827 return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
2828}
2829
2830static xxh_u32 XXH_readBE32(const void* ptr)
2831{
2832 return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
2833}
2834#endif
2835
2836XXH_FORCE_INLINE xxh_u32
2837XXH_readLE32_align(const void* ptr, XXH_alignment align)
2838{
2839 if (align==XXH_unaligned) {
2840 return XXH_readLE32(ptr);
2841 } else {
2842 return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
2843 }
2844}
2845
2846
2847/* *************************************
2848* Misc
2849***************************************/
2850/*! @ingroup public */
2851XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
2852
2853
2854/* *******************************************************************
2855* 32-bit hash functions
2856*********************************************************************/
2857/*!
2858 * @}
2859 * @defgroup XXH32_impl XXH32 implementation
2860 * @ingroup impl
2861 *
2862 * Details on the XXH32 implementation.
2863 * @{
2864 */
2865 /* #define instead of static const, to be used as initializers */
2866#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */
2867#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */
2868#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */
2869#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */
2870#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */
2871
2872#ifdef XXH_OLD_NAMES
2873# define PRIME32_1 XXH_PRIME32_1
2874# define PRIME32_2 XXH_PRIME32_2
2875# define PRIME32_3 XXH_PRIME32_3
2876# define PRIME32_4 XXH_PRIME32_4
2877# define PRIME32_5 XXH_PRIME32_5
2878#endif
2879
2880/*!
2881 * @internal
2882 * @brief Normal stripe processing routine.
2883 *
2884 * This shuffles the bits so that any bit from @p input impacts several bits in
2885 * @p acc.
2886 *
2887 * @param acc The accumulator lane.
2888 * @param input The stripe of input to mix.
2889 * @return The mixed accumulator lane.
2890 */
2891static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
2892{
2893 acc += input * XXH_PRIME32_2;
2894 acc = XXH_rotl32(acc, 13);
2895 acc *= XXH_PRIME32_1;
2896#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
2897 /*
2898 * UGLY HACK:
2899 * A compiler fence is used to prevent GCC and Clang from
2900 * autovectorizing the XXH32 loop (pragmas and attributes don't work for some
2901 * reason) without globally disabling SSE4.1.
2902 *
2903 * The reason we want to avoid vectorization is because despite working on
2904 * 4 integers at a time, there are multiple factors slowing XXH32 down on
2905 * SSE4:
2906 * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
2907 * newer chips!) making it slightly slower to multiply four integers at
2908 * once compared to four integers independently. Even when pmulld was
2909 * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
2910 * just to multiply unless doing a long operation.
2911 *
2912 * - Four instructions are required to rotate,
2913 * movqda tmp, v // not required with VEX encoding
2914 * pslld tmp, 13 // tmp <<= 13
2915 * psrld v, 19 // x >>= 19
2916 * por v, tmp // x |= tmp
2917 * compared to one for scalar:
2918 * roll v, 13 // reliably fast across the board
2919 * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason
2920 *
2921 * - Instruction level parallelism is actually more beneficial here because
2922 * the SIMD actually serializes this operation: While v1 is rotating, v2
2923 * can load data, while v3 can multiply. SSE forces them to operate
2924 * together.
2925 *
2926 * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing
2927 * the loop. NEON is only faster on the A53, and with the newer cores, it is less
2928 * than half the speed.
2929 *
2930 * Additionally, this is used on WASM SIMD128 because it JITs to the same
2931 * SIMD instructions and has the same issue.
2932 */
2933 XXH_COMPILER_GUARD(acc);
2934#endif
2935 return acc;
2936}
2937
2938/*!
2939 * @internal
2940 * @brief Mixes all bits to finalize the hash.
2941 *
2942 * The final mix ensures that all input bits have a chance to impact any bit in
2943 * the output digest, resulting in an unbiased distribution.
2944 *
2945 * @param hash The hash to avalanche.
2946 * @return The avalanched hash.
2947 */
2948static xxh_u32 XXH32_avalanche(xxh_u32 hash)
2949{
2950 hash ^= hash >> 15;
2951 hash *= XXH_PRIME32_2;
2952 hash ^= hash >> 13;
2953 hash *= XXH_PRIME32_3;
2954 hash ^= hash >> 16;
2955 return hash;
2956}
2957
2958#define XXH_get32bits(p) XXH_readLE32_align(p, align)
2959
2960/*!
2961 * @internal
2962 * @brief Processes the last 0-15 bytes of @p ptr.
2963 *
2964 * There may be up to 15 bytes remaining to consume from the input.
2965 * This final stage will digest them to ensure that all input bytes are present
2966 * in the final mix.
2967 *
2968 * @param hash The hash to finalize.
2969 * @param ptr The pointer to the remaining input.
2970 * @param len The remaining length, modulo 16.
2971 * @param align Whether @p ptr is aligned.
2972 * @return The finalized hash.
2973 * @see XXH64_finalize().
2974 */
2975static XXH_PUREF xxh_u32
2976XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
2977{
2978#define XXH_PROCESS1 do { \
2979 hash += (*ptr++) * XXH_PRIME32_5; \
2980 hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \
2981} while (0)
2982
2983#define XXH_PROCESS4 do { \
2984 hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \
2985 ptr += 4; \
2986 hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \
2987} while (0)
2988
2989 if (ptr==NULL) XXH_ASSERT(len == 0);
2990
2991 /* Compact rerolled version; generally faster */
2992 if (!XXH32_ENDJMP) {
2993 len &= 15;
2994 while (len >= 4) {
2995 XXH_PROCESS4;
2996 len -= 4;
2997 }
2998 while (len > 0) {
2999 XXH_PROCESS1;
3000 --len;
3001 }
3002 return XXH32_avalanche(hash);
3003 } else {
3004 switch(len&15) /* or switch(bEnd - p) */ {
3005 case 12: XXH_PROCESS4;
3006 XXH_FALLTHROUGH; /* fallthrough */
3007 case 8: XXH_PROCESS4;
3008 XXH_FALLTHROUGH; /* fallthrough */
3009 case 4: XXH_PROCESS4;
3010 return XXH32_avalanche(hash);
3011
3012 case 13: XXH_PROCESS4;
3013 XXH_FALLTHROUGH; /* fallthrough */
3014 case 9: XXH_PROCESS4;
3015 XXH_FALLTHROUGH; /* fallthrough */
3016 case 5: XXH_PROCESS4;
3017 XXH_PROCESS1;
3018 return XXH32_avalanche(hash);
3019
3020 case 14: XXH_PROCESS4;
3021 XXH_FALLTHROUGH; /* fallthrough */
3022 case 10: XXH_PROCESS4;
3023 XXH_FALLTHROUGH; /* fallthrough */
3024 case 6: XXH_PROCESS4;
3025 XXH_PROCESS1;
3026 XXH_PROCESS1;
3027 return XXH32_avalanche(hash);
3028
3029 case 15: XXH_PROCESS4;
3030 XXH_FALLTHROUGH; /* fallthrough */
3031 case 11: XXH_PROCESS4;
3032 XXH_FALLTHROUGH; /* fallthrough */
3033 case 7: XXH_PROCESS4;
3034 XXH_FALLTHROUGH; /* fallthrough */
3035 case 3: XXH_PROCESS1;
3036 XXH_FALLTHROUGH; /* fallthrough */
3037 case 2: XXH_PROCESS1;
3038 XXH_FALLTHROUGH; /* fallthrough */
3039 case 1: XXH_PROCESS1;
3040 XXH_FALLTHROUGH; /* fallthrough */
3041 case 0: return XXH32_avalanche(hash);
3042 }
3043 XXH_ASSERT(0);
3044 return hash; /* reaching this point is deemed impossible */
3045 }
3046}
3047
3048#ifdef XXH_OLD_NAMES
3049# define PROCESS1 XXH_PROCESS1
3050# define PROCESS4 XXH_PROCESS4
3051#else
3052# undef XXH_PROCESS1
3053# undef XXH_PROCESS4
3054#endif
3055
3056/*!
3057 * @internal
3058 * @brief The implementation for @ref XXH32().
3059 *
3060 * @param input , len , seed Directly passed from @ref XXH32().
3061 * @param align Whether @p input is aligned.
3062 * @return The calculated hash.
3063 */
3064XXH_FORCE_INLINE XXH_PUREF xxh_u32
3065XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
3066{
3067 xxh_u32 h32;
3068
3069 if (input==NULL) XXH_ASSERT(len == 0);
3070
3071 if (len>=16) {
3072 const xxh_u8* const bEnd = input + len;
3073 const xxh_u8* const limit = bEnd - 15;
3074 xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
3075 xxh_u32 v2 = seed + XXH_PRIME32_2;
3076 xxh_u32 v3 = seed + 0;
3077 xxh_u32 v4 = seed - XXH_PRIME32_1;
3078
3079 do {
3080 v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
3081 v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
3082 v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
3083 v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
3084 } while (input < limit);
3085
3086 h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)
3087 + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
3088 } else {
3089 h32 = seed + XXH_PRIME32_5;
3090 }
3091
3092 h32 += (xxh_u32)len;
3093
3094 return XXH32_finalize(h32, input, len&15, align);
3095}
3096
3097/*! @ingroup XXH32_family */
3098XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
3099{
3100#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
3101 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
3102 XXH32_state_t state;
3103 XXH32_reset(&state, seed);
3104 XXH32_update(&state, (const xxh_u8*)input, len);
3105 return XXH32_digest(&state);
3106#else
3107 if (XXH_FORCE_ALIGN_CHECK) {
3108 if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
3109 return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
3110 } }
3111
3112 return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
3113#endif
3114}
3115
3116
3117
3118/******* Hash streaming *******/
3119#ifndef XXH_NO_STREAM
3120/*! @ingroup XXH32_family */
3121XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
3122{
3123 return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
3124}
3125/*! @ingroup XXH32_family */
3126XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
3127{
3128 XXH_free(statePtr);
3129 return XXH_OK;
3130}
3131
3132/*! @ingroup XXH32_family */
3133XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
3134{
3135 XXH_memcpy(dstState, srcState, sizeof(*dstState));
3136}
3137
3138/*! @ingroup XXH32_family */
3139XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
3140{
3141 XXH_ASSERT(statePtr != NULL);
3142 memset(statePtr, 0, sizeof(*statePtr));
3143 statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
3144 statePtr->v[1] = seed + XXH_PRIME32_2;
3145 statePtr->v[2] = seed + 0;
3146 statePtr->v[3] = seed - XXH_PRIME32_1;
3147 return XXH_OK;
3148}
3149
3150
3151/*! @ingroup XXH32_family */
3152XXH_PUBLIC_API XXH_errorcode
3153XXH32_update(XXH32_state_t* state, const void* input, size_t len)
3154{
3155 if (input==NULL) {
3156 XXH_ASSERT(len == 0);
3157 return XXH_OK;
3158 }
3159
3160 { const xxh_u8* p = (const xxh_u8*)input;
3161 const xxh_u8* const bEnd = p + len;
3162
3163 state->total_len_32 += (XXH32_hash_t)len;
3164 state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
3165
3166 if (state->memsize + len < 16) { /* fill in tmp buffer */
3167 XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
3168 state->memsize += (XXH32_hash_t)len;
3169 return XXH_OK;
3170 }
3171
3172 if (state->memsize) { /* some data left from previous update */
3173 XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
3174 { const xxh_u32* p32 = state->mem32;
3175 state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
3176 state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
3177 state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
3178 state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
3179 }
3180 p += 16-state->memsize;
3181 state->memsize = 0;
3182 }
3183
3184 if (p <= bEnd-16) {
3185 const xxh_u8* const limit = bEnd - 16;
3186
3187 do {
3188 state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
3189 state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
3190 state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
3191 state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
3192 } while (p<=limit);
3193
3194 }
3195
3196 if (p < bEnd) {
3197 XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
3198 state->memsize = (unsigned)(bEnd-p);
3199 }
3200 }
3201
3202 return XXH_OK;
3203}
3204
3205
3206/*! @ingroup XXH32_family */
3207XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
3208{
3209 xxh_u32 h32;
3210
3211 if (state->large_len) {
3212 h32 = XXH_rotl32(state->v[0], 1)
3213 + XXH_rotl32(state->v[1], 7)
3214 + XXH_rotl32(state->v[2], 12)
3215 + XXH_rotl32(state->v[3], 18);
3216 } else {
3217 h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
3218 }
3219
3220 h32 += state->total_len_32;
3221
3222 return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
3223}
3224#endif /* !XXH_NO_STREAM */
3225
3226/******* Canonical representation *******/
3227
3228/*! @ingroup XXH32_family */
3229XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
3230{
3231 XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
3232 if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
3233 XXH_memcpy(dst, &hash, sizeof(*dst));
3234}
3235/*! @ingroup XXH32_family */
3236XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
3237{
3238 return XXH_readBE32(src);
3239}
3240
3241
3242#ifndef XXH_NO_LONG_LONG
3243
3244/* *******************************************************************
3245* 64-bit hash functions
3246*********************************************************************/
3247/*!
3248 * @}
3249 * @ingroup impl
3250 * @{
3251 */
3252/******* Memory access *******/
3253
3254typedef XXH64_hash_t xxh_u64;
3255
3256#ifdef XXH_OLD_NAMES
3257# define U64 xxh_u64
3258#endif
3259
3260#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
3261/*
3262 * Manual byteshift. Best for old compilers which don't inline memcpy.
3263 * We actually directly use XXH_readLE64 and XXH_readBE64.
3264 */
3265#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
3266
3267/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
3268static xxh_u64 XXH_read64(const void* memPtr)
3269{
3270 return *(const xxh_u64*) memPtr;
3271}
3272
3273#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
3274
3275/*
3276 * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
3277 * documentation claimed that it only increased the alignment, but actually it
3278 * can decrease it on gcc, clang, and icc:
3279 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
3280 * https://gcc.godbolt.org/z/xYez1j67Y.
3281 */
3282#ifdef XXH_OLD_NAMES
3283typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((__packed__)) unalign64;
3284#endif
3285static xxh_u64 XXH_read64(const void* ptr)
3286{
3287 typedef __attribute__((__aligned__(1))) xxh_u64 xxh_unalign64;
3288 return *((const xxh_unalign64*)ptr);
3289}
3290
3291#else
3292
3293/*
3294 * Portable and safe solution. Generally efficient.
3295 * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
3296 */
3297static xxh_u64 XXH_read64(const void* memPtr)
3298{
3299 xxh_u64 val;
3300 XXH_memcpy(&val, memPtr, sizeof(val));
3301 return val;
3302}
3303
3304#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
3305
3306#if defined(_MSC_VER) /* Visual Studio */
3307# define XXH_swap64 _byteswap_uint64
3308#elif XXH_GCC_VERSION >= 403
3309# define XXH_swap64 __builtin_bswap64
3310#else
3311static xxh_u64 XXH_swap64(xxh_u64 x)
3312{
3313 return ((x << 56) & 0xff00000000000000ULL) |
3314 ((x << 40) & 0x00ff000000000000ULL) |
3315 ((x << 24) & 0x0000ff0000000000ULL) |
3316 ((x << 8) & 0x000000ff00000000ULL) |
3317 ((x >> 8) & 0x00000000ff000000ULL) |
3318 ((x >> 24) & 0x0000000000ff0000ULL) |
3319 ((x >> 40) & 0x000000000000ff00ULL) |
3320 ((x >> 56) & 0x00000000000000ffULL);
3321}
3322#endif
3323
3324
3325/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
3326#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
3327
3328XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
3329{
3330 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
3331 return bytePtr[0]
3332 | ((xxh_u64)bytePtr[1] << 8)
3333 | ((xxh_u64)bytePtr[2] << 16)
3334 | ((xxh_u64)bytePtr[3] << 24)
3335 | ((xxh_u64)bytePtr[4] << 32)
3336 | ((xxh_u64)bytePtr[5] << 40)
3337 | ((xxh_u64)bytePtr[6] << 48)
3338 | ((xxh_u64)bytePtr[7] << 56);
3339}
3340
3341XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
3342{
3343 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
3344 return bytePtr[7]
3345 | ((xxh_u64)bytePtr[6] << 8)
3346 | ((xxh_u64)bytePtr[5] << 16)
3347 | ((xxh_u64)bytePtr[4] << 24)
3348 | ((xxh_u64)bytePtr[3] << 32)
3349 | ((xxh_u64)bytePtr[2] << 40)
3350 | ((xxh_u64)bytePtr[1] << 48)
3351 | ((xxh_u64)bytePtr[0] << 56);
3352}
3353
3354#else
3355XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
3356{
3357 return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
3358}
3359
3360static xxh_u64 XXH_readBE64(const void* ptr)
3361{
3362 return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
3363}
3364#endif
3365
3366XXH_FORCE_INLINE xxh_u64
3367XXH_readLE64_align(const void* ptr, XXH_alignment align)
3368{
3369 if (align==XXH_unaligned)
3370 return XXH_readLE64(ptr);
3371 else
3372 return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
3373}
3374
3375
3376/******* xxh64 *******/
3377/*!
3378 * @}
3379 * @defgroup XXH64_impl XXH64 implementation
3380 * @ingroup impl
3381 *
3382 * Details on the XXH64 implementation.
3383 * @{
3384 */
3385/* #define rather that static const, to be used as initializers */
3386#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */
3387#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */
3388#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */
3389#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */
3390#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */
3391
3392#ifdef XXH_OLD_NAMES
3393# define PRIME64_1 XXH_PRIME64_1
3394# define PRIME64_2 XXH_PRIME64_2
3395# define PRIME64_3 XXH_PRIME64_3
3396# define PRIME64_4 XXH_PRIME64_4
3397# define PRIME64_5 XXH_PRIME64_5
3398#endif
3399
3400/*! @copydoc XXH32_round */
3401static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
3402{
3403 acc += input * XXH_PRIME64_2;
3404 acc = XXH_rotl64(acc, 31);
3405 acc *= XXH_PRIME64_1;
3406#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
3407 /*
3408 * DISABLE AUTOVECTORIZATION:
3409 * A compiler fence is used to prevent GCC and Clang from
3410 * autovectorizing the XXH64 loop (pragmas and attributes don't work for some
3411 * reason) without globally disabling AVX512.
3412 *
3413 * Autovectorization of XXH64 tends to be detrimental,
3414 * though the exact outcome may change depending on exact cpu and compiler version.
3415 * For information, it has been reported as detrimental for Skylake-X,
3416 * but possibly beneficial for Zen4.
3417 *
3418 * The default is to disable auto-vectorization,
3419 * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable.
3420 */
3421 XXH_COMPILER_GUARD(acc);
3422#endif
3423 return acc;
3424}
3425
3426static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
3427{
3428 val = XXH64_round(0, val);
3429 acc ^= val;
3430 acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
3431 return acc;
3432}
3433
3434/*! @copydoc XXH32_avalanche */
3435static xxh_u64 XXH64_avalanche(xxh_u64 hash)
3436{
3437 hash ^= hash >> 33;
3438 hash *= XXH_PRIME64_2;
3439 hash ^= hash >> 29;
3440 hash *= XXH_PRIME64_3;
3441 hash ^= hash >> 32;
3442 return hash;
3443}
3444
3445
3446#define XXH_get64bits(p) XXH_readLE64_align(p, align)
3447
3448/*!
3449 * @internal
3450 * @brief Processes the last 0-31 bytes of @p ptr.
3451 *
3452 * There may be up to 31 bytes remaining to consume from the input.
3453 * This final stage will digest them to ensure that all input bytes are present
3454 * in the final mix.
3455 *
3456 * @param hash The hash to finalize.
3457 * @param ptr The pointer to the remaining input.
3458 * @param len The remaining length, modulo 32.
3459 * @param align Whether @p ptr is aligned.
3460 * @return The finalized hash
3461 * @see XXH32_finalize().
3462 */
3463static XXH_PUREF xxh_u64
3464XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
3465{
3466 if (ptr==NULL) XXH_ASSERT(len == 0);
3467 len &= 31;
3468 while (len >= 8) {
3469 xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
3470 ptr += 8;
3471 hash ^= k1;
3472 hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
3473 len -= 8;
3474 }
3475 if (len >= 4) {
3476 hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
3477 ptr += 4;
3478 hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
3479 len -= 4;
3480 }
3481 while (len > 0) {
3482 hash ^= (*ptr++) * XXH_PRIME64_5;
3483 hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1;
3484 --len;
3485 }
3486 return XXH64_avalanche(hash);
3487}
3488
3489#ifdef XXH_OLD_NAMES
3490# define PROCESS1_64 XXH_PROCESS1_64
3491# define PROCESS4_64 XXH_PROCESS4_64
3492# define PROCESS8_64 XXH_PROCESS8_64
3493#else
3494# undef XXH_PROCESS1_64
3495# undef XXH_PROCESS4_64
3496# undef XXH_PROCESS8_64
3497#endif
3498
3499/*!
3500 * @internal
3501 * @brief The implementation for @ref XXH64().
3502 *
3503 * @param input , len , seed Directly passed from @ref XXH64().
3504 * @param align Whether @p input is aligned.
3505 * @return The calculated hash.
3506 */
3507XXH_FORCE_INLINE XXH_PUREF xxh_u64
3508XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
3509{
3510 xxh_u64 h64;
3511 if (input==NULL) XXH_ASSERT(len == 0);
3512
3513 if (len>=32) {
3514 const xxh_u8* const bEnd = input + len;
3515 const xxh_u8* const limit = bEnd - 31;
3516 xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
3517 xxh_u64 v2 = seed + XXH_PRIME64_2;
3518 xxh_u64 v3 = seed + 0;
3519 xxh_u64 v4 = seed - XXH_PRIME64_1;
3520
3521 do {
3522 v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
3523 v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
3524 v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
3525 v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
3526 } while (input<limit);
3527
3528 h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
3529 h64 = XXH64_mergeRound(h64, v1);
3530 h64 = XXH64_mergeRound(h64, v2);
3531 h64 = XXH64_mergeRound(h64, v3);
3532 h64 = XXH64_mergeRound(h64, v4);
3533
3534 } else {
3535 h64 = seed + XXH_PRIME64_5;
3536 }
3537
3538 h64 += (xxh_u64) len;
3539
3540 return XXH64_finalize(h64, input, len, align);
3541}
3542
3543
3544/*! @ingroup XXH64_family */
3545XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
3546{
3547#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
3548 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
3549 XXH64_state_t state;
3550 XXH64_reset(&state, seed);
3551 XXH64_update(&state, (const xxh_u8*)input, len);
3552 return XXH64_digest(&state);
3553#else
3554 if (XXH_FORCE_ALIGN_CHECK) {
3555 if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
3556 return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
3557 } }
3558
3559 return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
3560
3561#endif
3562}
3563
3564/******* Hash Streaming *******/
3565#ifndef XXH_NO_STREAM
3566/*! @ingroup XXH64_family*/
3567XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
3568{
3569 return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
3570}
3571/*! @ingroup XXH64_family */
3572XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
3573{
3574 XXH_free(statePtr);
3575 return XXH_OK;
3576}
3577
3578/*! @ingroup XXH64_family */
3579XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState)
3580{
3581 XXH_memcpy(dstState, srcState, sizeof(*dstState));
3582}
3583
3584/*! @ingroup XXH64_family */
3585XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed)
3586{
3587 XXH_ASSERT(statePtr != NULL);
3588 memset(statePtr, 0, sizeof(*statePtr));
3589 statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
3590 statePtr->v[1] = seed + XXH_PRIME64_2;
3591 statePtr->v[2] = seed + 0;
3592 statePtr->v[3] = seed - XXH_PRIME64_1;
3593 return XXH_OK;
3594}
3595
3596/*! @ingroup XXH64_family */
3597XXH_PUBLIC_API XXH_errorcode
3598XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len)
3599{
3600 if (input==NULL) {
3601 XXH_ASSERT(len == 0);
3602 return XXH_OK;
3603 }
3604
3605 { const xxh_u8* p = (const xxh_u8*)input;
3606 const xxh_u8* const bEnd = p + len;
3607
3608 state->total_len += len;
3609
3610 if (state->memsize + len < 32) { /* fill in tmp buffer */
3611 XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
3612 state->memsize += (xxh_u32)len;
3613 return XXH_OK;
3614 }
3615
3616 if (state->memsize) { /* tmp buffer is full */
3617 XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
3618 state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
3619 state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
3620 state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
3621 state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
3622 p += 32 - state->memsize;
3623 state->memsize = 0;
3624 }
3625
3626 if (p+32 <= bEnd) {
3627 const xxh_u8* const limit = bEnd - 32;
3628
3629 do {
3630 state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
3631 state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
3632 state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
3633 state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
3634 } while (p<=limit);
3635
3636 }
3637
3638 if (p < bEnd) {
3639 XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
3640 state->memsize = (unsigned)(bEnd-p);
3641 }
3642 }
3643
3644 return XXH_OK;
3645}
3646
3647
3648/*! @ingroup XXH64_family */
3649XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state)
3650{
3651 xxh_u64 h64;
3652
3653 if (state->total_len >= 32) {
3654 h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
3655 h64 = XXH64_mergeRound(h64, state->v[0]);
3656 h64 = XXH64_mergeRound(h64, state->v[1]);
3657 h64 = XXH64_mergeRound(h64, state->v[2]);
3658 h64 = XXH64_mergeRound(h64, state->v[3]);
3659 } else {
3660 h64 = state->v[2] /*seed*/ + XXH_PRIME64_5;
3661 }
3662
3663 h64 += (xxh_u64) state->total_len;
3664
3665 return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
3666}
3667#endif /* !XXH_NO_STREAM */
3668
3669/******* Canonical representation *******/
3670
3671/*! @ingroup XXH64_family */
3672XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash)
3673{
3674 XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
3675 if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
3676 XXH_memcpy(dst, &hash, sizeof(*dst));
3677}
3678
3679/*! @ingroup XXH64_family */
3680XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src)
3681{
3682 return XXH_readBE64(src);
3683}
3684
3685#ifndef XXH_NO_XXH3
3686
3687/* *********************************************************************
3688* XXH3
3689* New generation hash designed for speed on small keys and vectorization
3690************************************************************************ */
3691/*!
3692 * @}
3693 * @defgroup XXH3_impl XXH3 implementation
3694 * @ingroup impl
3695 * @{
3696 */
3697
3698/* === Compiler specifics === */
3699
3700#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
3701# define XXH_RESTRICT /* disable */
3702#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
3703# define XXH_RESTRICT restrict
3704#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \
3705 || (defined (__clang__)) \
3706 || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \
3707 || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300))
3708/*
3709 * There are a LOT more compilers that recognize __restrict but this
3710 * covers the major ones.
3711 */
3712# define XXH_RESTRICT __restrict
3713#else
3714# define XXH_RESTRICT /* disable */
3715#endif
3716
3717#if (defined(__GNUC__) && (__GNUC__ >= 3)) \
3718 || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
3719 || defined(__clang__)
3720# define XXH_likely(x) __builtin_expect(x, 1)
3721# define XXH_unlikely(x) __builtin_expect(x, 0)
3722#else
3723# define XXH_likely(x) (x)
3724# define XXH_unlikely(x) (x)
3725#endif
3726
3727#ifndef XXH_HAS_INCLUDE
3728# ifdef __has_include
3729/*
3730 * Not defined as XXH_HAS_INCLUDE(x) (function-like) because
3731 * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion)
3732 */
3733# define XXH_HAS_INCLUDE __has_include
3734# else
3735# define XXH_HAS_INCLUDE(x) 0
3736# endif
3737#endif
3738
3739#if defined(__GNUC__) || defined(__clang__)
3740# if defined(__ARM_FEATURE_SVE)
3741# include <arm_sve.h>
3742# endif
3743# if defined(__ARM_NEON__) || defined(__ARM_NEON) \
3744 || (defined(_M_ARM) && _M_ARM >= 7) \
3745 || defined(_M_ARM64) || defined(_M_ARM64EC) \
3746 || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */
3747# define inline __inline__ /* circumvent a clang bug */
3748# include <arm_neon.h>
3749# undef inline
3750# elif defined(__AVX2__)
3751# include <immintrin.h>
3752# elif defined(__SSE2__)
3753# include <emmintrin.h>
3754# endif
3755#endif
3756
3757#if defined(_MSC_VER)
3758# include <intrin.h>
3759#endif
3760
3761/*
3762 * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
3763 * remaining a true 64-bit/128-bit hash function.
3764 *
3765 * This is done by prioritizing a subset of 64-bit operations that can be
3766 * emulated without too many steps on the average 32-bit machine.
3767 *
3768 * For example, these two lines seem similar, and run equally fast on 64-bit:
3769 *
3770 * xxh_u64 x;
3771 * x ^= (x >> 47); // good
3772 * x ^= (x >> 13); // bad
3773 *
3774 * However, to a 32-bit machine, there is a major difference.
3775 *
3776 * x ^= (x >> 47) looks like this:
3777 *
3778 * x.lo ^= (x.hi >> (47 - 32));
3779 *
3780 * while x ^= (x >> 13) looks like this:
3781 *
3782 * // note: funnel shifts are not usually cheap.
3783 * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
3784 * x.hi ^= (x.hi >> 13);
3785 *
3786 * The first one is significantly faster than the second, simply because the
3787 * shift is larger than 32. This means:
3788 * - All the bits we need are in the upper 32 bits, so we can ignore the lower
3789 * 32 bits in the shift.
3790 * - The shift result will always fit in the lower 32 bits, and therefore,
3791 * we can ignore the upper 32 bits in the xor.
3792 *
3793 * Thanks to this optimization, XXH3 only requires these features to be efficient:
3794 *
3795 * - Usable unaligned access
3796 * - A 32-bit or 64-bit ALU
3797 * - If 32-bit, a decent ADC instruction
3798 * - A 32 or 64-bit multiply with a 64-bit result
3799 * - For the 128-bit variant, a decent byteswap helps short inputs.
3800 *
3801 * The first two are already required by XXH32, and almost all 32-bit and 64-bit
3802 * platforms which can run XXH32 can run XXH3 efficiently.
3803 *
3804 * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
3805 * notable exception.
3806 *
3807 * First of all, Thumb-1 lacks support for the UMULL instruction which
3808 * performs the important long multiply. This means numerous __aeabi_lmul
3809 * calls.
3810 *
3811 * Second of all, the 8 functional registers are just not enough.
3812 * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
3813 * Lo registers, and this shuffling results in thousands more MOVs than A32.
3814 *
3815 * A32 and T32 don't have this limitation. They can access all 14 registers,
3816 * do a 32->64 multiply with UMULL, and the flexible operand allowing free
3817 * shifts is helpful, too.
3818 *
3819 * Therefore, we do a quick sanity check.
3820 *
3821 * If compiling Thumb-1 for a target which supports ARM instructions, we will
3822 * emit a warning, as it is not a "sane" platform to compile for.
3823 *
3824 * Usually, if this happens, it is because of an accident and you probably need
3825 * to specify -march, as you likely meant to compile for a newer architecture.
3826 *
3827 * Credit: large sections of the vectorial and asm source code paths
3828 * have been contributed by @easyaspi314
3829 */
3830#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
3831# warning "XXH3 is highly inefficient without ARM or Thumb-2."
3832#endif
3833
3834/* ==========================================
3835 * Vectorization detection
3836 * ========================================== */
3837
3838#ifdef XXH_DOXYGEN
3839/*!
3840 * @ingroup tuning
3841 * @brief Overrides the vectorization implementation chosen for XXH3.
3842 *
3843 * Can be defined to 0 to disable SIMD or any of the values mentioned in
3844 * @ref XXH_VECTOR_TYPE.
3845 *
3846 * If this is not defined, it uses predefined macros to determine the best
3847 * implementation.
3848 */
3849# define XXH_VECTOR XXH_SCALAR
3850/*!
3851 * @ingroup tuning
3852 * @brief Possible values for @ref XXH_VECTOR.
3853 *
3854 * Note that these are actually implemented as macros.
3855 *
3856 * If this is not defined, it is detected automatically.
3857 * internal macro XXH_X86DISPATCH overrides this.
3858 */
3859enum XXH_VECTOR_TYPE /* fake enum */ {
3860 XXH_SCALAR = 0, /*!< Portable scalar version */
3861 XXH_SSE2 = 1, /*!<
3862 * SSE2 for Pentium 4, Opteron, all x86_64.
3863 *
3864 * @note SSE2 is also guaranteed on Windows 10, macOS, and
3865 * Android x86.
3866 */
3867 XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */
3868 XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */
3869 XXH_NEON = 4, /*!<
3870 * NEON for most ARMv7-A, all AArch64, and WASM SIMD128
3871 * via the SIMDeverywhere polyfill provided with the
3872 * Emscripten SDK.
3873 */
3874 XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */
3875 XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */
3876};
3877/*!
3878 * @ingroup tuning
3879 * @brief Selects the minimum alignment for XXH3's accumulators.
3880 *
3881 * When using SIMD, this should match the alignment required for said vector
3882 * type, so, for example, 32 for AVX2.
3883 *
3884 * Default: Auto detected.
3885 */
3886# define XXH_ACC_ALIGN 8
3887#endif
3888
3889/* Actual definition */
3890#ifndef XXH_DOXYGEN
3891# define XXH_SCALAR 0
3892# define XXH_SSE2 1
3893# define XXH_AVX2 2
3894# define XXH_AVX512 3
3895# define XXH_NEON 4
3896# define XXH_VSX 5
3897# define XXH_SVE 6
3898#endif
3899
3900#ifndef XXH_VECTOR /* can be defined on command line */
3901# if defined(__ARM_FEATURE_SVE)
3902# define XXH_VECTOR XXH_SVE
3903# elif ( \
3904 defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \
3905 || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \
3906 || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \
3907 ) && ( \
3908 defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \
3909 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
3910 )
3911# define XXH_VECTOR XXH_NEON
3912# elif defined(__AVX512F__)
3913# define XXH_VECTOR XXH_AVX512
3914# elif defined(__AVX2__)
3915# define XXH_VECTOR XXH_AVX2
3916# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
3917# define XXH_VECTOR XXH_SSE2
3918# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
3919 || (defined(__s390x__) && defined(__VEC__)) \
3920 && defined(__GNUC__) /* TODO: IBM XL */
3921# define XXH_VECTOR XXH_VSX
3922# else
3923# define XXH_VECTOR XXH_SCALAR
3924# endif
3925#endif
3926
3927/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */
3928#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE)
3929# ifdef _MSC_VER
3930# pragma warning(once : 4606)
3931# else
3932# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead."
3933# endif
3934# undef XXH_VECTOR
3935# define XXH_VECTOR XXH_SCALAR
3936#endif
3937
3938/*
3939 * Controls the alignment of the accumulator,
3940 * for compatibility with aligned vector loads, which are usually faster.
3941 */
3942#ifndef XXH_ACC_ALIGN
3943# if defined(XXH_X86DISPATCH)
3944# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */
3945# elif XXH_VECTOR == XXH_SCALAR /* scalar */
3946# define XXH_ACC_ALIGN 8
3947# elif XXH_VECTOR == XXH_SSE2 /* sse2 */
3948# define XXH_ACC_ALIGN 16
3949# elif XXH_VECTOR == XXH_AVX2 /* avx2 */
3950# define XXH_ACC_ALIGN 32
3951# elif XXH_VECTOR == XXH_NEON /* neon */
3952# define XXH_ACC_ALIGN 16
3953# elif XXH_VECTOR == XXH_VSX /* vsx */
3954# define XXH_ACC_ALIGN 16
3955# elif XXH_VECTOR == XXH_AVX512 /* avx512 */
3956# define XXH_ACC_ALIGN 64
3957# elif XXH_VECTOR == XXH_SVE /* sve */
3958# define XXH_ACC_ALIGN 64
3959# endif
3960#endif
3961
3962#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
3963 || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
3964# define XXH_SEC_ALIGN XXH_ACC_ALIGN
3965#elif XXH_VECTOR == XXH_SVE
3966# define XXH_SEC_ALIGN XXH_ACC_ALIGN
3967#else
3968# define XXH_SEC_ALIGN 8
3969#endif
3970
3971#if defined(__GNUC__) || defined(__clang__)
3972# define XXH_ALIASING __attribute__((__may_alias__))
3973#else
3974# define XXH_ALIASING /* nothing */
3975#endif
3976
3977/*
3978 * UGLY HACK:
3979 * GCC usually generates the best code with -O3 for xxHash.
3980 *
3981 * However, when targeting AVX2, it is overzealous in its unrolling resulting
3982 * in code roughly 3/4 the speed of Clang.
3983 *
3984 * There are other issues, such as GCC splitting _mm256_loadu_si256 into
3985 * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
3986 * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
3987 *
3988 * That is why when compiling the AVX2 version, it is recommended to use either
3989 * -O2 -mavx2 -march=haswell
3990 * or
3991 * -O2 -mavx2 -mno-avx256-split-unaligned-load
3992 * for decent performance, or to use Clang instead.
3993 *
3994 * Fortunately, we can control the first one with a pragma that forces GCC into
3995 * -O2, but the other one we can't control without "failed to inline always
3996 * inline function due to target mismatch" warnings.
3997 */
3998#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
3999 && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
4000 && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
4001# pragma GCC push_options
4002# pragma GCC optimize("-O2")
4003#endif
4004
4005#if XXH_VECTOR == XXH_NEON
4006
4007/*
4008 * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3
4009 * optimizes out the entire hashLong loop because of the aliasing violation.
4010 *
4011 * However, GCC is also inefficient at load-store optimization with vld1q/vst1q,
4012 * so the only option is to mark it as aliasing.
4013 */
4014typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING;
4015
4016/*!
4017 * @internal
4018 * @brief `vld1q_u64` but faster and alignment-safe.
4019 *
4020 * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only
4021 * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86).
4022 *
4023 * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it
4024 * prohibits load-store optimizations. Therefore, a direct dereference is used.
4025 *
4026 * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe
4027 * unaligned load.
4028 */
4029#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)
4030XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */
4031{
4032 return *(xxh_aliasing_uint64x2_t const *)ptr;
4033}
4034#else
4035XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr)
4036{
4037 return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr));
4038}
4039#endif
4040
4041/*!
4042 * @internal
4043 * @brief `vmlal_u32` on low and high halves of a vector.
4044 *
4045 * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with
4046 * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32`
4047 * with `vmlal_u32`.
4048 */
4049#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11
4050XXH_FORCE_INLINE uint64x2_t
4051XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
4052{
4053 /* Inline assembly is the only way */
4054 __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs));
4055 return acc;
4056}
4057XXH_FORCE_INLINE uint64x2_t
4058XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
4059{
4060 /* This intrinsic works as expected */
4061 return vmlal_high_u32(acc, lhs, rhs);
4062}
4063#else
4064/* Portable intrinsic versions */
4065XXH_FORCE_INLINE uint64x2_t
4066XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
4067{
4068 return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs));
4069}
4070/*! @copydoc XXH_vmlal_low_u32
4071 * Assume the compiler converts this to vmlal_high_u32 on aarch64 */
4072XXH_FORCE_INLINE uint64x2_t
4073XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
4074{
4075 return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs));
4076}
4077#endif
4078
4079/*!
4080 * @ingroup tuning
4081 * @brief Controls the NEON to scalar ratio for XXH3
4082 *
4083 * This can be set to 2, 4, 6, or 8.
4084 *
4085 * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used.
4086 *
4087 * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those
4088 * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU
4089 * bandwidth.
4090 *
4091 * This is even more noticeable on the more advanced cores like the Cortex-A76 which
4092 * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once.
4093 *
4094 * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes
4095 * and 2 scalar lanes, which is chosen by default.
4096 *
4097 * This does not apply to Apple processors or 32-bit processors, which run better with
4098 * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes.
4099 *
4100 * This change benefits CPUs with large micro-op buffers without negatively affecting
4101 * most other CPUs:
4102 *
4103 * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. |
4104 * |:----------------------|:--------------------|----------:|-----------:|------:|
4105 * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% |
4106 * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% |
4107 * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% |
4108 * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% |
4109 *
4110 * It also seems to fix some bad codegen on GCC, making it almost as fast as clang.
4111 *
4112 * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning
4113 * it effectively becomes worse 4.
4114 *
4115 * @see XXH3_accumulate_512_neon()
4116 */
4117# ifndef XXH3_NEON_LANES
4118# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \
4119 && !defined(__APPLE__) && XXH_SIZE_OPT <= 0
4120# define XXH3_NEON_LANES 6
4121# else
4122# define XXH3_NEON_LANES XXH_ACC_NB
4123# endif
4124# endif
4125#endif /* XXH_VECTOR == XXH_NEON */
4126
4127/*
4128 * VSX and Z Vector helpers.
4129 *
4130 * This is very messy, and any pull requests to clean this up are welcome.
4131 *
4132 * There are a lot of problems with supporting VSX and s390x, due to
4133 * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
4134 */
4135#if XXH_VECTOR == XXH_VSX
4136/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`,
4137 * and `pixel`. This is a problem for obvious reasons.
4138 *
4139 * These keywords are unnecessary; the spec literally says they are
4140 * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd
4141 * after including the header.
4142 *
4143 * We use pragma push_macro/pop_macro to keep the namespace clean. */
4144# pragma push_macro("bool")
4145# pragma push_macro("vector")
4146# pragma push_macro("pixel")
4147/* silence potential macro redefined warnings */
4148# undef bool
4149# undef vector
4150# undef pixel
4151
4152# if defined(__s390x__)
4153# include <s390intrin.h>
4154# else
4155# include <altivec.h>
4156# endif
4157
4158/* Restore the original macro values, if applicable. */
4159# pragma pop_macro("pixel")
4160# pragma pop_macro("vector")
4161# pragma pop_macro("bool")
4162
4163typedef __vector unsigned long long xxh_u64x2;
4164typedef __vector unsigned char xxh_u8x16;
4165typedef __vector unsigned xxh_u32x4;
4166
4167/*
4168 * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue.
4169 */
4170typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING;
4171
4172# ifndef XXH_VSX_BE
4173# if defined(__BIG_ENDIAN__) \
4174 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
4175# define XXH_VSX_BE 1
4176# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
4177# warning "-maltivec=be is not recommended. Please use native endianness."
4178# define XXH_VSX_BE 1
4179# else
4180# define XXH_VSX_BE 0
4181# endif
4182# endif /* !defined(XXH_VSX_BE) */
4183
4184# if XXH_VSX_BE
4185# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
4186# define XXH_vec_revb vec_revb
4187# else
4188/*!
4189 * A polyfill for POWER9's vec_revb().
4190 */
4191XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
4192{
4193 xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
4194 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
4195 return vec_perm(val, val, vByteSwap);
4196}
4197# endif
4198# endif /* XXH_VSX_BE */
4199
4200/*!
4201 * Performs an unaligned vector load and byte swaps it on big endian.
4202 */
4203XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
4204{
4205 xxh_u64x2 ret;
4206 XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
4207# if XXH_VSX_BE
4208 ret = XXH_vec_revb(ret);
4209# endif
4210 return ret;
4211}
4212
4213/*
4214 * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
4215 *
4216 * These intrinsics weren't added until GCC 8, despite existing for a while,
4217 * and they are endian dependent. Also, their meaning swap depending on version.
4218 * */
4219# if defined(__s390x__)
4220 /* s390x is always big endian, no issue on this platform */
4221# define XXH_vec_mulo vec_mulo
4222# define XXH_vec_mule vec_mule
4223# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__)
4224/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
4225 /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */
4226# define XXH_vec_mulo __builtin_altivec_vmulouw
4227# define XXH_vec_mule __builtin_altivec_vmuleuw
4228# else
4229/* gcc needs inline assembly */
4230/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
4231XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
4232{
4233 xxh_u64x2 result;
4234 __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
4235 return result;
4236}
4237XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
4238{
4239 xxh_u64x2 result;
4240 __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
4241 return result;
4242}
4243# endif /* XXH_vec_mulo, XXH_vec_mule */
4244#endif /* XXH_VECTOR == XXH_VSX */
4245
4246#if XXH_VECTOR == XXH_SVE
4247#define ACCRND(acc, offset) \
4248do { \
4249 svuint64_t input_vec = svld1_u64(mask, xinput + offset); \
4250 svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \
4251 svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \
4252 svuint64_t swapped = svtbl_u64(input_vec, kSwap); \
4253 svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \
4254 svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \
4255 svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \
4256 acc = svadd_u64_x(mask, acc, mul); \
4257} while (0)
4258#endif /* XXH_VECTOR == XXH_SVE */
4259
4260/* prefetch
4261 * can be disabled, by declaring XXH_NO_PREFETCH build macro */
4262#if defined(XXH_NO_PREFETCH)
4263# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
4264#else
4265# if XXH_SIZE_OPT >= 1
4266# define XXH_PREFETCH(ptr) (void)(ptr)
4267# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */
4268# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
4269# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
4270# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
4271# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
4272# else
4273# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
4274# endif
4275#endif /* XXH_NO_PREFETCH */
4276
4277
4278/* ==========================================
4279 * XXH3 default settings
4280 * ========================================== */
4281
4282#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */
4283
4284#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
4285# error "default keyset is not large enough"
4286#endif
4287
4288/*! Pseudorandom secret taken directly from FARSH. */
4289XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
4290 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
4291 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
4292 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
4293 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
4294 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
4295 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
4296 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
4297 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
4298 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
4299 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
4300 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
4301 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
4302};
4303
4304static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */
4305static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */
4306
4307#ifdef XXH_OLD_NAMES
4308# define kSecret XXH3_kSecret
4309#endif
4310
4311#ifdef XXH_DOXYGEN
4312/*!
4313 * @brief Calculates a 32-bit to 64-bit long multiply.
4314 *
4315 * Implemented as a macro.
4316 *
4317 * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
4318 * need to (but it shouldn't need to anyways, it is about 7 instructions to do
4319 * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
4320 * use that instead of the normal method.
4321 *
4322 * If you are compiling for platforms like Thumb-1 and don't have a better option,
4323 * you may also want to write your own long multiply routine here.
4324 *
4325 * @param x, y Numbers to be multiplied
4326 * @return 64-bit product of the low 32 bits of @p x and @p y.
4327 */
4328XXH_FORCE_INLINE xxh_u64
4329XXH_mult32to64(xxh_u64 x, xxh_u64 y)
4330{
4331 return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
4332}
4333#elif defined(_MSC_VER) && defined(_M_IX86)
4334# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
4335#else
4336/*
4337 * Downcast + upcast is usually better than masking on older compilers like
4338 * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
4339 *
4340 * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
4341 * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
4342 */
4343# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
4344#endif
4345
4346/*!
4347 * @brief Calculates a 64->128-bit long multiply.
4348 *
4349 * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
4350 * version.
4351 *
4352 * @param lhs , rhs The 64-bit integers to be multiplied
4353 * @return The 128-bit result represented in an @ref XXH128_hash_t.
4354 */
4355static XXH128_hash_t
4356XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
4357{
4358 /*
4359 * GCC/Clang __uint128_t method.
4360 *
4361 * On most 64-bit targets, GCC and Clang define a __uint128_t type.
4362 * This is usually the best way as it usually uses a native long 64-bit
4363 * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
4364 *
4365 * Usually.
4366 *
4367 * Despite being a 32-bit platform, Clang (and emscripten) define this type
4368 * despite not having the arithmetic for it. This results in a laggy
4369 * compiler builtin call which calculates a full 128-bit multiply.
4370 * In that case it is best to use the portable one.
4371 * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
4372 */
4373#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \
4374 && defined(__SIZEOF_INT128__) \
4375 || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
4376
4377 __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
4378 XXH128_hash_t r128;
4379 r128.low64 = (xxh_u64)(product);
4380 r128.high64 = (xxh_u64)(product >> 64);
4381 return r128;
4382
4383 /*
4384 * MSVC for x64's _umul128 method.
4385 *
4386 * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
4387 *
4388 * This compiles to single operand MUL on x64.
4389 */
4390#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC)
4391
4392#ifndef _MSC_VER
4393# pragma intrinsic(_umul128)
4394#endif
4395 xxh_u64 product_high;
4396 xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
4397 XXH128_hash_t r128;
4398 r128.low64 = product_low;
4399 r128.high64 = product_high;
4400 return r128;
4401
4402 /*
4403 * MSVC for ARM64's __umulh method.
4404 *
4405 * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
4406 */
4407#elif defined(_M_ARM64) || defined(_M_ARM64EC)
4408
4409#ifndef _MSC_VER
4410# pragma intrinsic(__umulh)
4411#endif
4412 XXH128_hash_t r128;
4413 r128.low64 = lhs * rhs;
4414 r128.high64 = __umulh(lhs, rhs);
4415 return r128;
4416
4417#else
4418 /*
4419 * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
4420 *
4421 * This is a fast and simple grade school multiply, which is shown below
4422 * with base 10 arithmetic instead of base 0x100000000.
4423 *
4424 * 9 3 // D2 lhs = 93
4425 * x 7 5 // D2 rhs = 75
4426 * ----------
4427 * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
4428 * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
4429 * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
4430 * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
4431 * ---------
4432 * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
4433 * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
4434 * ---------
4435 * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
4436 *
4437 * The reasons for adding the products like this are:
4438 * 1. It avoids manual carry tracking. Just like how
4439 * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
4440 * This avoids a lot of complexity.
4441 *
4442 * 2. It hints for, and on Clang, compiles to, the powerful UMAAL
4443 * instruction available in ARM's Digital Signal Processing extension
4444 * in 32-bit ARMv6 and later, which is shown below:
4445 *
4446 * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
4447 * {
4448 * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
4449 * *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
4450 * *RdHi = (xxh_u32)(product >> 32);
4451 * }
4452 *
4453 * This instruction was designed for efficient long multiplication, and
4454 * allows this to be calculated in only 4 instructions at speeds
4455 * comparable to some 64-bit ALUs.
4456 *
4457 * 3. It isn't terrible on other platforms. Usually this will be a couple
4458 * of 32-bit ADD/ADCs.
4459 */
4460
4461 /* First calculate all of the cross products. */
4462 xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
4463 xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF);
4464 xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
4465 xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32);
4466
4467 /* Now add the products together. These will never overflow. */
4468 xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
4469 xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
4470 xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
4471
4472 XXH128_hash_t r128;
4473 r128.low64 = lower;
4474 r128.high64 = upper;
4475 return r128;
4476#endif
4477}
4478
4479/*!
4480 * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
4481 *
4482 * The reason for the separate function is to prevent passing too many structs
4483 * around by value. This will hopefully inline the multiply, but we don't force it.
4484 *
4485 * @param lhs , rhs The 64-bit integers to multiply
4486 * @return The low 64 bits of the product XOR'd by the high 64 bits.
4487 * @see XXH_mult64to128()
4488 */
4489static xxh_u64
4490XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
4491{
4492 XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
4493 return product.low64 ^ product.high64;
4494}
4495
4496/*! Seems to produce slightly better code on GCC for some reason. */
4497XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
4498{
4499 XXH_ASSERT(0 <= shift && shift < 64);
4500 return v64 ^ (v64 >> shift);
4501}
4502
4503/*
4504 * This is a fast avalanche stage,
4505 * suitable when input bits are already partially mixed
4506 */
4507static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
4508{
4509 h64 = XXH_xorshift64(h64, 37);
4510 h64 *= PRIME_MX1;
4511 h64 = XXH_xorshift64(h64, 32);
4512 return h64;
4513}
4514
4515/*
4516 * This is a stronger avalanche,
4517 * inspired by Pelle Evensen's rrmxmx
4518 * preferable when input has not been previously mixed
4519 */
4520static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
4521{
4522 /* this mix is inspired by Pelle Evensen's rrmxmx */
4523 h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
4524 h64 *= PRIME_MX2;
4525 h64 ^= (h64 >> 35) + len ;
4526 h64 *= PRIME_MX2;
4527 return XXH_xorshift64(h64, 28);
4528}
4529
4530
4531/* ==========================================
4532 * Short keys
4533 * ==========================================
4534 * One of the shortcomings of XXH32 and XXH64 was that their performance was
4535 * sub-optimal on short lengths. It used an iterative algorithm which strongly
4536 * favored lengths that were a multiple of 4 or 8.
4537 *
4538 * Instead of iterating over individual inputs, we use a set of single shot
4539 * functions which piece together a range of lengths and operate in constant time.
4540 *
4541 * Additionally, the number of multiplies has been significantly reduced. This
4542 * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
4543 *
4544 * Depending on the platform, this may or may not be faster than XXH32, but it
4545 * is almost guaranteed to be faster than XXH64.
4546 */
4547
4548/*
4549 * At very short lengths, there isn't enough input to fully hide secrets, or use
4550 * the entire secret.
4551 *
4552 * There is also only a limited amount of mixing we can do before significantly
4553 * impacting performance.
4554 *
4555 * Therefore, we use different sections of the secret and always mix two secret
4556 * samples with an XOR. This should have no effect on performance on the
4557 * seedless or withSeed variants because everything _should_ be constant folded
4558 * by modern compilers.
4559 *
4560 * The XOR mixing hides individual parts of the secret and increases entropy.
4561 *
4562 * This adds an extra layer of strength for custom secrets.
4563 */
4564XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4565XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4566{
4567 XXH_ASSERT(input != NULL);
4568 XXH_ASSERT(1 <= len && len <= 3);
4569 XXH_ASSERT(secret != NULL);
4570 /*
4571 * len = 1: combined = { input[0], 0x01, input[0], input[0] }
4572 * len = 2: combined = { input[1], 0x02, input[0], input[1] }
4573 * len = 3: combined = { input[2], 0x03, input[0], input[1] }
4574 */
4575 { xxh_u8 const c1 = input[0];
4576 xxh_u8 const c2 = input[len >> 1];
4577 xxh_u8 const c3 = input[len - 1];
4578 xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24)
4579 | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
4580 xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
4581 xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
4582 return XXH64_avalanche(keyed);
4583 }
4584}
4585
4586XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4587XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4588{
4589 XXH_ASSERT(input != NULL);
4590 XXH_ASSERT(secret != NULL);
4591 XXH_ASSERT(4 <= len && len <= 8);
4592 seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
4593 { xxh_u32 const input1 = XXH_readLE32(input);
4594 xxh_u32 const input2 = XXH_readLE32(input + len - 4);
4595 xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
4596 xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
4597 xxh_u64 const keyed = input64 ^ bitflip;
4598 return XXH3_rrmxmx(keyed, len);
4599 }
4600}
4601
4602XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4603XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4604{
4605 XXH_ASSERT(input != NULL);
4606 XXH_ASSERT(secret != NULL);
4607 XXH_ASSERT(9 <= len && len <= 16);
4608 { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
4609 xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
4610 xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1;
4611 xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
4612 xxh_u64 const acc = len
4613 + XXH_swap64(input_lo) + input_hi
4614 + XXH3_mul128_fold64(input_lo, input_hi);
4615 return XXH3_avalanche(acc);
4616 }
4617}
4618
4619XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4620XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4621{
4622 XXH_ASSERT(len <= 16);
4623 { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed);
4624 if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
4625 if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
4626 return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
4627 }
4628}
4629
4630/*
4631 * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
4632 * multiplication by zero, affecting hashes of lengths 17 to 240.
4633 *
4634 * However, they are very unlikely.
4635 *
4636 * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
4637 * unseeded non-cryptographic hashes, it does not attempt to defend itself
4638 * against specially crafted inputs, only random inputs.
4639 *
4640 * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
4641 * cancelling out the secret is taken an arbitrary number of times (addressed
4642 * in XXH3_accumulate_512), this collision is very unlikely with random inputs
4643 * and/or proper seeding:
4644 *
4645 * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
4646 * function that is only called up to 16 times per hash with up to 240 bytes of
4647 * input.
4648 *
4649 * This is not too bad for a non-cryptographic hash function, especially with
4650 * only 64 bit outputs.
4651 *
4652 * The 128-bit variant (which trades some speed for strength) is NOT affected
4653 * by this, although it is always a good idea to use a proper seed if you care
4654 * about strength.
4655 */
4656XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
4657 const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
4658{
4659#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
4660 && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \
4661 && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */
4662 /*
4663 * UGLY HACK:
4664 * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
4665 * slower code.
4666 *
4667 * By forcing seed64 into a register, we disrupt the cost model and
4668 * cause it to scalarize. See `XXH32_round()`
4669 *
4670 * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
4671 * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
4672 * GCC 9.2, despite both emitting scalar code.
4673 *
4674 * GCC generates much better scalar code than Clang for the rest of XXH3,
4675 * which is why finding a more optimal codepath is an interest.
4676 */
4677 XXH_COMPILER_GUARD(seed64);
4678#endif
4679 { xxh_u64 const input_lo = XXH_readLE64(input);
4680 xxh_u64 const input_hi = XXH_readLE64(input+8);
4681 return XXH3_mul128_fold64(
4682 input_lo ^ (XXH_readLE64(secret) + seed64),
4683 input_hi ^ (XXH_readLE64(secret+8) - seed64)
4684 );
4685 }
4686}
4687
4688/* For mid range keys, XXH3 uses a Mum-hash variant. */
4689XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4690XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
4691 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
4692 XXH64_hash_t seed)
4693{
4694 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
4695 XXH_ASSERT(16 < len && len <= 128);
4696
4697 { xxh_u64 acc = len * XXH_PRIME64_1;
4698#if XXH_SIZE_OPT >= 1
4699 /* Smaller and cleaner, but slightly slower. */
4700 unsigned int i = (unsigned int)(len - 1) / 32;
4701 do {
4702 acc += XXH3_mix16B(input+16 * i, secret+32*i, seed);
4703 acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed);
4704 } while (i-- != 0);
4705#else
4706 if (len > 32) {
4707 if (len > 64) {
4708 if (len > 96) {
4709 acc += XXH3_mix16B(input+48, secret+96, seed);
4710 acc += XXH3_mix16B(input+len-64, secret+112, seed);
4711 }
4712 acc += XXH3_mix16B(input+32, secret+64, seed);
4713 acc += XXH3_mix16B(input+len-48, secret+80, seed);
4714 }
4715 acc += XXH3_mix16B(input+16, secret+32, seed);
4716 acc += XXH3_mix16B(input+len-32, secret+48, seed);
4717 }
4718 acc += XXH3_mix16B(input+0, secret+0, seed);
4719 acc += XXH3_mix16B(input+len-16, secret+16, seed);
4720#endif
4721 return XXH3_avalanche(acc);
4722 }
4723}
4724
4725XXH_NO_INLINE XXH_PUREF XXH64_hash_t
4726XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
4727 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
4728 XXH64_hash_t seed)
4729{
4730 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
4731 XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
4732
4733 #define XXH3_MIDSIZE_STARTOFFSET 3
4734 #define XXH3_MIDSIZE_LASTOFFSET 17
4735
4736 { xxh_u64 acc = len * XXH_PRIME64_1;
4737 xxh_u64 acc_end;
4738 unsigned int const nbRounds = (unsigned int)len / 16;
4739 unsigned int i;
4740 XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
4741 for (i=0; i<8; i++) {
4742 acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
4743 }
4744 /* last bytes */
4745 acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
4746 XXH_ASSERT(nbRounds >= 8);
4747 acc = XXH3_avalanche(acc);
4748#if defined(__clang__) /* Clang */ \
4749 && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
4750 && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
4751 /*
4752 * UGLY HACK:
4753 * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
4754 * In everywhere else, it uses scalar code.
4755 *
4756 * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
4757 * would still be slower than UMAAL (see XXH_mult64to128).
4758 *
4759 * Unfortunately, Clang doesn't handle the long multiplies properly and
4760 * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
4761 * scalarized into an ugly mess of VMOV.32 instructions.
4762 *
4763 * This mess is difficult to avoid without turning autovectorization
4764 * off completely, but they are usually relatively minor and/or not
4765 * worth it to fix.
4766 *
4767 * This loop is the easiest to fix, as unlike XXH32, this pragma
4768 * _actually works_ because it is a loop vectorization instead of an
4769 * SLP vectorization.
4770 */
4771 #pragma clang loop vectorize(disable)
4772#endif
4773 for (i=8 ; i < nbRounds; i++) {
4774 /*
4775 * Prevents clang for unrolling the acc loop and interleaving with this one.
4776 */
4777 XXH_COMPILER_GUARD(acc);
4778 acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
4779 }
4780 return XXH3_avalanche(acc + acc_end);
4781 }
4782}
4783
4784
4785/* ======= Long Keys ======= */
4786
4787#define XXH_STRIPE_LEN 64
4788#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */
4789#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
4790
4791#ifdef XXH_OLD_NAMES
4792# define STRIPE_LEN XXH_STRIPE_LEN
4793# define ACC_NB XXH_ACC_NB
4794#endif
4795
4796#ifndef XXH_PREFETCH_DIST
4797# ifdef __clang__
4798# define XXH_PREFETCH_DIST 320
4799# else
4800# if (XXH_VECTOR == XXH_AVX512)
4801# define XXH_PREFETCH_DIST 512
4802# else
4803# define XXH_PREFETCH_DIST 384
4804# endif
4805# endif /* __clang__ */
4806#endif /* XXH_PREFETCH_DIST */
4807
4808/*
4809 * These macros are to generate an XXH3_accumulate() function.
4810 * The two arguments select the name suffix and target attribute.
4811 *
4812 * The name of this symbol is XXH3_accumulate_<name>() and it calls
4813 * XXH3_accumulate_512_<name>().
4814 *
4815 * It may be useful to hand implement this function if the compiler fails to
4816 * optimize the inline function.
4817 */
4818#define XXH3_ACCUMULATE_TEMPLATE(name) \
4819void \
4820XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \
4821 const xxh_u8* XXH_RESTRICT input, \
4822 const xxh_u8* XXH_RESTRICT secret, \
4823 size_t nbStripes) \
4824{ \
4825 size_t n; \
4826 for (n = 0; n < nbStripes; n++ ) { \
4827 const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \
4828 XXH_PREFETCH(in + XXH_PREFETCH_DIST); \
4829 XXH3_accumulate_512_##name( \
4830 acc, \
4831 in, \
4832 secret + n*XXH_SECRET_CONSUME_RATE); \
4833 } \
4834}
4835
4836
4837XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
4838{
4839 if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
4840 XXH_memcpy(dst, &v64, sizeof(v64));
4841}
4842
4843/* Several intrinsic functions below are supposed to accept __int64 as argument,
4844 * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
4845 * However, several environments do not define __int64 type,
4846 * requiring a workaround.
4847 */
4848#if !defined (__VMS) \
4849 && (defined (__cplusplus) \
4850 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
4851 typedef int64_t xxh_i64;
4852#else
4853 /* the following type must have a width of 64-bit */
4854 typedef long long xxh_i64;
4855#endif
4856
4857
4858/*
4859 * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
4860 *
4861 * It is a hardened version of UMAC, based off of FARSH's implementation.
4862 *
4863 * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
4864 * implementations, and it is ridiculously fast.
4865 *
4866 * We harden it by mixing the original input to the accumulators as well as the product.
4867 *
4868 * This means that in the (relatively likely) case of a multiply by zero, the
4869 * original input is preserved.
4870 *
4871 * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
4872 * cross-pollination, as otherwise the upper and lower halves would be
4873 * essentially independent.
4874 *
4875 * This doesn't matter on 64-bit hashes since they all get merged together in
4876 * the end, so we skip the extra step.
4877 *
4878 * Both XXH3_64bits and XXH3_128bits use this subroutine.
4879 */
4880
4881#if (XXH_VECTOR == XXH_AVX512) \
4882 || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)
4883
4884#ifndef XXH_TARGET_AVX512
4885# define XXH_TARGET_AVX512 /* disable attribute target */
4886#endif
4887
4888XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4889XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
4890 const void* XXH_RESTRICT input,
4891 const void* XXH_RESTRICT secret)
4892{
4893 __m512i* const xacc = (__m512i *) acc;
4894 XXH_ASSERT((((size_t)acc) & 63) == 0);
4895 XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
4896
4897 {
4898 /* data_vec = input[0]; */
4899 __m512i const data_vec = _mm512_loadu_si512 (input);
4900 /* key_vec = secret[0]; */
4901 __m512i const key_vec = _mm512_loadu_si512 (secret);
4902 /* data_key = data_vec ^ key_vec; */
4903 __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
4904 /* data_key_lo = data_key >> 32; */
4905 __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32);
4906 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
4907 __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo);
4908 /* xacc[0] += swap(data_vec); */
4909 __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
4910 __m512i const sum = _mm512_add_epi64(*xacc, data_swap);
4911 /* xacc[0] += product; */
4912 *xacc = _mm512_add_epi64(product, sum);
4913 }
4914}
4915XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512)
4916
4917/*
4918 * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
4919 *
4920 * Multiplication isn't perfect, as explained by Google in HighwayHash:
4921 *
4922 * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
4923 * // varying degrees. In descending order of goodness, bytes
4924 * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
4925 * // As expected, the upper and lower bytes are much worse.
4926 *
4927 * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
4928 *
4929 * Since our algorithm uses a pseudorandom secret to add some variance into the
4930 * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
4931 *
4932 * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
4933 * extraction.
4934 *
4935 * Both XXH3_64bits and XXH3_128bits use this subroutine.
4936 */
4937
4938XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4939XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
4940{
4941 XXH_ASSERT((((size_t)acc) & 63) == 0);
4942 XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
4943 { __m512i* const xacc = (__m512i*) acc;
4944 const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
4945
4946 /* xacc[0] ^= (xacc[0] >> 47) */
4947 __m512i const acc_vec = *xacc;
4948 __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47);
4949 /* xacc[0] ^= secret; */
4950 __m512i const key_vec = _mm512_loadu_si512 (secret);
4951 __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */);
4952
4953 /* xacc[0] *= XXH_PRIME32_1; */
4954 __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32);
4955 __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32);
4956 __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32);
4957 *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
4958 }
4959}
4960
4961XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4962XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
4963{
4964 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
4965 XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
4966 XXH_ASSERT(((size_t)customSecret & 63) == 0);
4967 (void)(&XXH_writeLE64);
4968 { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
4969 __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64);
4970 __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos);
4971
4972 const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret);
4973 __m512i* const dest = ( __m512i*) customSecret;
4974 int i;
4975 XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
4976 XXH_ASSERT(((size_t)dest & 63) == 0);
4977 for (i=0; i < nbRounds; ++i) {
4978 dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed);
4979 } }
4980}
4981
4982#endif
4983
4984#if (XXH_VECTOR == XXH_AVX2) \
4985 || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)
4986
4987#ifndef XXH_TARGET_AVX2
4988# define XXH_TARGET_AVX2 /* disable attribute target */
4989#endif
4990
4991XXH_FORCE_INLINE XXH_TARGET_AVX2 void
4992XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
4993 const void* XXH_RESTRICT input,
4994 const void* XXH_RESTRICT secret)
4995{
4996 XXH_ASSERT((((size_t)acc) & 31) == 0);
4997 { __m256i* const xacc = (__m256i *) acc;
4998 /* Unaligned. This is mainly for pointer arithmetic, and because
4999 * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
5000 const __m256i* const xinput = (const __m256i *) input;
5001 /* Unaligned. This is mainly for pointer arithmetic, and because
5002 * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
5003 const __m256i* const xsecret = (const __m256i *) secret;
5004
5005 size_t i;
5006 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
5007 /* data_vec = xinput[i]; */
5008 __m256i const data_vec = _mm256_loadu_si256 (xinput+i);
5009 /* key_vec = xsecret[i]; */
5010 __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
5011 /* data_key = data_vec ^ key_vec; */
5012 __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
5013 /* data_key_lo = data_key >> 32; */
5014 __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32);
5015 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
5016 __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo);
5017 /* xacc[i] += swap(data_vec); */
5018 __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
5019 __m256i const sum = _mm256_add_epi64(xacc[i], data_swap);
5020 /* xacc[i] += product; */
5021 xacc[i] = _mm256_add_epi64(product, sum);
5022 } }
5023}
5024XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2)
5025
5026XXH_FORCE_INLINE XXH_TARGET_AVX2 void
5027XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5028{
5029 XXH_ASSERT((((size_t)acc) & 31) == 0);
5030 { __m256i* const xacc = (__m256i*) acc;
5031 /* Unaligned. This is mainly for pointer arithmetic, and because
5032 * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
5033 const __m256i* const xsecret = (const __m256i *) secret;
5034 const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
5035
5036 size_t i;
5037 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
5038 /* xacc[i] ^= (xacc[i] >> 47) */
5039 __m256i const acc_vec = xacc[i];
5040 __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47);
5041 __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted);
5042 /* xacc[i] ^= xsecret; */
5043 __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
5044 __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
5045
5046 /* xacc[i] *= XXH_PRIME32_1; */
5047 __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32);
5048 __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32);
5049 __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32);
5050 xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
5051 }
5052 }
5053}
5054
5055XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
5056{
5057 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
5058 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
5059 XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
5060 (void)(&XXH_writeLE64);
5061 XXH_PREFETCH(customSecret);
5062 { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);
5063
5064 const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret);
5065 __m256i* dest = ( __m256i*) customSecret;
5066
5067# if defined(__GNUC__) || defined(__clang__)
5068 /*
5069 * On GCC & Clang, marking 'dest' as modified will cause the compiler:
5070 * - do not extract the secret from sse registers in the internal loop
5071 * - use less common registers, and avoid pushing these reg into stack
5072 */
5073 XXH_COMPILER_GUARD(dest);
5074# endif
5075 XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
5076 XXH_ASSERT(((size_t)dest & 31) == 0);
5077
5078 /* GCC -O2 need unroll loop manually */
5079 dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed);
5080 dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed);
5081 dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed);
5082 dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed);
5083 dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed);
5084 dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed);
5085 }
5086}
5087
5088#endif
5089
5090/* x86dispatch always generates SSE2 */
5091#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)
5092
5093#ifndef XXH_TARGET_SSE2
5094# define XXH_TARGET_SSE2 /* disable attribute target */
5095#endif
5096
5097XXH_FORCE_INLINE XXH_TARGET_SSE2 void
5098XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
5099 const void* XXH_RESTRICT input,
5100 const void* XXH_RESTRICT secret)
5101{
5102 /* SSE2 is just a half-scale version of the AVX2 version. */
5103 XXH_ASSERT((((size_t)acc) & 15) == 0);
5104 { __m128i* const xacc = (__m128i *) acc;
5105 /* Unaligned. This is mainly for pointer arithmetic, and because
5106 * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
5107 const __m128i* const xinput = (const __m128i *) input;
5108 /* Unaligned. This is mainly for pointer arithmetic, and because
5109 * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
5110 const __m128i* const xsecret = (const __m128i *) secret;
5111
5112 size_t i;
5113 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
5114 /* data_vec = xinput[i]; */
5115 __m128i const data_vec = _mm_loadu_si128 (xinput+i);
5116 /* key_vec = xsecret[i]; */
5117 __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
5118 /* data_key = data_vec ^ key_vec; */
5119 __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
5120 /* data_key_lo = data_key >> 32; */
5121 __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
5122 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
5123 __m128i const product = _mm_mul_epu32 (data_key, data_key_lo);
5124 /* xacc[i] += swap(data_vec); */
5125 __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
5126 __m128i const sum = _mm_add_epi64(xacc[i], data_swap);
5127 /* xacc[i] += product; */
5128 xacc[i] = _mm_add_epi64(product, sum);
5129 } }
5130}
5131XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2)
5132
5133XXH_FORCE_INLINE XXH_TARGET_SSE2 void
5134XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5135{
5136 XXH_ASSERT((((size_t)acc) & 15) == 0);
5137 { __m128i* const xacc = (__m128i*) acc;
5138 /* Unaligned. This is mainly for pointer arithmetic, and because
5139 * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
5140 const __m128i* const xsecret = (const __m128i *) secret;
5141 const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
5142
5143 size_t i;
5144 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
5145 /* xacc[i] ^= (xacc[i] >> 47) */
5146 __m128i const acc_vec = xacc[i];
5147 __m128i const shifted = _mm_srli_epi64 (acc_vec, 47);
5148 __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted);
5149 /* xacc[i] ^= xsecret[i]; */
5150 __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
5151 __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
5152
5153 /* xacc[i] *= XXH_PRIME32_1; */
5154 __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
5155 __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32);
5156 __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32);
5157 xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
5158 }
5159 }
5160}
5161
5162XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
5163{
5164 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
5165 (void)(&XXH_writeLE64);
5166 { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
5167
5168# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
5169 /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */
5170 XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) };
5171 __m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
5172# else
5173 __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
5174# endif
5175 int i;
5176
5177 const void* const src16 = XXH3_kSecret;
5178 __m128i* dst16 = (__m128i*) customSecret;
5179# if defined(__GNUC__) || defined(__clang__)
5180 /*
5181 * On GCC & Clang, marking 'dest' as modified will cause the compiler:
5182 * - do not extract the secret from sse registers in the internal loop
5183 * - use less common registers, and avoid pushing these reg into stack
5184 */
5185 XXH_COMPILER_GUARD(dst16);
5186# endif
5187 XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
5188 XXH_ASSERT(((size_t)dst16 & 15) == 0);
5189
5190 for (i=0; i < nbRounds; ++i) {
5191 dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
5192 } }
5193}
5194
5195#endif
5196
5197#if (XXH_VECTOR == XXH_NEON)
5198
5199/* forward declarations for the scalar routines */
5200XXH_FORCE_INLINE void
5201XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input,
5202 void const* XXH_RESTRICT secret, size_t lane);
5203
5204XXH_FORCE_INLINE void
5205XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
5206 void const* XXH_RESTRICT secret, size_t lane);
5207
5208/*!
5209 * @internal
5210 * @brief The bulk processing loop for NEON and WASM SIMD128.
5211 *
5212 * The NEON code path is actually partially scalar when running on AArch64. This
5213 * is to optimize the pipelining and can have up to 15% speedup depending on the
5214 * CPU, and it also mitigates some GCC codegen issues.
5215 *
5216 * @see XXH3_NEON_LANES for configuring this and details about this optimization.
5217 *
5218 * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit
5219 * integers instead of the other platforms which mask full 64-bit vectors,
5220 * so the setup is more complicated than just shifting right.
5221 *
5222 * Additionally, there is an optimization for 4 lanes at once noted below.
5223 *
5224 * Since, as stated, the most optimal amount of lanes for Cortexes is 6,
5225 * there needs to be *three* versions of the accumulate operation used
5226 * for the remaining 2 lanes.
5227 *
5228 * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap
5229 * nearly perfectly.
5230 */
5231
5232XXH_FORCE_INLINE void
5233XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
5234 const void* XXH_RESTRICT input,
5235 const void* XXH_RESTRICT secret)
5236{
5237 XXH_ASSERT((((size_t)acc) & 15) == 0);
5238 XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0);
5239 { /* GCC for darwin arm64 does not like aliasing here */
5240 xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc;
5241 /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
5242 uint8_t const* xinput = (const uint8_t *) input;
5243 uint8_t const* xsecret = (const uint8_t *) secret;
5244
5245 size_t i;
5246#ifdef __wasm_simd128__
5247 /*
5248 * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret
5249 * is constant propagated, which results in it converting it to this
5250 * inside the loop:
5251 *
5252 * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0)
5253 * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0)
5254 * ...
5255 *
5256 * This requires a full 32-bit address immediate (and therefore a 6 byte
5257 * instruction) as well as an add for each offset.
5258 *
5259 * Putting an asm guard prevents it from folding (at the cost of losing
5260 * the alignment hint), and uses the free offset in `v128.load` instead
5261 * of adding secret_offset each time which overall reduces code size by
5262 * about a kilobyte and improves performance.
5263 */
5264 XXH_COMPILER_GUARD(xsecret);
5265#endif
5266 /* Scalar lanes use the normal scalarRound routine */
5267 for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
5268 XXH3_scalarRound(acc, input, secret, i);
5269 }
5270 i = 0;
5271 /* 4 NEON lanes at a time. */
5272 for (; i+1 < XXH3_NEON_LANES / 2; i+=2) {
5273 /* data_vec = xinput[i]; */
5274 uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16));
5275 uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16));
5276 /* key_vec = xsecret[i]; */
5277 uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16));
5278 uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16));
5279 /* data_swap = swap(data_vec) */
5280 uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1);
5281 uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1);
5282 /* data_key = data_vec ^ key_vec; */
5283 uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1);
5284 uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2);
5285
5286 /*
5287 * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a
5288 * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to
5289 * get one vector with the low 32 bits of each lane, and one vector
5290 * with the high 32 bits of each lane.
5291 *
5292 * The intrinsic returns a double vector because the original ARMv7-a
5293 * instruction modified both arguments in place. AArch64 and SIMD128 emit
5294 * two instructions from this intrinsic.
5295 *
5296 * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ]
5297 * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ]
5298 */
5299 uint32x4x2_t unzipped = vuzpq_u32(
5300 vreinterpretq_u32_u64(data_key_1),
5301 vreinterpretq_u32_u64(data_key_2)
5302 );
5303 /* data_key_lo = data_key & 0xFFFFFFFF */
5304 uint32x4_t data_key_lo = unzipped.val[0];
5305 /* data_key_hi = data_key >> 32 */
5306 uint32x4_t data_key_hi = unzipped.val[1];
5307 /*
5308 * Then, we can split the vectors horizontally and multiply which, as for most
5309 * widening intrinsics, have a variant that works on both high half vectors
5310 * for free on AArch64. A similar instruction is available on SIMD128.
5311 *
5312 * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi
5313 */
5314 uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi);
5315 uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi);
5316 /*
5317 * Clang reorders
5318 * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s
5319 * c += a; // add acc.2d, acc.2d, swap.2d
5320 * to
5321 * c += a; // add acc.2d, acc.2d, swap.2d
5322 * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s
5323 *
5324 * While it would make sense in theory since the addition is faster,
5325 * for reasons likely related to umlal being limited to certain NEON
5326 * pipelines, this is worse. A compiler guard fixes this.
5327 */
5328 XXH_COMPILER_GUARD_CLANG_NEON(sum_1);
5329 XXH_COMPILER_GUARD_CLANG_NEON(sum_2);
5330 /* xacc[i] = acc_vec + sum; */
5331 xacc[i] = vaddq_u64(xacc[i], sum_1);
5332 xacc[i+1] = vaddq_u64(xacc[i+1], sum_2);
5333 }
5334 /* Operate on the remaining NEON lanes 2 at a time. */
5335 for (; i < XXH3_NEON_LANES / 2; i++) {
5336 /* data_vec = xinput[i]; */
5337 uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16));
5338 /* key_vec = xsecret[i]; */
5339 uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16));
5340 /* acc_vec_2 = swap(data_vec) */
5341 uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1);
5342 /* data_key = data_vec ^ key_vec; */
5343 uint64x2_t data_key = veorq_u64(data_vec, key_vec);
5344 /* For two lanes, just use VMOVN and VSHRN. */
5345 /* data_key_lo = data_key & 0xFFFFFFFF; */
5346 uint32x2_t data_key_lo = vmovn_u64(data_key);
5347 /* data_key_hi = data_key >> 32; */
5348 uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32);
5349 /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */
5350 uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi);
5351 /* Same Clang workaround as before */
5352 XXH_COMPILER_GUARD_CLANG_NEON(sum);
5353 /* xacc[i] = acc_vec + sum; */
5354 xacc[i] = vaddq_u64 (xacc[i], sum);
5355 }
5356 }
5357}
5358XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon)
5359
5360XXH_FORCE_INLINE void
5361XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5362{
5363 XXH_ASSERT((((size_t)acc) & 15) == 0);
5364
5365 { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc;
5366 uint8_t const* xsecret = (uint8_t const*) secret;
5367
5368 size_t i;
5369 /* WASM uses operator overloads and doesn't need these. */
5370#ifndef __wasm_simd128__
5371 /* { prime32_1, prime32_1 } */
5372 uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1);
5373 /* { 0, prime32_1, 0, prime32_1 } */
5374 uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32));
5375#endif
5376
5377 /* AArch64 uses both scalar and neon at the same time */
5378 for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
5379 XXH3_scalarScrambleRound(acc, secret, i);
5380 }
5381 for (i=0; i < XXH3_NEON_LANES / 2; i++) {
5382 /* xacc[i] ^= (xacc[i] >> 47); */
5383 uint64x2_t acc_vec = xacc[i];
5384 uint64x2_t shifted = vshrq_n_u64(acc_vec, 47);
5385 uint64x2_t data_vec = veorq_u64(acc_vec, shifted);
5386
5387 /* xacc[i] ^= xsecret[i]; */
5388 uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16));
5389 uint64x2_t data_key = veorq_u64(data_vec, key_vec);
5390 /* xacc[i] *= XXH_PRIME32_1 */
5391#ifdef __wasm_simd128__
5392 /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */
5393 xacc[i] = data_key * XXH_PRIME32_1;
5394#else
5395 /*
5396 * Expanded version with portable NEON intrinsics
5397 *
5398 * lo(x) * lo(y) + (hi(x) * lo(y) << 32)
5399 *
5400 * prod_hi = hi(data_key) * lo(prime) << 32
5401 *
5402 * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector
5403 * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits
5404 * and avoid the shift.
5405 */
5406 uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi);
5407 /* Extract low bits for vmlal_u32 */
5408 uint32x2_t data_key_lo = vmovn_u64(data_key);
5409 /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */
5410 xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo);
5411#endif
5412 }
5413 }
5414}
5415#endif
5416
5417#if (XXH_VECTOR == XXH_VSX)
5418
5419XXH_FORCE_INLINE void
5420XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc,
5421 const void* XXH_RESTRICT input,
5422 const void* XXH_RESTRICT secret)
5423{
5424 /* presumed aligned */
5425 xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
5426 xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */
5427 xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */
5428 xxh_u64x2 const v32 = { 32, 32 };
5429 size_t i;
5430 for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
5431 /* data_vec = xinput[i]; */
5432 xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i);
5433 /* key_vec = xsecret[i]; */
5434 xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i);
5435 xxh_u64x2 const data_key = data_vec ^ key_vec;
5436 /* shuffled = (data_key << 32) | (data_key >> 32); */
5437 xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
5438 /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
5439 xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
5440 /* acc_vec = xacc[i]; */
5441 xxh_u64x2 acc_vec = xacc[i];
5442 acc_vec += product;
5443
5444 /* swap high and low halves */
5445#ifdef __s390x__
5446 acc_vec += vec_permi(data_vec, data_vec, 2);
5447#else
5448 acc_vec += vec_xxpermdi(data_vec, data_vec, 2);
5449#endif
5450 xacc[i] = acc_vec;
5451 }
5452}
5453XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx)
5454
5455XXH_FORCE_INLINE void
5456XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5457{
5458 XXH_ASSERT((((size_t)acc) & 15) == 0);
5459
5460 { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
5461 const xxh_u8* const xsecret = (const xxh_u8*) secret;
5462 /* constants */
5463 xxh_u64x2 const v32 = { 32, 32 };
5464 xxh_u64x2 const v47 = { 47, 47 };
5465 xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
5466 size_t i;
5467 for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
5468 /* xacc[i] ^= (xacc[i] >> 47); */
5469 xxh_u64x2 const acc_vec = xacc[i];
5470 xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
5471
5472 /* xacc[i] ^= xsecret[i]; */
5473 xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i);
5474 xxh_u64x2 const data_key = data_vec ^ key_vec;
5475
5476 /* xacc[i] *= XXH_PRIME32_1 */
5477 /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */
5478 xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime);
5479 /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */
5480 xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime);
5481 xacc[i] = prod_odd + (prod_even << v32);
5482 } }
5483}
5484
5485#endif
5486
5487#if (XXH_VECTOR == XXH_SVE)
5488
5489XXH_FORCE_INLINE void
5490XXH3_accumulate_512_sve( void* XXH_RESTRICT acc,
5491 const void* XXH_RESTRICT input,
5492 const void* XXH_RESTRICT secret)
5493{
5494 uint64_t *xacc = (uint64_t *)acc;
5495 const uint64_t *xinput = (const uint64_t *)(const void *)input;
5496 const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
5497 svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
5498 uint64_t element_count = svcntd();
5499 if (element_count >= 8) {
5500 svbool_t mask = svptrue_pat_b64(SV_VL8);
5501 svuint64_t vacc = svld1_u64(mask, xacc);
5502 ACCRND(vacc, 0);
5503 svst1_u64(mask, xacc, vacc);
5504 } else if (element_count == 2) { /* sve128 */
5505 svbool_t mask = svptrue_pat_b64(SV_VL2);
5506 svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5507 svuint64_t acc1 = svld1_u64(mask, xacc + 2);
5508 svuint64_t acc2 = svld1_u64(mask, xacc + 4);
5509 svuint64_t acc3 = svld1_u64(mask, xacc + 6);
5510 ACCRND(acc0, 0);
5511 ACCRND(acc1, 2);
5512 ACCRND(acc2, 4);
5513 ACCRND(acc3, 6);
5514 svst1_u64(mask, xacc + 0, acc0);
5515 svst1_u64(mask, xacc + 2, acc1);
5516 svst1_u64(mask, xacc + 4, acc2);
5517 svst1_u64(mask, xacc + 6, acc3);
5518 } else {
5519 svbool_t mask = svptrue_pat_b64(SV_VL4);
5520 svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5521 svuint64_t acc1 = svld1_u64(mask, xacc + 4);
5522 ACCRND(acc0, 0);
5523 ACCRND(acc1, 4);
5524 svst1_u64(mask, xacc + 0, acc0);
5525 svst1_u64(mask, xacc + 4, acc1);
5526 }
5527}
5528
5529XXH_FORCE_INLINE void
5530XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc,
5531 const xxh_u8* XXH_RESTRICT input,
5532 const xxh_u8* XXH_RESTRICT secret,
5533 size_t nbStripes)
5534{
5535 if (nbStripes != 0) {
5536 uint64_t *xacc = (uint64_t *)acc;
5537 const uint64_t *xinput = (const uint64_t *)(const void *)input;
5538 const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
5539 svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
5540 uint64_t element_count = svcntd();
5541 if (element_count >= 8) {
5542 svbool_t mask = svptrue_pat_b64(SV_VL8);
5543 svuint64_t vacc = svld1_u64(mask, xacc + 0);
5544 do {
5545 /* svprfd(svbool_t, void *, enum svfprop); */
5546 svprfd(mask, xinput + 128, SV_PLDL1STRM);
5547 ACCRND(vacc, 0);
5548 xinput += 8;
5549 xsecret += 1;
5550 nbStripes--;
5551 } while (nbStripes != 0);
5552
5553 svst1_u64(mask, xacc + 0, vacc);
5554 } else if (element_count == 2) { /* sve128 */
5555 svbool_t mask = svptrue_pat_b64(SV_VL2);
5556 svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5557 svuint64_t acc1 = svld1_u64(mask, xacc + 2);
5558 svuint64_t acc2 = svld1_u64(mask, xacc + 4);
5559 svuint64_t acc3 = svld1_u64(mask, xacc + 6);
5560 do {
5561 svprfd(mask, xinput + 128, SV_PLDL1STRM);
5562 ACCRND(acc0, 0);
5563 ACCRND(acc1, 2);
5564 ACCRND(acc2, 4);
5565 ACCRND(acc3, 6);
5566 xinput += 8;
5567 xsecret += 1;
5568 nbStripes--;
5569 } while (nbStripes != 0);
5570
5571 svst1_u64(mask, xacc + 0, acc0);
5572 svst1_u64(mask, xacc + 2, acc1);
5573 svst1_u64(mask, xacc + 4, acc2);
5574 svst1_u64(mask, xacc + 6, acc3);
5575 } else {
5576 svbool_t mask = svptrue_pat_b64(SV_VL4);
5577 svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5578 svuint64_t acc1 = svld1_u64(mask, xacc + 4);
5579 do {
5580 svprfd(mask, xinput + 128, SV_PLDL1STRM);
5581 ACCRND(acc0, 0);
5582 ACCRND(acc1, 4);
5583 xinput += 8;
5584 xsecret += 1;
5585 nbStripes--;
5586 } while (nbStripes != 0);
5587
5588 svst1_u64(mask, xacc + 0, acc0);
5589 svst1_u64(mask, xacc + 4, acc1);
5590 }
5591 }
5592}
5593
5594#endif
5595
5596/* scalar variants - universal */
5597
5598#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__))
5599/*
5600 * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they
5601 * emit an excess mask and a full 64-bit multiply-add (MADD X-form).
5602 *
5603 * While this might not seem like much, as AArch64 is a 64-bit architecture, only
5604 * big Cortex designs have a full 64-bit multiplier.
5605 *
5606 * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit
5607 * multiplies expand to 2-3 multiplies in microcode. This has a major penalty
5608 * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline.
5609 *
5610 * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does
5611 * not have this penalty and does the mask automatically.
5612 */
5613XXH_FORCE_INLINE xxh_u64
5614XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
5615{
5616 xxh_u64 ret;
5617 /* note: %x = 64-bit register, %w = 32-bit register */
5618 __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc));
5619 return ret;
5620}
5621#else
5622XXH_FORCE_INLINE xxh_u64
5623XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
5624{
5625 return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc;
5626}
5627#endif
5628
5629/*!
5630 * @internal
5631 * @brief Scalar round for @ref XXH3_accumulate_512_scalar().
5632 *
5633 * This is extracted to its own function because the NEON path uses a combination
5634 * of NEON and scalar.
5635 */
5636XXH_FORCE_INLINE void
5637XXH3_scalarRound(void* XXH_RESTRICT acc,
5638 void const* XXH_RESTRICT input,
5639 void const* XXH_RESTRICT secret,
5640 size_t lane)
5641{
5642 xxh_u64* xacc = (xxh_u64*) acc;
5643 xxh_u8 const* xinput = (xxh_u8 const*) input;
5644 xxh_u8 const* xsecret = (xxh_u8 const*) secret;
5645 XXH_ASSERT(lane < XXH_ACC_NB);
5646 XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
5647 {
5648 xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8);
5649 xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8);
5650 xacc[lane ^ 1] += data_val; /* swap adjacent lanes */
5651 xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]);
5652 }
5653}
5654
5655/*!
5656 * @internal
5657 * @brief Processes a 64 byte block of data using the scalar path.
5658 */
5659XXH_FORCE_INLINE void
5660XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
5661 const void* XXH_RESTRICT input,
5662 const void* XXH_RESTRICT secret)
5663{
5664 size_t i;
5665 /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */
5666#if defined(__GNUC__) && !defined(__clang__) \
5667 && (defined(__arm__) || defined(__thumb2__)) \
5668 && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \
5669 && XXH_SIZE_OPT <= 0
5670# pragma GCC unroll 8
5671#endif
5672 for (i=0; i < XXH_ACC_NB; i++) {
5673 XXH3_scalarRound(acc, input, secret, i);
5674 }
5675}
5676XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar)
5677
5678/*!
5679 * @internal
5680 * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar().
5681 *
5682 * This is extracted to its own function because the NEON path uses a combination
5683 * of NEON and scalar.
5684 */
5685XXH_FORCE_INLINE void
5686XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
5687 void const* XXH_RESTRICT secret,
5688 size_t lane)
5689{
5690 xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */
5691 const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */
5692 XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
5693 XXH_ASSERT(lane < XXH_ACC_NB);
5694 {
5695 xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8);
5696 xxh_u64 acc64 = xacc[lane];
5697 acc64 = XXH_xorshift64(acc64, 47);
5698 acc64 ^= key64;
5699 acc64 *= XXH_PRIME32_1;
5700 xacc[lane] = acc64;
5701 }
5702}
5703
5704/*!
5705 * @internal
5706 * @brief Scrambles the accumulators after a large chunk has been read
5707 */
5708XXH_FORCE_INLINE void
5709XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5710{
5711 size_t i;
5712 for (i=0; i < XXH_ACC_NB; i++) {
5713 XXH3_scalarScrambleRound(acc, secret, i);
5714 }
5715}
5716
5717XXH_FORCE_INLINE void
5718XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
5719{
5720 /*
5721 * We need a separate pointer for the hack below,
5722 * which requires a non-const pointer.
5723 * Any decent compiler will optimize this out otherwise.
5724 */
5725 const xxh_u8* kSecretPtr = XXH3_kSecret;
5726 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
5727
5728#if defined(__GNUC__) && defined(__aarch64__)
5729 /*
5730 * UGLY HACK:
5731 * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are
5732 * placed sequentially, in order, at the top of the unrolled loop.
5733 *
5734 * While MOVK is great for generating constants (2 cycles for a 64-bit
5735 * constant compared to 4 cycles for LDR), it fights for bandwidth with
5736 * the arithmetic instructions.
5737 *
5738 * I L S
5739 * MOVK
5740 * MOVK
5741 * MOVK
5742 * MOVK
5743 * ADD
5744 * SUB STR
5745 * STR
5746 * By forcing loads from memory (as the asm line causes the compiler to assume
5747 * that XXH3_kSecretPtr has been changed), the pipelines are used more
5748 * efficiently:
5749 * I L S
5750 * LDR
5751 * ADD LDR
5752 * SUB STR
5753 * STR
5754 *
5755 * See XXH3_NEON_LANES for details on the pipsline.
5756 *
5757 * XXH3_64bits_withSeed, len == 256, Snapdragon 835
5758 * without hack: 2654.4 MB/s
5759 * with hack: 3202.9 MB/s
5760 */
5761 XXH_COMPILER_GUARD(kSecretPtr);
5762#endif
5763 { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
5764 int i;
5765 for (i=0; i < nbRounds; i++) {
5766 /*
5767 * The asm hack causes the compiler to assume that kSecretPtr aliases with
5768 * customSecret, and on aarch64, this prevented LDP from merging two
5769 * loads together for free. Putting the loads together before the stores
5770 * properly generates LDP.
5771 */
5772 xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64;
5773 xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
5774 XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo);
5775 XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi);
5776 } }
5777}
5778
5779
5780typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t);
5781typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*);
5782typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64);
5783
5784
5785#if (XXH_VECTOR == XXH_AVX512)
5786
5787#define XXH3_accumulate_512 XXH3_accumulate_512_avx512
5788#define XXH3_accumulate XXH3_accumulate_avx512
5789#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512
5790#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512
5791
5792#elif (XXH_VECTOR == XXH_AVX2)
5793
5794#define XXH3_accumulate_512 XXH3_accumulate_512_avx2
5795#define XXH3_accumulate XXH3_accumulate_avx2
5796#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2
5797#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2
5798
5799#elif (XXH_VECTOR == XXH_SSE2)
5800
5801#define XXH3_accumulate_512 XXH3_accumulate_512_sse2
5802#define XXH3_accumulate XXH3_accumulate_sse2
5803#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2
5804#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2
5805
5806#elif (XXH_VECTOR == XXH_NEON)
5807
5808#define XXH3_accumulate_512 XXH3_accumulate_512_neon
5809#define XXH3_accumulate XXH3_accumulate_neon
5810#define XXH3_scrambleAcc XXH3_scrambleAcc_neon
5811#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5812
5813#elif (XXH_VECTOR == XXH_VSX)
5814
5815#define XXH3_accumulate_512 XXH3_accumulate_512_vsx
5816#define XXH3_accumulate XXH3_accumulate_vsx
5817#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx
5818#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5819
5820#elif (XXH_VECTOR == XXH_SVE)
5821#define XXH3_accumulate_512 XXH3_accumulate_512_sve
5822#define XXH3_accumulate XXH3_accumulate_sve
5823#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
5824#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5825
5826#else /* scalar */
5827
5828#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
5829#define XXH3_accumulate XXH3_accumulate_scalar
5830#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
5831#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5832
5833#endif
5834
5835#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */
5836# undef XXH3_initCustomSecret
5837# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5838#endif
5839
5840XXH_FORCE_INLINE void
5841XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
5842 const xxh_u8* XXH_RESTRICT input, size_t len,
5843 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
5844 XXH3_f_accumulate f_acc,
5845 XXH3_f_scrambleAcc f_scramble)
5846{
5847 size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
5848 size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
5849 size_t const nb_blocks = (len - 1) / block_len;
5850
5851 size_t n;
5852
5853 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
5854
5855 for (n = 0; n < nb_blocks; n++) {
5856 f_acc(acc, input + n*block_len, secret, nbStripesPerBlock);
5857 f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
5858 }
5859
5860 /* last partial block */
5861 XXH_ASSERT(len > XXH_STRIPE_LEN);
5862 { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
5863 XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
5864 f_acc(acc, input + nb_blocks*block_len, secret, nbStripes);
5865
5866 /* last stripe */
5867 { const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
5868#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */
5869 XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
5870 } }
5871}
5872
5873XXH_FORCE_INLINE xxh_u64
5874XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
5875{
5876 return XXH3_mul128_fold64(
5877 acc[0] ^ XXH_readLE64(secret),
5878 acc[1] ^ XXH_readLE64(secret+8) );
5879}
5880
5881static XXH64_hash_t
5882XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
5883{
5884 xxh_u64 result64 = start;
5885 size_t i = 0;
5886
5887 for (i = 0; i < 4; i++) {
5888 result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i);
5889#if defined(__clang__) /* Clang */ \
5890 && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \
5891 && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
5892 && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
5893 /*
5894 * UGLY HACK:
5895 * Prevent autovectorization on Clang ARMv7-a. Exact same problem as
5896 * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
5897 * XXH3_64bits, len == 256, Snapdragon 835:
5898 * without hack: 2063.7 MB/s
5899 * with hack: 2560.7 MB/s
5900 */
5901 XXH_COMPILER_GUARD(result64);
5902#endif
5903 }
5904
5905 return XXH3_avalanche(result64);
5906}
5907
5908#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
5909 XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
5910
5911XXH_FORCE_INLINE XXH64_hash_t
5912XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
5913 const void* XXH_RESTRICT secret, size_t secretSize,
5914 XXH3_f_accumulate f_acc,
5915 XXH3_f_scrambleAcc f_scramble)
5916{
5917 XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
5918
5919 XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble);
5920
5921 /* converge into final hash */
5922 XXH_STATIC_ASSERT(sizeof(acc) == 64);
5923 /* do not align on 8, so that the secret is different from the accumulator */
5924#define XXH_SECRET_MERGEACCS_START 11
5925 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
5926 return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1);
5927}
5928
5929/*
5930 * It's important for performance to transmit secret's size (when it's static)
5931 * so that the compiler can properly optimize the vectorized loop.
5932 * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
5933 * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
5934 * breaks -Og, this is XXH_NO_INLINE.
5935 */
5936XXH3_WITH_SECRET_INLINE XXH64_hash_t
5937XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
5938 XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
5939{
5940 (void)seed64;
5941 return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc);
5942}
5943
5944/*
5945 * It's preferable for performance that XXH3_hashLong is not inlined,
5946 * as it results in a smaller function for small data, easier to the instruction cache.
5947 * Note that inside this no_inline function, we do inline the internal loop,
5948 * and provide a statically defined secret size to allow optimization of vector loop.
5949 */
5950XXH_NO_INLINE XXH_PUREF XXH64_hash_t
5951XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
5952 XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
5953{
5954 (void)seed64; (void)secret; (void)secretLen;
5955 return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc);
5956}
5957
5958/*
5959 * XXH3_hashLong_64b_withSeed():
5960 * Generate a custom key based on alteration of default XXH3_kSecret with the seed,
5961 * and then use this key for long mode hashing.
5962 *
5963 * This operation is decently fast but nonetheless costs a little bit of time.
5964 * Try to avoid it whenever possible (typically when seed==0).
5965 *
5966 * It's important for performance that XXH3_hashLong is not inlined. Not sure
5967 * why (uop cache maybe?), but the difference is large and easily measurable.
5968 */
5969XXH_FORCE_INLINE XXH64_hash_t
5970XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
5971 XXH64_hash_t seed,
5972 XXH3_f_accumulate f_acc,
5973 XXH3_f_scrambleAcc f_scramble,
5974 XXH3_f_initCustomSecret f_initSec)
5975{
5976#if XXH_SIZE_OPT <= 0
5977 if (seed == 0)
5978 return XXH3_hashLong_64b_internal(input, len,
5979 XXH3_kSecret, sizeof(XXH3_kSecret),
5980 f_acc, f_scramble);
5981#endif
5982 { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
5983 f_initSec(secret, seed);
5984 return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
5985 f_acc, f_scramble);
5986 }
5987}
5988
5989/*
5990 * It's important for performance that XXH3_hashLong is not inlined.
5991 */
5992XXH_NO_INLINE XXH64_hash_t
5993XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len,
5994 XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
5995{
5996 (void)secret; (void)secretLen;
5997 return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
5998 XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
5999}
6000
6001
6002typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t,
6003 XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);
6004
6005XXH_FORCE_INLINE XXH64_hash_t
6006XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
6007 XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
6008 XXH3_hashLong64_f f_hashLong)
6009{
6010 XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
6011 /*
6012 * If an action is to be taken if `secretLen` condition is not respected,
6013 * it should be done here.
6014 * For now, it's a contract pre-condition.
6015 * Adding a check and a branch here would cost performance at every hash.
6016 * Also, note that function signature doesn't offer room to return an error.
6017 */
6018 if (len <= 16)
6019 return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
6020 if (len <= 128)
6021 return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6022 if (len <= XXH3_MIDSIZE_MAX)
6023 return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6024 return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
6025}
6026
6027
6028/* === Public entry point === */
6029
6030/*! @ingroup XXH3_family */
6031XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length)
6032{
6033 return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
6034}
6035
6036/*! @ingroup XXH3_family */
6037XXH_PUBLIC_API XXH64_hash_t
6038XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize)
6039{
6040 return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
6041}
6042
6043/*! @ingroup XXH3_family */
6044XXH_PUBLIC_API XXH64_hash_t
6045XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed)
6046{
6047 return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
6048}
6049
6050XXH_PUBLIC_API XXH64_hash_t
6051XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
6052{
6053 if (length <= XXH3_MIDSIZE_MAX)
6054 return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
6055 return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize);
6056}
6057
6058
6059/* === XXH3 streaming === */
6060#ifndef XXH_NO_STREAM
6061/*
6062 * Malloc's a pointer that is always aligned to align.
6063 *
6064 * This must be freed with `XXH_alignedFree()`.
6065 *
6066 * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
6067 * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
6068 * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
6069 *
6070 * This underalignment previously caused a rather obvious crash which went
6071 * completely unnoticed due to XXH3_createState() not actually being tested.
6072 * Credit to RedSpah for noticing this bug.
6073 *
6074 * The alignment is done manually: Functions like posix_memalign or _mm_malloc
6075 * are avoided: To maintain portability, we would have to write a fallback
6076 * like this anyways, and besides, testing for the existence of library
6077 * functions without relying on external build tools is impossible.
6078 *
6079 * The method is simple: Overallocate, manually align, and store the offset
6080 * to the original behind the returned pointer.
6081 *
6082 * Align must be a power of 2 and 8 <= align <= 128.
6083 */
6084static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align)
6085{
6086 XXH_ASSERT(align <= 128 && align >= 8); /* range check */
6087 XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */
6088 XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */
6089 { /* Overallocate to make room for manual realignment and an offset byte */
6090 xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
6091 if (base != NULL) {
6092 /*
6093 * Get the offset needed to align this pointer.
6094 *
6095 * Even if the returned pointer is aligned, there will always be
6096 * at least one byte to store the offset to the original pointer.
6097 */
6098 size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
6099 /* Add the offset for the now-aligned pointer */
6100 xxh_u8* ptr = base + offset;
6101
6102 XXH_ASSERT((size_t)ptr % align == 0);
6103
6104 /* Store the offset immediately before the returned pointer. */
6105 ptr[-1] = (xxh_u8)offset;
6106 return ptr;
6107 }
6108 return NULL;
6109 }
6110}
6111/*
6112 * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
6113 * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
6114 */
6115static void XXH_alignedFree(void* p)
6116{
6117 if (p != NULL) {
6118 xxh_u8* ptr = (xxh_u8*)p;
6119 /* Get the offset byte we added in XXH_malloc. */
6120 xxh_u8 offset = ptr[-1];
6121 /* Free the original malloc'd pointer */
6122 xxh_u8* base = ptr - offset;
6123 XXH_free(base);
6124 }
6125}
6126/*! @ingroup XXH3_family */
6127/*!
6128 * @brief Allocate an @ref XXH3_state_t.
6129 *
6130 * @return An allocated pointer of @ref XXH3_state_t on success.
6131 * @return `NULL` on failure.
6132 *
6133 * @note Must be freed with XXH3_freeState().
6134 *
6135 * @see @ref streaming_example "Streaming Example"
6136 */
6137XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
6138{
6139 XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
6140 if (state==NULL) return NULL;
6141 XXH3_INITSTATE(state);
6142 return state;
6143}
6144
6145/*! @ingroup XXH3_family */
6146/*!
6147 * @brief Frees an @ref XXH3_state_t.
6148 *
6149 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
6150 *
6151 * @return @ref XXH_OK.
6152 *
6153 * @note Must be allocated with XXH3_createState().
6154 *
6155 * @see @ref streaming_example "Streaming Example"
6156 */
6157XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
6158{
6159 XXH_alignedFree(statePtr);
6160 return XXH_OK;
6161}
6162
6163/*! @ingroup XXH3_family */
6164XXH_PUBLIC_API void
6165XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state)
6166{
6167 XXH_memcpy(dst_state, src_state, sizeof(*dst_state));
6168}
6169
6170static void
6171XXH3_reset_internal(XXH3_state_t* statePtr,
6172 XXH64_hash_t seed,
6173 const void* secret, size_t secretSize)
6174{
6175 size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
6176 size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
6177 XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
6178 XXH_ASSERT(statePtr != NULL);
6179 /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
6180 memset((char*)statePtr + initStart, 0, initLength);
6181 statePtr->acc[0] = XXH_PRIME32_3;
6182 statePtr->acc[1] = XXH_PRIME64_1;
6183 statePtr->acc[2] = XXH_PRIME64_2;
6184 statePtr->acc[3] = XXH_PRIME64_3;
6185 statePtr->acc[4] = XXH_PRIME64_4;
6186 statePtr->acc[5] = XXH_PRIME32_2;
6187 statePtr->acc[6] = XXH_PRIME64_5;
6188 statePtr->acc[7] = XXH_PRIME32_1;
6189 statePtr->seed = seed;
6190 statePtr->useSeed = (seed != 0);
6191 statePtr->extSecret = (const unsigned char*)secret;
6192 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
6193 statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
6194 statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
6195}
6196
6197/*! @ingroup XXH3_family */
6198XXH_PUBLIC_API XXH_errorcode
6199XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
6200{
6201 if (statePtr == NULL) return XXH_ERROR;
6202 XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
6203 return XXH_OK;
6204}
6205
6206/*! @ingroup XXH3_family */
6207XXH_PUBLIC_API XXH_errorcode
6208XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
6209{
6210 if (statePtr == NULL) return XXH_ERROR;
6211 XXH3_reset_internal(statePtr, 0, secret, secretSize);
6212 if (secret == NULL) return XXH_ERROR;
6213 if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
6214 return XXH_OK;
6215}
6216
6217/*! @ingroup XXH3_family */
6218XXH_PUBLIC_API XXH_errorcode
6219XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
6220{
6221 if (statePtr == NULL) return XXH_ERROR;
6222 if (seed==0) return XXH3_64bits_reset(statePtr);
6223 if ((seed != statePtr->seed) || (statePtr->extSecret != NULL))
6224 XXH3_initCustomSecret(statePtr->customSecret, seed);
6225 XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
6226 return XXH_OK;
6227}
6228
6229/*! @ingroup XXH3_family */
6230XXH_PUBLIC_API XXH_errorcode
6231XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64)
6232{
6233 if (statePtr == NULL) return XXH_ERROR;
6234 if (secret == NULL) return XXH_ERROR;
6235 if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
6236 XXH3_reset_internal(statePtr, seed64, secret, secretSize);
6237 statePtr->useSeed = 1; /* always, even if seed64==0 */
6238 return XXH_OK;
6239}
6240
6241/*!
6242 * @internal
6243 * @brief Processes a large input for XXH3_update() and XXH3_digest_long().
6244 *
6245 * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block.
6246 *
6247 * @param acc Pointer to the 8 accumulator lanes
6248 * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block*
6249 * @param nbStripesPerBlock Number of stripes in a block
6250 * @param input Input pointer
6251 * @param nbStripes Number of stripes to process
6252 * @param secret Secret pointer
6253 * @param secretLimit Offset of the last block in @p secret
6254 * @param f_acc Pointer to an XXH3_accumulate implementation
6255 * @param f_scramble Pointer to an XXH3_scrambleAcc implementation
6256 * @return Pointer past the end of @p input after processing
6257 */
6258XXH_FORCE_INLINE const xxh_u8 *
6259XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
6260 size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
6261 const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
6262 const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
6263 XXH3_f_accumulate f_acc,
6264 XXH3_f_scrambleAcc f_scramble)
6265{
6266 const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE;
6267 /* Process full blocks */
6268 if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) {
6269 /* Process the initial partial block... */
6270 size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr;
6271
6272 do {
6273 /* Accumulate and scramble */
6274 f_acc(acc, input, initialSecret, nbStripesThisIter);
6275 f_scramble(acc, secret + secretLimit);
6276 input += nbStripesThisIter * XXH_STRIPE_LEN;
6277 nbStripes -= nbStripesThisIter;
6278 /* Then continue the loop with the full block size */
6279 nbStripesThisIter = nbStripesPerBlock;
6280 initialSecret = secret;
6281 } while (nbStripes >= nbStripesPerBlock);
6282 *nbStripesSoFarPtr = 0;
6283 }
6284 /* Process a partial block */
6285 if (nbStripes > 0) {
6286 f_acc(acc, input, initialSecret, nbStripes);
6287 input += nbStripes * XXH_STRIPE_LEN;
6288 *nbStripesSoFarPtr += nbStripes;
6289 }
6290 /* Return end pointer */
6291 return input;
6292}
6293
6294#ifndef XXH3_STREAM_USE_STACK
6295# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */
6296# define XXH3_STREAM_USE_STACK 1
6297# endif
6298#endif
6299/*
6300 * Both XXH3_64bits_update and XXH3_128bits_update use this routine.
6301 */
6302XXH_FORCE_INLINE XXH_errorcode
6303XXH3_update(XXH3_state_t* XXH_RESTRICT const state,
6304 const xxh_u8* XXH_RESTRICT input, size_t len,
6305 XXH3_f_accumulate f_acc,
6306 XXH3_f_scrambleAcc f_scramble)
6307{
6308 if (input==NULL) {
6309 XXH_ASSERT(len == 0);
6310 return XXH_OK;
6311 }
6312
6313 XXH_ASSERT(state != NULL);
6314 { const xxh_u8* const bEnd = input + len;
6315 const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
6316#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
6317 /* For some reason, gcc and MSVC seem to suffer greatly
6318 * when operating accumulators directly into state.
6319 * Operating into stack space seems to enable proper optimization.
6320 * clang, on the other hand, doesn't seem to need this trick */
6321 XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8];
6322 XXH_memcpy(acc, state->acc, sizeof(acc));
6323#else
6324 xxh_u64* XXH_RESTRICT const acc = state->acc;
6325#endif
6326 state->totalLen += len;
6327 XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE);
6328
6329 /* small input : just fill in tmp buffer */
6330 if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) {
6331 XXH_memcpy(state->buffer + state->bufferedSize, input, len);
6332 state->bufferedSize += (XXH32_hash_t)len;
6333 return XXH_OK;
6334 }
6335
6336 /* total input is now > XXH3_INTERNALBUFFER_SIZE */
6337 #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
6338 XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */
6339
6340 /*
6341 * Internal buffer is partially filled (always, except at beginning)
6342 * Complete it, then consume it.
6343 */
6344 if (state->bufferedSize) {
6345 size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
6346 XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
6347 input += loadSize;
6348 XXH3_consumeStripes(acc,
6349 &state->nbStripesSoFar, state->nbStripesPerBlock,
6350 state->buffer, XXH3_INTERNALBUFFER_STRIPES,
6351 secret, state->secretLimit,
6352 f_acc, f_scramble);
6353 state->bufferedSize = 0;
6354 }
6355 XXH_ASSERT(input < bEnd);
6356 if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) {
6357 size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
6358 input = XXH3_consumeStripes(acc,
6359 &state->nbStripesSoFar, state->nbStripesPerBlock,
6360 input, nbStripes,
6361 secret, state->secretLimit,
6362 f_acc, f_scramble);
6363 XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
6364
6365 }
6366 /* Some remaining input (always) : buffer it */
6367 XXH_ASSERT(input < bEnd);
6368 XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE);
6369 XXH_ASSERT(state->bufferedSize == 0);
6370 XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
6371 state->bufferedSize = (XXH32_hash_t)(bEnd-input);
6372#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
6373 /* save stack accumulators into state */
6374 XXH_memcpy(state->acc, acc, sizeof(acc));
6375#endif
6376 }
6377
6378 return XXH_OK;
6379}
6380
6381/*! @ingroup XXH3_family */
6382XXH_PUBLIC_API XXH_errorcode
6383XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
6384{
6385 return XXH3_update(state, (const xxh_u8*)input, len,
6386 XXH3_accumulate, XXH3_scrambleAcc);
6387}
6388
6389
6390XXH_FORCE_INLINE void
6391XXH3_digest_long (XXH64_hash_t* acc,
6392 const XXH3_state_t* state,
6393 const unsigned char* secret)
6394{
6395 xxh_u8 lastStripe[XXH_STRIPE_LEN];
6396 const xxh_u8* lastStripePtr;
6397
6398 /*
6399 * Digest on a local copy. This way, the state remains unaltered, and it can
6400 * continue ingesting more input afterwards.
6401 */
6402 XXH_memcpy(acc, state->acc, sizeof(state->acc));
6403 if (state->bufferedSize >= XXH_STRIPE_LEN) {
6404 /* Consume remaining stripes then point to remaining data in buffer */
6405 size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
6406 size_t nbStripesSoFar = state->nbStripesSoFar;
6407 XXH3_consumeStripes(acc,
6408 &nbStripesSoFar, state->nbStripesPerBlock,
6409 state->buffer, nbStripes,
6410 secret, state->secretLimit,
6411 XXH3_accumulate, XXH3_scrambleAcc);
6412 lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN;
6413 } else { /* bufferedSize < XXH_STRIPE_LEN */
6414 /* Copy to temp buffer */
6415 size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
6416 XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */
6417 XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
6418 XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
6419 lastStripePtr = lastStripe;
6420 }
6421 /* Last stripe */
6422 XXH3_accumulate_512(acc,
6423 lastStripePtr,
6424 secret + state->secretLimit - XXH_SECRET_LASTACC_START);
6425}
6426
6427/*! @ingroup XXH3_family */
6428XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
6429{
6430 const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
6431 if (state->totalLen > XXH3_MIDSIZE_MAX) {
6432 XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
6433 XXH3_digest_long(acc, state, secret);
6434 return XXH3_mergeAccs(acc,
6435 secret + XXH_SECRET_MERGEACCS_START,
6436 (xxh_u64)state->totalLen * XXH_PRIME64_1);
6437 }
6438 /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
6439 if (state->useSeed)
6440 return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
6441 return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
6442 secret, state->secretLimit + XXH_STRIPE_LEN);
6443}
6444#endif /* !XXH_NO_STREAM */
6445
6446
6447/* ==========================================
6448 * XXH3 128 bits (a.k.a XXH128)
6449 * ==========================================
6450 * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
6451 * even without counting the significantly larger output size.
6452 *
6453 * For example, extra steps are taken to avoid the seed-dependent collisions
6454 * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
6455 *
6456 * This strength naturally comes at the cost of some speed, especially on short
6457 * lengths. Note that longer hashes are about as fast as the 64-bit version
6458 * due to it using only a slight modification of the 64-bit loop.
6459 *
6460 * XXH128 is also more oriented towards 64-bit machines. It is still extremely
6461 * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
6462 */
6463
6464XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6465XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6466{
6467 /* A doubled version of 1to3_64b with different constants. */
6468 XXH_ASSERT(input != NULL);
6469 XXH_ASSERT(1 <= len && len <= 3);
6470 XXH_ASSERT(secret != NULL);
6471 /*
6472 * len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
6473 * len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
6474 * len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
6475 */
6476 { xxh_u8 const c1 = input[0];
6477 xxh_u8 const c2 = input[len >> 1];
6478 xxh_u8 const c3 = input[len - 1];
6479 xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24)
6480 | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
6481 xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
6482 xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
6483 xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
6484 xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
6485 xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
6486 XXH128_hash_t h128;
6487 h128.low64 = XXH64_avalanche(keyed_lo);
6488 h128.high64 = XXH64_avalanche(keyed_hi);
6489 return h128;
6490 }
6491}
6492
6493XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6494XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6495{
6496 XXH_ASSERT(input != NULL);
6497 XXH_ASSERT(secret != NULL);
6498 XXH_ASSERT(4 <= len && len <= 8);
6499 seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
6500 { xxh_u32 const input_lo = XXH_readLE32(input);
6501 xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
6502 xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
6503 xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
6504 xxh_u64 const keyed = input_64 ^ bitflip;
6505
6506 /* Shift len to the left to ensure it is even, this avoids even multiplies. */
6507 XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
6508
6509 m128.high64 += (m128.low64 << 1);
6510 m128.low64 ^= (m128.high64 >> 3);
6511
6512 m128.low64 = XXH_xorshift64(m128.low64, 35);
6513 m128.low64 *= PRIME_MX2;
6514 m128.low64 = XXH_xorshift64(m128.low64, 28);
6515 m128.high64 = XXH3_avalanche(m128.high64);
6516 return m128;
6517 }
6518}
6519
6520XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6521XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6522{
6523 XXH_ASSERT(input != NULL);
6524 XXH_ASSERT(secret != NULL);
6525 XXH_ASSERT(9 <= len && len <= 16);
6526 { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
6527 xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
6528 xxh_u64 const input_lo = XXH_readLE64(input);
6529 xxh_u64 input_hi = XXH_readLE64(input + len - 8);
6530 XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
6531 /*
6532 * Put len in the middle of m128 to ensure that the length gets mixed to
6533 * both the low and high bits in the 128x64 multiply below.
6534 */
6535 m128.low64 += (xxh_u64)(len - 1) << 54;
6536 input_hi ^= bitfliph;
6537 /*
6538 * Add the high 32 bits of input_hi to the high 32 bits of m128, then
6539 * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
6540 * the high 64 bits of m128.
6541 *
6542 * The best approach to this operation is different on 32-bit and 64-bit.
6543 */
6544 if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */
6545 /*
6546 * 32-bit optimized version, which is more readable.
6547 *
6548 * On 32-bit, it removes an ADC and delays a dependency between the two
6549 * halves of m128.high64, but it generates an extra mask on 64-bit.
6550 */
6551 m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
6552 } else {
6553 /*
6554 * 64-bit optimized (albeit more confusing) version.
6555 *
6556 * Uses some properties of addition and multiplication to remove the mask:
6557 *
6558 * Let:
6559 * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
6560 * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
6561 * c = XXH_PRIME32_2
6562 *
6563 * a + (b * c)
6564 * Inverse Property: x + y - x == y
6565 * a + (b * (1 + c - 1))
6566 * Distributive Property: x * (y + z) == (x * y) + (x * z)
6567 * a + (b * 1) + (b * (c - 1))
6568 * Identity Property: x * 1 == x
6569 * a + b + (b * (c - 1))
6570 *
6571 * Substitute a, b, and c:
6572 * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
6573 *
6574 * Since input_hi.hi + input_hi.lo == input_hi, we get this:
6575 * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
6576 */
6577 m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
6578 }
6579 /* m128 ^= XXH_swap64(m128 >> 64); */
6580 m128.low64 ^= XXH_swap64(m128.high64);
6581
6582 { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
6583 XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
6584 h128.high64 += m128.high64 * XXH_PRIME64_2;
6585
6586 h128.low64 = XXH3_avalanche(h128.low64);
6587 h128.high64 = XXH3_avalanche(h128.high64);
6588 return h128;
6589 } }
6590}
6591
6592/*
6593 * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
6594 */
6595XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6596XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6597{
6598 XXH_ASSERT(len <= 16);
6599 { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
6600 if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
6601 if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
6602 { XXH128_hash_t h128;
6603 xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
6604 xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
6605 h128.low64 = XXH64_avalanche(seed ^ bitflipl);
6606 h128.high64 = XXH64_avalanche( seed ^ bitfliph);
6607 return h128;
6608 } }
6609}
6610
6611/*
6612 * A bit slower than XXH3_mix16B, but handles multiply by zero better.
6613 */
6614XXH_FORCE_INLINE XXH128_hash_t
6615XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
6616 const xxh_u8* secret, XXH64_hash_t seed)
6617{
6618 acc.low64 += XXH3_mix16B (input_1, secret+0, seed);
6619 acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
6620 acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
6621 acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
6622 return acc;
6623}
6624
6625
6626XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
6627XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
6628 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
6629 XXH64_hash_t seed)
6630{
6631 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
6632 XXH_ASSERT(16 < len && len <= 128);
6633
6634 { XXH128_hash_t acc;
6635 acc.low64 = len * XXH_PRIME64_1;
6636 acc.high64 = 0;
6637
6638#if XXH_SIZE_OPT >= 1
6639 {
6640 /* Smaller, but slightly slower. */
6641 unsigned int i = (unsigned int)(len - 1) / 32;
6642 do {
6643 acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed);
6644 } while (i-- != 0);
6645 }
6646#else
6647 if (len > 32) {
6648 if (len > 64) {
6649 if (len > 96) {
6650 acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
6651 }
6652 acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
6653 }
6654 acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
6655 }
6656 acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
6657#endif
6658 { XXH128_hash_t h128;
6659 h128.low64 = acc.low64 + acc.high64;
6660 h128.high64 = (acc.low64 * XXH_PRIME64_1)
6661 + (acc.high64 * XXH_PRIME64_4)
6662 + ((len - seed) * XXH_PRIME64_2);
6663 h128.low64 = XXH3_avalanche(h128.low64);
6664 h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
6665 return h128;
6666 }
6667 }
6668}
6669
6670XXH_NO_INLINE XXH_PUREF XXH128_hash_t
6671XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
6672 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
6673 XXH64_hash_t seed)
6674{
6675 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
6676 XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
6677
6678 { XXH128_hash_t acc;
6679 unsigned i;
6680 acc.low64 = len * XXH_PRIME64_1;
6681 acc.high64 = 0;
6682 /*
6683 * We set as `i` as offset + 32. We do this so that unchanged
6684 * `len` can be used as upper bound. This reaches a sweet spot
6685 * where both x86 and aarch64 get simple agen and good codegen
6686 * for the loop.
6687 */
6688 for (i = 32; i < 160; i += 32) {
6689 acc = XXH128_mix32B(acc,
6690 input + i - 32,
6691 input + i - 16,
6692 secret + i - 32,
6693 seed);
6694 }
6695 acc.low64 = XXH3_avalanche(acc.low64);
6696 acc.high64 = XXH3_avalanche(acc.high64);
6697 /*
6698 * NB: `i <= len` will duplicate the last 32-bytes if
6699 * len % 32 was zero. This is an unfortunate necessity to keep
6700 * the hash result stable.
6701 */
6702 for (i=160; i <= len; i += 32) {
6703 acc = XXH128_mix32B(acc,
6704 input + i - 32,
6705 input + i - 16,
6706 secret + XXH3_MIDSIZE_STARTOFFSET + i - 160,
6707 seed);
6708 }
6709 /* last bytes */
6710 acc = XXH128_mix32B(acc,
6711 input + len - 16,
6712 input + len - 32,
6713 secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
6714 (XXH64_hash_t)0 - seed);
6715
6716 { XXH128_hash_t h128;
6717 h128.low64 = acc.low64 + acc.high64;
6718 h128.high64 = (acc.low64 * XXH_PRIME64_1)
6719 + (acc.high64 * XXH_PRIME64_4)
6720 + ((len - seed) * XXH_PRIME64_2);
6721 h128.low64 = XXH3_avalanche(h128.low64);
6722 h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
6723 return h128;
6724 }
6725 }
6726}
6727
6728XXH_FORCE_INLINE XXH128_hash_t
6729XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
6730 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
6731 XXH3_f_accumulate f_acc,
6732 XXH3_f_scrambleAcc f_scramble)
6733{
6734 XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
6735
6736 XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble);
6737
6738 /* converge into final hash */
6739 XXH_STATIC_ASSERT(sizeof(acc) == 64);
6740 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
6741 { XXH128_hash_t h128;
6742 h128.low64 = XXH3_mergeAccs(acc,
6743 secret + XXH_SECRET_MERGEACCS_START,
6744 (xxh_u64)len * XXH_PRIME64_1);
6745 h128.high64 = XXH3_mergeAccs(acc,
6746 secret + secretSize
6747 - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
6748 ~((xxh_u64)len * XXH_PRIME64_2));
6749 return h128;
6750 }
6751}
6752
6753/*
6754 * It's important for performance that XXH3_hashLong() is not inlined.
6755 */
6756XXH_NO_INLINE XXH_PUREF XXH128_hash_t
6757XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
6758 XXH64_hash_t seed64,
6759 const void* XXH_RESTRICT secret, size_t secretLen)
6760{
6761 (void)seed64; (void)secret; (void)secretLen;
6762 return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
6763 XXH3_accumulate, XXH3_scrambleAcc);
6764}
6765
6766/*
6767 * It's important for performance to pass @p secretLen (when it's static)
6768 * to the compiler, so that it can properly optimize the vectorized loop.
6769 *
6770 * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
6771 * breaks -Og, this is XXH_NO_INLINE.
6772 */
6773XXH3_WITH_SECRET_INLINE XXH128_hash_t
6774XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
6775 XXH64_hash_t seed64,
6776 const void* XXH_RESTRICT secret, size_t secretLen)
6777{
6778 (void)seed64;
6779 return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
6780 XXH3_accumulate, XXH3_scrambleAcc);
6781}
6782
6783XXH_FORCE_INLINE XXH128_hash_t
6784XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
6785 XXH64_hash_t seed64,
6786 XXH3_f_accumulate f_acc,
6787 XXH3_f_scrambleAcc f_scramble,
6788 XXH3_f_initCustomSecret f_initSec)
6789{
6790 if (seed64 == 0)
6791 return XXH3_hashLong_128b_internal(input, len,
6792 XXH3_kSecret, sizeof(XXH3_kSecret),
6793 f_acc, f_scramble);
6794 { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
6795 f_initSec(secret, seed64);
6796 return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
6797 f_acc, f_scramble);
6798 }
6799}
6800
6801/*
6802 * It's important for performance that XXH3_hashLong is not inlined.
6803 */
6804XXH_NO_INLINE XXH128_hash_t
6805XXH3_hashLong_128b_withSeed(const void* input, size_t len,
6806 XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
6807{
6808 (void)secret; (void)secretLen;
6809 return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
6810 XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
6811}
6812
6813typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t,
6814 XXH64_hash_t, const void* XXH_RESTRICT, size_t);
6815
6816XXH_FORCE_INLINE XXH128_hash_t
6817XXH3_128bits_internal(const void* input, size_t len,
6818 XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
6819 XXH3_hashLong128_f f_hl128)
6820{
6821 XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
6822 /*
6823 * If an action is to be taken if `secret` conditions are not respected,
6824 * it should be done here.
6825 * For now, it's a contract pre-condition.
6826 * Adding a check and a branch here would cost performance at every hash.
6827 */
6828 if (len <= 16)
6829 return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
6830 if (len <= 128)
6831 return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6832 if (len <= XXH3_MIDSIZE_MAX)
6833 return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6834 return f_hl128(input, len, seed64, secret, secretLen);
6835}
6836
6837
6838/* === Public XXH128 API === */
6839
6840/*! @ingroup XXH3_family */
6841XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len)
6842{
6843 return XXH3_128bits_internal(input, len, 0,
6844 XXH3_kSecret, sizeof(XXH3_kSecret),
6845 XXH3_hashLong_128b_default);
6846}
6847
6848/*! @ingroup XXH3_family */
6849XXH_PUBLIC_API XXH128_hash_t
6850XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize)
6851{
6852 return XXH3_128bits_internal(input, len, 0,
6853 (const xxh_u8*)secret, secretSize,
6854 XXH3_hashLong_128b_withSecret);
6855}
6856
6857/*! @ingroup XXH3_family */
6858XXH_PUBLIC_API XXH128_hash_t
6859XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
6860{
6861 return XXH3_128bits_internal(input, len, seed,
6862 XXH3_kSecret, sizeof(XXH3_kSecret),
6863 XXH3_hashLong_128b_withSeed);
6864}
6865
6866/*! @ingroup XXH3_family */
6867XXH_PUBLIC_API XXH128_hash_t
6868XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
6869{
6870 if (len <= XXH3_MIDSIZE_MAX)
6871 return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
6872 return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
6873}
6874
6875/*! @ingroup XXH3_family */
6876XXH_PUBLIC_API XXH128_hash_t
6877XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
6878{
6879 return XXH3_128bits_withSeed(input, len, seed);
6880}
6881
6882
6883/* === XXH3 128-bit streaming === */
6884#ifndef XXH_NO_STREAM
6885/*
6886 * All initialization and update functions are identical to 64-bit streaming variant.
6887 * The only difference is the finalization routine.
6888 */
6889
6890/*! @ingroup XXH3_family */
6891XXH_PUBLIC_API XXH_errorcode
6892XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
6893{
6894 return XXH3_64bits_reset(statePtr);
6895}
6896
6897/*! @ingroup XXH3_family */
6898XXH_PUBLIC_API XXH_errorcode
6899XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
6900{
6901 return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize);
6902}
6903
6904/*! @ingroup XXH3_family */
6905XXH_PUBLIC_API XXH_errorcode
6906XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
6907{
6908 return XXH3_64bits_reset_withSeed(statePtr, seed);
6909}
6910
6911/*! @ingroup XXH3_family */
6912XXH_PUBLIC_API XXH_errorcode
6913XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
6914{
6915 return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
6916}
6917
6918/*! @ingroup XXH3_family */
6919XXH_PUBLIC_API XXH_errorcode
6920XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
6921{
6922 return XXH3_64bits_update(state, input, len);
6923}
6924
6925/*! @ingroup XXH3_family */
6926XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
6927{
6928 const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
6929 if (state->totalLen > XXH3_MIDSIZE_MAX) {
6930 XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
6931 XXH3_digest_long(acc, state, secret);
6932 XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
6933 { XXH128_hash_t h128;
6934 h128.low64 = XXH3_mergeAccs(acc,
6935 secret + XXH_SECRET_MERGEACCS_START,
6936 (xxh_u64)state->totalLen * XXH_PRIME64_1);
6937 h128.high64 = XXH3_mergeAccs(acc,
6938 secret + state->secretLimit + XXH_STRIPE_LEN
6939 - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
6940 ~((xxh_u64)state->totalLen * XXH_PRIME64_2));
6941 return h128;
6942 }
6943 }
6944 /* len <= XXH3_MIDSIZE_MAX : short code */
6945 if (state->useSeed)
6946 return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
6947 return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
6948 secret, state->secretLimit + XXH_STRIPE_LEN);
6949}
6950#endif /* !XXH_NO_STREAM */
6951/* 128-bit utility functions */
6952
6953#include <string.h> /* memcmp, memcpy */
6954
6955/* return : 1 is equal, 0 if different */
6956/*! @ingroup XXH3_family */
6957XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
6958{
6959 /* note : XXH128_hash_t is compact, it has no padding byte */
6960 return !(memcmp(&h1, &h2, sizeof(h1)));
6961}
6962
6963/* This prototype is compatible with stdlib's qsort().
6964 * @return : >0 if *h128_1 > *h128_2
6965 * <0 if *h128_1 < *h128_2
6966 * =0 if *h128_1 == *h128_2 */
6967/*! @ingroup XXH3_family */
6968XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2)
6969{
6970 XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
6971 XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
6972 int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
6973 /* note : bets that, in most cases, hash values are different */
6974 if (hcmp) return hcmp;
6975 return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
6976}
6977
6978
6979/*====== Canonical representation ======*/
6980/*! @ingroup XXH3_family */
6981XXH_PUBLIC_API void
6982XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash)
6983{
6984 XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
6985 if (XXH_CPU_LITTLE_ENDIAN) {
6986 hash.high64 = XXH_swap64(hash.high64);
6987 hash.low64 = XXH_swap64(hash.low64);
6988 }
6989 XXH_memcpy(dst, &hash.high64, sizeof(hash.high64));
6990 XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
6991}
6992
6993/*! @ingroup XXH3_family */
6994XXH_PUBLIC_API XXH128_hash_t
6995XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src)
6996{
6997 XXH128_hash_t h;
6998 h.high64 = XXH_readBE64(src);
6999 h.low64 = XXH_readBE64(src->digest + 8);
7000 return h;
7001}
7002
7003
7004
7005/* ==========================================
7006 * Secret generators
7007 * ==========================================
7008 */
7009#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))
7010
7011XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128)
7012{
7013 XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 );
7014 XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 );
7015}
7016
7017/*! @ingroup XXH3_family */
7018XXH_PUBLIC_API XXH_errorcode
7019XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize)
7020{
7021#if (XXH_DEBUGLEVEL >= 1)
7022 XXH_ASSERT(secretBuffer != NULL);
7023 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
7024#else
7025 /* production mode, assert() are disabled */
7026 if (secretBuffer == NULL) return XXH_ERROR;
7027 if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
7028#endif
7029
7030 if (customSeedSize == 0) {
7031 customSeed = XXH3_kSecret;
7032 customSeedSize = XXH_SECRET_DEFAULT_SIZE;
7033 }
7034#if (XXH_DEBUGLEVEL >= 1)
7035 XXH_ASSERT(customSeed != NULL);
7036#else
7037 if (customSeed == NULL) return XXH_ERROR;
7038#endif
7039
7040 /* Fill secretBuffer with a copy of customSeed - repeat as needed */
7041 { size_t pos = 0;
7042 while (pos < secretSize) {
7043 size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize);
7044 memcpy((char*)secretBuffer + pos, customSeed, toCopy);
7045 pos += toCopy;
7046 } }
7047
7048 { size_t const nbSeg16 = secretSize / 16;
7049 size_t n;
7050 XXH128_canonical_t scrambler;
7051 XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
7052 for (n=0; n<nbSeg16; n++) {
7053 XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n);
7054 XXH3_combine16((char*)secretBuffer + n*16, h128);
7055 }
7056 /* last segment */
7057 XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler));
7058 }
7059 return XXH_OK;
7060}
7061
7062/*! @ingroup XXH3_family */
7063XXH_PUBLIC_API void
7064XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed)
7065{
7066 XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
7067 XXH3_initCustomSecret(secret, seed);
7068 XXH_ASSERT(secretBuffer != NULL);
7069 memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE);
7070}
7071
7072
7073
7074/* Pop our optimization override from above */
7075#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
7076 && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
7077 && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
7078# pragma GCC pop_options
7079#endif
7080
7081#endif /* XXH_NO_LONG_LONG */
7082
7083#endif /* XXH_NO_XXH3 */
7084
7085/*!
7086 * @}
7087 */
7088#endif /* XXH_IMPLEMENTATION */
7089
7090
7091#if defined (__cplusplus)
7092} /* extern "C" */
7093#endif