summaryrefslogtreecommitdiff
path: root/examples/redis-unstable/src/hyperloglog.c
diff options
context:
space:
mode:
authorMitja Felicijan <mitja.felicijan@gmail.com>2026-01-21 22:52:54 +0100
committerMitja Felicijan <mitja.felicijan@gmail.com>2026-01-21 22:52:54 +0100
commitdcacc00e3750300617ba6e16eb346713f91a783a (patch)
tree38e2d4fb5ed9d119711d4295c6eda4b014af73fd /examples/redis-unstable/src/hyperloglog.c
parent58dac10aeb8f5a041c46bddbeaf4c7966a99b998 (diff)
downloadcrep-dcacc00e3750300617ba6e16eb346713f91a783a.tar.gz
Remove testing data
Diffstat (limited to 'examples/redis-unstable/src/hyperloglog.c')
-rw-r--r--examples/redis-unstable/src/hyperloglog.c2099
1 files changed, 0 insertions, 2099 deletions
diff --git a/examples/redis-unstable/src/hyperloglog.c b/examples/redis-unstable/src/hyperloglog.c
deleted file mode 100644
index 05a5152..0000000
--- a/examples/redis-unstable/src/hyperloglog.c
+++ /dev/null
@@ -1,2099 +0,0 @@
-/* hyperloglog.c - Redis HyperLogLog probabilistic cardinality approximation.
- * This file implements the algorithm and the exported Redis commands.
- *
- * Copyright (c) 2014-Present, Redis Ltd.
- * All rights reserved.
- *
- * Copyright (c) 2024-present, Valkey contributors.
- * All rights reserved.
- *
- * Licensed under your choice of (a) the Redis Source Available License 2.0
- * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
- * GNU Affero General Public License v3 (AGPLv3).
- *
- * Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.
- */
-
-#include "server.h"
-
-#include <stdint.h>
-#include <math.h>
-
-#ifdef HAVE_AVX2
-/* Define __MM_MALLOC_H to prevent importing the memory aligned
- * allocation functions, which we don't use. */
-#define __MM_MALLOC_H
-#include <immintrin.h>
-#endif
-
-#ifdef HAVE_AARCH64_NEON
-#include <arm_neon.h>
-#endif
-
-#undef MAX
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
-
-/* The Redis HyperLogLog implementation is based on the following ideas:
- *
- * * The use of a 64 bit hash function as proposed in [1], in order to estimate
- * cardinalities larger than 10^9, at the cost of just 1 additional bit per
- * register.
- * * The use of 16384 6-bit registers for a great level of accuracy, using
- * a total of 12k per key.
- * * The use of the Redis string data type. No new type is introduced.
- * * No attempt is made to compress the data structure as in [1]. Also the
- * algorithm used is the original HyperLogLog Algorithm as in [2], with
- * the only difference that a 64 bit hash function is used, so no correction
- * is performed for values near 2^32 as in [1].
- *
- * [1] Heule, Nunkesser, Hall: HyperLogLog in Practice: Algorithmic
- * Engineering of a State of The Art Cardinality Estimation Algorithm.
- *
- * [2] P. Flajolet, Éric Fusy, O. Gandouet, and F. Meunier. Hyperloglog: The
- * analysis of a near-optimal cardinality estimation algorithm.
- *
- * Redis uses two representations:
- *
- * 1) A "dense" representation where every entry is represented by
- * a 6-bit integer.
- * 2) A "sparse" representation using run length compression suitable
- * for representing HyperLogLogs with many registers set to 0 in
- * a memory efficient way.
- *
- *
- * HLL header
- * ===
- *
- * Both the dense and sparse representation have a 16 byte header as follows:
- *
- * +------+---+-----+----------+
- * | HYLL | E | N/U | Cardin. |
- * +------+---+-----+----------+
- *
- * The first 4 bytes are a magic string set to the bytes "HYLL".
- * "E" is one byte encoding, currently set to HLL_DENSE or
- * HLL_SPARSE. N/U are three not used bytes.
- *
- * The "Cardin." field is a 64 bit integer stored in little endian format
- * with the latest cardinality computed that can be reused if the data
- * structure was not modified since the last computation (this is useful
- * because there are high probabilities that HLLADD operations don't
- * modify the actual data structure and hence the approximated cardinality).
- *
- * When the most significant bit in the most significant byte of the cached
- * cardinality is set, it means that the data structure was modified and
- * we can't reuse the cached value that must be recomputed.
- *
- * Dense representation
- * ===
- *
- * The dense representation used by Redis is the following:
- *
- * +--------+--------+--------+------// //--+
- * |11000000|22221111|33333322|55444444 .... |
- * +--------+--------+--------+------// //--+
- *
- * The 6 bits counters are encoded one after the other starting from the
- * LSB to the MSB, and using the next bytes as needed.
- *
- * Sparse representation
- * ===
- *
- * The sparse representation encodes registers using a run length
- * encoding composed of three opcodes, two using one byte, and one using
- * of two bytes. The opcodes are called ZERO, XZERO and VAL.
- *
- * ZERO opcode is represented as 00xxxxxx. The 6-bit integer represented
- * by the six bits 'xxxxxx', plus 1, means that there are N registers set
- * to 0. This opcode can represent from 1 to 64 contiguous registers set
- * to the value of 0.
- *
- * XZERO opcode is represented by two bytes 01xxxxxx yyyyyyyy. The 14-bit
- * integer represented by the bits 'xxxxxx' as most significant bits and
- * 'yyyyyyyy' as least significant bits, plus 1, means that there are N
- * registers set to 0. This opcode can represent from 0 to 16384 contiguous
- * registers set to the value of 0.
- *
- * VAL opcode is represented as 1vvvvvxx. It contains a 5-bit integer
- * representing the value of a register, and a 2-bit integer representing
- * the number of contiguous registers set to that value 'vvvvv'.
- * To obtain the value and run length, the integers vvvvv and xx must be
- * incremented by one. This opcode can represent values from 1 to 32,
- * repeated from 1 to 4 times.
- *
- * The sparse representation can't represent registers with a value greater
- * than 32, however it is very unlikely that we find such a register in an
- * HLL with a cardinality where the sparse representation is still more
- * memory efficient than the dense representation. When this happens the
- * HLL is converted to the dense representation.
- *
- * The sparse representation is purely positional. For example a sparse
- * representation of an empty HLL is just: XZERO:16384.
- *
- * An HLL having only 3 non-zero registers at position 1000, 1020, 1021
- * respectively set to 2, 3, 3, is represented by the following three
- * opcodes:
- *
- * XZERO:1000 (Registers 0-999 are set to 0)
- * VAL:2,1 (1 register set to value 2, that is register 1000)
- * ZERO:19 (Registers 1001-1019 set to 0)
- * VAL:3,2 (2 registers set to value 3, that is registers 1020,1021)
- * XZERO:15362 (Registers 1022-16383 set to 0)
- *
- * In the example the sparse representation used just 7 bytes instead
- * of 12k in order to represent the HLL registers. In general for low
- * cardinality there is a big win in terms of space efficiency, traded
- * with CPU time since the sparse representation is slower to access.
- *
- * The following table shows average cardinality vs bytes used, 100
- * samples per cardinality (when the set was not representable because
- * of registers with too big value, the dense representation size was used
- * as a sample).
- *
- * 100 267
- * 200 485
- * 300 678
- * 400 859
- * 500 1033
- * 600 1205
- * 700 1375
- * 800 1544
- * 900 1713
- * 1000 1882
- * 2000 3480
- * 3000 4879
- * 4000 6089
- * 5000 7138
- * 6000 8042
- * 7000 8823
- * 8000 9500
- * 9000 10088
- * 10000 10591
- *
- * The dense representation uses 12288 bytes, so there is a big win up to
- * a cardinality of ~2000-3000. For bigger cardinalities the constant times
- * involved in updating the sparse representation is not justified by the
- * memory savings. The exact maximum length of the sparse representation
- * when this implementation switches to the dense representation is
- * configured via the define server.hll_sparse_max_bytes.
- */
-
-struct hllhdr {
- char magic[4]; /* "HYLL" */
- uint8_t encoding; /* HLL_DENSE or HLL_SPARSE. */
- uint8_t notused[3]; /* Reserved for future use, must be zero. */
- uint8_t card[8]; /* Cached cardinality, little endian. */
- uint8_t registers[]; /* Data bytes. */
-};
-
-/* The cached cardinality MSB is used to signal validity of the cached value. */
-#define HLL_INVALIDATE_CACHE(hdr) (hdr)->card[7] |= (1<<7)
-#define HLL_VALID_CACHE(hdr) (((hdr)->card[7] & (1<<7)) == 0)
-
-#define HLL_P 14 /* The greater is P, the smaller the error. */
-#define HLL_Q (64-HLL_P) /* The number of bits of the hash value used for
- determining the number of leading zeros. */
-#define HLL_REGISTERS (1<<HLL_P) /* With P=14, 16384 registers. */
-#define HLL_P_MASK (HLL_REGISTERS-1) /* Mask to index register. */
-#define HLL_BITS 6 /* Enough to count up to 63 leading zeroes. */
-#define HLL_REGISTER_MAX ((1<<HLL_BITS)-1)
-#define HLL_HDR_SIZE sizeof(struct hllhdr)
-#define HLL_DENSE_SIZE (HLL_HDR_SIZE+((HLL_REGISTERS*HLL_BITS+7)/8))
-#define HLL_DENSE 0 /* Dense encoding. */
-#define HLL_SPARSE 1 /* Sparse encoding. */
-#define HLL_RAW 255 /* Only used internally, never exposed. */
-#define HLL_MAX_ENCODING 1
-
-static char *invalid_hll_err = "-INVALIDOBJ Corrupted HLL object detected";
-
-#if defined(HAVE_AVX2) || defined(HAVE_AARCH64_NEON)
-static int simd_enabled = 1;
-#endif
-
-#ifdef HAVE_AVX2
-#define HLL_USE_AVX2 (simd_enabled && __builtin_cpu_supports("avx2"))
-#else
-#define HLL_USE_AVX2 0
-#endif
-
-#ifdef HAVE_AARCH64_NEON
-#define HLL_USE_NEON (simd_enabled)
-#else
-#define HLL_USE_NEON 0
-#endif
-
-/* =========================== Low level bit macros ========================= */
-
-/* Macros to access the dense representation.
- *
- * We need to get and set 6 bit counters in an array of 8 bit bytes.
- * We use macros to make sure the code is inlined since speed is critical
- * especially in order to compute the approximated cardinality in
- * HLLCOUNT where we need to access all the registers at once.
- * For the same reason we also want to avoid conditionals in this code path.
- *
- * +--------+--------+--------+------//
- * |11000000|22221111|33333322|55444444
- * +--------+--------+--------+------//
- *
- * Note: in the above representation the most significant bit (MSB)
- * of every byte is on the left. We start using bits from the LSB to MSB,
- * and so forth passing to the next byte.
- *
- * Example, we want to access to counter at pos = 1 ("111111" in the
- * illustration above).
- *
- * The index of the first byte b0 containing our data is:
- *
- * b0 = 6 * pos / 8 = 0
- *
- * +--------+
- * |11000000| <- Our byte at b0
- * +--------+
- *
- * The position of the first bit (counting from the LSB = 0) in the byte
- * is given by:
- *
- * fb = 6 * pos % 8 -> 6
- *
- * Right shift b0 of 'fb' bits.
- *
- * +--------+
- * |11000000| <- Initial value of b0
- * |00000011| <- After right shift of 6 pos.
- * +--------+
- *
- * Left shift b1 of bits 8-fb bits (2 bits)
- *
- * +--------+
- * |22221111| <- Initial value of b1
- * |22111100| <- After left shift of 2 bits.
- * +--------+
- *
- * OR the two bits, and finally AND with 111111 (63 in decimal) to
- * clean the higher order bits we are not interested in:
- *
- * +--------+
- * |00000011| <- b0 right shifted
- * |22111100| <- b1 left shifted
- * |22111111| <- b0 OR b1
- * | 111111| <- (b0 OR b1) AND 63, our value.
- * +--------+
- *
- * We can try with a different example, like pos = 0. In this case
- * the 6-bit counter is actually contained in a single byte.
- *
- * b0 = 6 * pos / 8 = 0
- *
- * +--------+
- * |11000000| <- Our byte at b0
- * +--------+
- *
- * fb = 6 * pos % 8 = 0
- *
- * So we right shift of 0 bits (no shift in practice) and
- * left shift the next byte of 8 bits, even if we don't use it,
- * but this has the effect of clearing the bits so the result
- * will not be affected after the OR.
- *
- * -------------------------------------------------------------------------
- *
- * Setting the register is a bit more complex, let's assume that 'val'
- * is the value we want to set, already in the right range.
- *
- * We need two steps, in one we need to clear the bits, and in the other
- * we need to bitwise-OR the new bits.
- *
- * Let's try with 'pos' = 1, so our first byte at 'b' is 0,
- *
- * "fb" is 6 in this case.
- *
- * +--------+
- * |11000000| <- Our byte at b0
- * +--------+
- *
- * To create an AND-mask to clear the bits about this position, we just
- * initialize the mask with the value 63, left shift it of "fs" bits,
- * and finally invert the result.
- *
- * +--------+
- * |00111111| <- "mask" starts at 63
- * |11000000| <- "mask" after left shift of "ls" bits.
- * |00111111| <- "mask" after invert.
- * +--------+
- *
- * Now we can bitwise-AND the byte at "b" with the mask, and bitwise-OR
- * it with "val" left-shifted of "ls" bits to set the new bits.
- *
- * Now let's focus on the next byte b1:
- *
- * +--------+
- * |22221111| <- Initial value of b1
- * +--------+
- *
- * To build the AND mask we start again with the 63 value, right shift
- * it by 8-fb bits, and invert it.
- *
- * +--------+
- * |00111111| <- "mask" set at 2&6-1
- * |00001111| <- "mask" after the right shift by 8-fb = 2 bits
- * |11110000| <- "mask" after bitwise not.
- * +--------+
- *
- * Now we can mask it with b+1 to clear the old bits, and bitwise-OR
- * with "val" left-shifted by "rs" bits to set the new value.
- */
-
-/* Note: if we access the last counter, we will also access the b+1 byte
- * that is out of the array, but sds strings always have an implicit null
- * term, so the byte exists, and we can skip the conditional (or the need
- * to allocate 1 byte more explicitly). */
-
-/* Store the value of the register at position 'regnum' into variable 'target'.
- * 'p' is an array of unsigned bytes. */
-#define HLL_DENSE_GET_REGISTER(target,p,regnum) do { \
- uint8_t *_p = (uint8_t*) p; \
- unsigned long _byte = regnum*HLL_BITS/8; \
- unsigned long _fb = regnum*HLL_BITS&7; \
- unsigned long _fb8 = 8 - _fb; \
- unsigned long b0 = _p[_byte]; \
- unsigned long b1 = _p[_byte+1]; \
- target = ((b0 >> _fb) | (b1 << _fb8)) & HLL_REGISTER_MAX; \
-} while(0)
-
-/* Set the value of the register at position 'regnum' to 'val'.
- * 'p' is an array of unsigned bytes. */
-#define HLL_DENSE_SET_REGISTER(p,regnum,val) do { \
- uint8_t *_p = (uint8_t*) p; \
- unsigned long _byte = (regnum)*HLL_BITS/8; \
- unsigned long _fb = (regnum)*HLL_BITS&7; \
- unsigned long _fb8 = 8 - _fb; \
- unsigned long _v = (val); \
- _p[_byte] &= ~(HLL_REGISTER_MAX << _fb); \
- _p[_byte] |= _v << _fb; \
- _p[_byte+1] &= ~(HLL_REGISTER_MAX >> _fb8); \
- _p[_byte+1] |= _v >> _fb8; \
-} while(0)
-
-/* Macros to access the sparse representation.
- * The macros parameter is expected to be an uint8_t pointer. */
-#define HLL_SPARSE_XZERO_BIT 0x40 /* 01xxxxxx */
-#define HLL_SPARSE_VAL_BIT 0x80 /* 1vvvvvxx */
-#define HLL_SPARSE_IS_ZERO(p) (((*(p)) & 0xc0) == 0) /* 00xxxxxx */
-#define HLL_SPARSE_IS_XZERO(p) (((*(p)) & 0xc0) == HLL_SPARSE_XZERO_BIT)
-#define HLL_SPARSE_IS_VAL(p) ((*(p)) & HLL_SPARSE_VAL_BIT)
-#define HLL_SPARSE_ZERO_LEN(p) (((*(p)) & 0x3f)+1)
-#define HLL_SPARSE_XZERO_LEN(p) (((((*(p)) & 0x3f) << 8) | (*((p)+1)))+1)
-#define HLL_SPARSE_VAL_VALUE(p) ((((*(p)) >> 2) & 0x1f)+1)
-#define HLL_SPARSE_VAL_LEN(p) (((*(p)) & 0x3)+1)
-#define HLL_SPARSE_VAL_MAX_VALUE 32
-#define HLL_SPARSE_VAL_MAX_LEN 4
-#define HLL_SPARSE_ZERO_MAX_LEN 64
-#define HLL_SPARSE_XZERO_MAX_LEN 16384
-#define HLL_SPARSE_VAL_SET(p,val,len) do { \
- *(p) = (((val)-1)<<2|((len)-1))|HLL_SPARSE_VAL_BIT; \
-} while(0)
-#define HLL_SPARSE_ZERO_SET(p,len) do { \
- *(p) = (len)-1; \
-} while(0)
-#define HLL_SPARSE_XZERO_SET(p,len) do { \
- int _l = (len)-1; \
- *(p) = (_l>>8) | HLL_SPARSE_XZERO_BIT; \
- *((p)+1) = (_l&0xff); \
-} while(0)
-#define HLL_ALPHA_INF 0.721347520444481703680 /* constant for 0.5/ln(2) */
-
-/* ========================= HyperLogLog algorithm ========================= */
-
-/* Our hash function is MurmurHash2, 64 bit version.
- * It was modified for Redis in order to provide the same result in
- * big and little endian archs (endian neutral). */
-REDIS_NO_SANITIZE("alignment")
-uint64_t MurmurHash64A (const void * key, size_t len, unsigned int seed) {
- const uint64_t m = 0xc6a4a7935bd1e995;
- const int r = 47;
- uint64_t h = seed ^ (len * m);
- const uint8_t *data = (const uint8_t *)key;
- const uint8_t *end = data + (len-(len&7));
-
- while(data != end) {
- uint64_t k;
-
-#if (BYTE_ORDER == LITTLE_ENDIAN)
- #ifdef USE_ALIGNED_ACCESS
- memcpy(&k,data,sizeof(uint64_t));
- #else
- k = *((uint64_t*)data);
- #endif
-#else
- k = (uint64_t) data[0];
- k |= (uint64_t) data[1] << 8;
- k |= (uint64_t) data[2] << 16;
- k |= (uint64_t) data[3] << 24;
- k |= (uint64_t) data[4] << 32;
- k |= (uint64_t) data[5] << 40;
- k |= (uint64_t) data[6] << 48;
- k |= (uint64_t) data[7] << 56;
-#endif
-
- k *= m;
- k ^= k >> r;
- k *= m;
- h ^= k;
- h *= m;
- data += 8;
- }
-
- switch(len & 7) {
- case 7: h ^= (uint64_t)data[6] << 48; /* fall-thru */
- case 6: h ^= (uint64_t)data[5] << 40; /* fall-thru */
- case 5: h ^= (uint64_t)data[4] << 32; /* fall-thru */
- case 4: h ^= (uint64_t)data[3] << 24; /* fall-thru */
- case 3: h ^= (uint64_t)data[2] << 16; /* fall-thru */
- case 2: h ^= (uint64_t)data[1] << 8; /* fall-thru */
- case 1: h ^= (uint64_t)data[0];
- h *= m; /* fall-thru */
- };
-
- h ^= h >> r;
- h *= m;
- h ^= h >> r;
- return h;
-}
-
-/* Given a string element to add to the HyperLogLog, returns the length
- * of the pattern 000..1 of the element hash. As a side effect 'regp' is
- * set to the register index this element hashes to. */
-int hllPatLen(unsigned char *ele, size_t elesize, long *regp) {
- uint64_t hash, index;
- int count;
-
- /* Count the number of zeroes starting from bit HLL_REGISTERS
- * (that is a power of two corresponding to the first bit we don't use
- * as index). The max run can be 64-P+1 = Q+1 bits.
- *
- * Note that the final "1" ending the sequence of zeroes must be
- * included in the count, so if we find "001" the count is 3, and
- * the smallest count possible is no zeroes at all, just a 1 bit
- * at the first position, that is a count of 1. */
- hash = MurmurHash64A(ele,elesize,0xadc83b19ULL);
- index = hash & HLL_P_MASK; /* Register index. */
- hash >>= HLL_P; /* Remove bits used to address the register. */
- hash |= ((uint64_t)1<<HLL_Q); /* Make sure the loop terminates
- and count will be <= Q+1. */
-
- count = __builtin_ctzll(hash) + 1;
- *regp = (int) index;
- return count;
-}
-
-/* ================== Dense representation implementation ================== */
-
-/* Low level function to set the dense HLL register at 'index' to the
- * specified value if the current value is smaller than 'count'.
- *
- * 'registers' is expected to have room for HLL_REGISTERS plus an
- * additional byte on the right. This requirement is met by sds strings
- * automatically since they are implicitly null terminated.
- *
- * The function always succeed, however if as a result of the operation
- * the approximated cardinality changed, 1 is returned. Otherwise 0
- * is returned. */
-int hllDenseSet(uint8_t *registers, long index, uint8_t count) {
- uint8_t oldcount;
-
- HLL_DENSE_GET_REGISTER(oldcount,registers,index);
- if (count > oldcount) {
- HLL_DENSE_SET_REGISTER(registers,index,count);
- return 1;
- } else {
- return 0;
- }
-}
-
-/* "Add" the element in the dense hyperloglog data structure.
- * Actually nothing is added, but the max 0 pattern counter of the subset
- * the element belongs to is incremented if needed.
- *
- * This is just a wrapper to hllDenseSet(), performing the hashing of the
- * element in order to retrieve the index and zero-run count. */
-int hllDenseAdd(uint8_t *registers, unsigned char *ele, size_t elesize) {
- long index;
- uint8_t count = hllPatLen(ele,elesize,&index);
- /* Update the register if this element produced a longer run of zeroes. */
- return hllDenseSet(registers,index,count);
-}
-
-/* Compute the register histogram in the dense representation. */
-void hllDenseRegHisto(uint8_t *registers, int* reghisto) {
- int j;
-
- /* Redis default is to use 16384 registers 6 bits each. The code works
- * with other values by modifying the defines, but for our target value
- * we take a faster path with unrolled loops. */
- if (HLL_REGISTERS == 16384 && HLL_BITS == 6) {
- uint8_t *r = registers;
- unsigned long r0, r1, r2, r3, r4, r5, r6, r7, r8, r9,
- r10, r11, r12, r13, r14, r15;
- for (j = 0; j < 1024; j++) {
- /* Handle 16 registers per iteration. */
- r0 = r[0] & 63;
- r1 = (r[0] >> 6 | r[1] << 2) & 63;
- r2 = (r[1] >> 4 | r[2] << 4) & 63;
- r3 = (r[2] >> 2) & 63;
- r4 = r[3] & 63;
- r5 = (r[3] >> 6 | r[4] << 2) & 63;
- r6 = (r[4] >> 4 | r[5] << 4) & 63;
- r7 = (r[5] >> 2) & 63;
- r8 = r[6] & 63;
- r9 = (r[6] >> 6 | r[7] << 2) & 63;
- r10 = (r[7] >> 4 | r[8] << 4) & 63;
- r11 = (r[8] >> 2) & 63;
- r12 = r[9] & 63;
- r13 = (r[9] >> 6 | r[10] << 2) & 63;
- r14 = (r[10] >> 4 | r[11] << 4) & 63;
- r15 = (r[11] >> 2) & 63;
-
- reghisto[r0]++;
- reghisto[r1]++;
- reghisto[r2]++;
- reghisto[r3]++;
- reghisto[r4]++;
- reghisto[r5]++;
- reghisto[r6]++;
- reghisto[r7]++;
- reghisto[r8]++;
- reghisto[r9]++;
- reghisto[r10]++;
- reghisto[r11]++;
- reghisto[r12]++;
- reghisto[r13]++;
- reghisto[r14]++;
- reghisto[r15]++;
-
- r += 12;
- }
- } else {
- for(j = 0; j < HLL_REGISTERS; j++) {
- unsigned long reg;
- HLL_DENSE_GET_REGISTER(reg,registers,j);
- reghisto[reg]++;
- }
- }
-}
-
-/* ================== Sparse representation implementation ================= */
-
-/* Convert the HLL with sparse representation given as input in its dense
- * representation. Both representations are represented by SDS strings, and
- * the input representation is freed as a side effect.
- *
- * The function returns C_OK if the sparse representation was valid,
- * otherwise C_ERR is returned if the representation was corrupted. */
-int hllSparseToDense(robj *o) {
- sds sparse = o->ptr, dense;
- struct hllhdr *hdr, *oldhdr = (struct hllhdr*)sparse;
- int idx = 0, runlen, regval;
- uint8_t *p = (uint8_t*)sparse, *end = p+sdslen(sparse);
- int valid = 1;
-
- /* If the representation is already the right one return ASAP. */
- hdr = (struct hllhdr*) sparse;
- if (hdr->encoding == HLL_DENSE) return C_OK;
-
- /* Create a string of the right size filled with zero bytes.
- * Note that the cached cardinality is set to 0 as a side effect
- * that is exactly the cardinality of an empty HLL. */
- dense = sdsnewlen(NULL,HLL_DENSE_SIZE);
- hdr = (struct hllhdr*) dense;
- *hdr = *oldhdr; /* This will copy the magic and cached cardinality. */
- hdr->encoding = HLL_DENSE;
-
- /* Now read the sparse representation and set non-zero registers
- * accordingly. */
- p += HLL_HDR_SIZE;
- while(p < end) {
- if (HLL_SPARSE_IS_ZERO(p)) {
- runlen = HLL_SPARSE_ZERO_LEN(p);
- if ((runlen + idx) > HLL_REGISTERS) { /* Overflow. */
- valid = 0;
- break;
- }
- idx += runlen;
- p++;
- } else if (HLL_SPARSE_IS_XZERO(p)) {
- runlen = HLL_SPARSE_XZERO_LEN(p);
- if ((runlen + idx) > HLL_REGISTERS) { /* Overflow. */
- valid = 0;
- break;
- }
- idx += runlen;
- p += 2;
- } else {
- runlen = HLL_SPARSE_VAL_LEN(p);
- regval = HLL_SPARSE_VAL_VALUE(p);
- if ((runlen + idx) > HLL_REGISTERS) { /* Overflow. */
- valid = 0;
- break;
- }
- while(runlen--) {
- HLL_DENSE_SET_REGISTER(hdr->registers,idx,regval);
- idx++;
- }
- p++;
- }
- }
-
- /* If the sparse representation was valid, we expect to find idx
- * set to HLL_REGISTERS. */
- if (!valid || idx != HLL_REGISTERS) {
- sdsfree(dense);
- return C_ERR;
- }
-
- /* Free the old representation and set the new one. */
- sdsfree(o->ptr);
- o->ptr = dense;
- return C_OK;
-}
-
-/* Low level function to set the sparse HLL register at 'index' to the
- * specified value if the current value is smaller than 'count'.
- *
- * The object 'o' is the String object holding the HLL. The function requires
- * a reference to the object in order to be able to enlarge the string if
- * needed.
- *
- * On success, the function returns 1 if the cardinality changed, or 0
- * if the register for this element was not updated.
- * On error (if the representation is invalid) -1 is returned.
- *
- * As a side effect the function may promote the HLL representation from
- * sparse to dense: this happens when a register requires to be set to a value
- * not representable with the sparse representation, or when the resulting
- * size would be greater than server.hll_sparse_max_bytes. */
-int hllSparseSet(robj *o, long index, uint8_t count) {
- struct hllhdr *hdr;
- uint8_t oldcount, *sparse, *end, *p, *prev, *next;
- long first, span;
- long is_zero = 0, is_xzero = 0, is_val = 0, runlen = 0;
-
- /* If the count is too big to be representable by the sparse representation
- * switch to dense representation. */
- if (count > HLL_SPARSE_VAL_MAX_VALUE) goto promote;
-
- /* When updating a sparse representation, sometimes we may need to enlarge the
- * buffer for up to 3 bytes in the worst case (XZERO split into XZERO-VAL-XZERO),
- * and the following code does the enlarge job.
- * Actually, we use a greedy strategy, enlarge more than 3 bytes to avoid the need
- * for future reallocates on incremental growth. But we do not allocate more than
- * 'server.hll_sparse_max_bytes' bytes for the sparse representation.
- * If the available size of hyperloglog sds string is not enough for the increment
- * we need, we promote the hyperloglog to dense representation in 'step 3'.
- */
- if (sdsalloc(o->ptr) < server.hll_sparse_max_bytes && sdsavail(o->ptr) < 3) {
- size_t newlen = sdslen(o->ptr) + 3;
- newlen += min(newlen, 300); /* Greediness: double 'newlen' if it is smaller than 300, or add 300 to it when it exceeds 300 */
- if (newlen > server.hll_sparse_max_bytes)
- newlen = server.hll_sparse_max_bytes;
- o->ptr = sdsResize(o->ptr, newlen, 1);
- }
-
- /* Step 1: we need to locate the opcode we need to modify to check
- * if a value update is actually needed. */
- sparse = p = ((uint8_t*)o->ptr) + HLL_HDR_SIZE;
- end = p + sdslen(o->ptr) - HLL_HDR_SIZE;
-
- first = 0;
- prev = NULL; /* Points to previous opcode at the end of the loop. */
- next = NULL; /* Points to the next opcode at the end of the loop. */
- span = 0;
- while(p < end) {
- long oplen;
-
- /* Set span to the number of registers covered by this opcode.
- *
- * This is the most performance critical loop of the sparse
- * representation. Sorting the conditionals from the most to the
- * least frequent opcode in many-bytes sparse HLLs is faster. */
- oplen = 1;
- if (HLL_SPARSE_IS_ZERO(p)) {
- span = HLL_SPARSE_ZERO_LEN(p);
- } else if (HLL_SPARSE_IS_VAL(p)) {
- span = HLL_SPARSE_VAL_LEN(p);
- } else { /* XZERO. */
- span = HLL_SPARSE_XZERO_LEN(p);
- oplen = 2;
- }
- /* Break if this opcode covers the register as 'index'. */
- if (index <= first+span-1) break;
- prev = p;
- p += oplen;
- first += span;
- }
- if (span == 0 || p >= end) return -1; /* Invalid format. */
-
- next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1;
- if (next >= end) next = NULL;
-
- /* Cache current opcode type to avoid using the macro again and
- * again for something that will not change.
- * Also cache the run-length of the opcode. */
- if (HLL_SPARSE_IS_ZERO(p)) {
- is_zero = 1;
- runlen = HLL_SPARSE_ZERO_LEN(p);
- } else if (HLL_SPARSE_IS_XZERO(p)) {
- is_xzero = 1;
- runlen = HLL_SPARSE_XZERO_LEN(p);
- } else {
- is_val = 1;
- runlen = HLL_SPARSE_VAL_LEN(p);
- }
-
- /* Step 2: After the loop:
- *
- * 'first' stores to the index of the first register covered
- * by the current opcode, which is pointed by 'p'.
- *
- * 'next' ad 'prev' store respectively the next and previous opcode,
- * or NULL if the opcode at 'p' is respectively the last or first.
- *
- * 'span' is set to the number of registers covered by the current
- * opcode.
- *
- * There are different cases in order to update the data structure
- * in place without generating it from scratch:
- *
- * A) If it is a VAL opcode already set to a value >= our 'count'
- * no update is needed, regardless of the VAL run-length field.
- * In this case PFADD returns 0 since no changes are performed.
- *
- * B) If it is a VAL opcode with len = 1 (representing only our
- * register) and the value is less than 'count', we just update it
- * since this is a trivial case. */
- if (is_val) {
- oldcount = HLL_SPARSE_VAL_VALUE(p);
- /* Case A. */
- if (oldcount >= count) return 0;
-
- /* Case B. */
- if (runlen == 1) {
- HLL_SPARSE_VAL_SET(p,count,1);
- goto updated;
- }
- }
-
- /* C) Another trivial to handle case is a ZERO opcode with a len of 1.
- * We can just replace it with a VAL opcode with our value and len of 1. */
- if (is_zero && runlen == 1) {
- HLL_SPARSE_VAL_SET(p,count,1);
- goto updated;
- }
-
- /* D) General case.
- *
- * The other cases are more complex: our register requires to be updated
- * and is either currently represented by a VAL opcode with len > 1,
- * by a ZERO opcode with len > 1, or by an XZERO opcode.
- *
- * In those cases the original opcode must be split into multiple
- * opcodes. The worst case is an XZERO split in the middle resulting into
- * XZERO - VAL - XZERO, so the resulting sequence max length is
- * 5 bytes.
- *
- * We perform the split writing the new sequence into the 'new' buffer
- * with 'newlen' as length. Later the new sequence is inserted in place
- * of the old one, possibly moving what is on the right a few bytes
- * if the new sequence is longer than the older one. */
- uint8_t seq[5], *n = seq;
- int last = first+span-1; /* Last register covered by the sequence. */
- int len;
-
- if (is_zero || is_xzero) {
- /* Handle splitting of ZERO / XZERO. */
- if (index != first) {
- len = index-first;
- if (len > HLL_SPARSE_ZERO_MAX_LEN) {
- HLL_SPARSE_XZERO_SET(n,len);
- n += 2;
- } else {
- HLL_SPARSE_ZERO_SET(n,len);
- n++;
- }
- }
- HLL_SPARSE_VAL_SET(n,count,1);
- n++;
- if (index != last) {
- len = last-index;
- if (len > HLL_SPARSE_ZERO_MAX_LEN) {
- HLL_SPARSE_XZERO_SET(n,len);
- n += 2;
- } else {
- HLL_SPARSE_ZERO_SET(n,len);
- n++;
- }
- }
- } else {
- /* Handle splitting of VAL. */
- int curval = HLL_SPARSE_VAL_VALUE(p);
-
- if (index != first) {
- len = index-first;
- HLL_SPARSE_VAL_SET(n,curval,len);
- n++;
- }
- HLL_SPARSE_VAL_SET(n,count,1);
- n++;
- if (index != last) {
- len = last-index;
- HLL_SPARSE_VAL_SET(n,curval,len);
- n++;
- }
- }
-
- /* Step 3: substitute the new sequence with the old one.
- *
- * Note that we already allocated space on the sds string
- * calling sdsResize(). */
- int seqlen = n-seq;
- int oldlen = is_xzero ? 2 : 1;
- int deltalen = seqlen-oldlen;
-
- if (deltalen > 0 &&
- sdslen(o->ptr) + deltalen > server.hll_sparse_max_bytes) goto promote;
- serverAssert(sdslen(o->ptr) + deltalen <= sdsalloc(o->ptr));
- if (deltalen && next) memmove(next+deltalen,next,end-next);
- sdsIncrLen(o->ptr,deltalen);
- memcpy(p,seq,seqlen);
- end += deltalen;
-
-updated:
- /* Step 4: Merge adjacent values if possible.
- *
- * The representation was updated, however the resulting representation
- * may not be optimal: adjacent VAL opcodes can sometimes be merged into
- * a single one. */
- p = prev ? prev : sparse;
- int scanlen = 5; /* Scan up to 5 upcodes starting from prev. */
- while (p < end && scanlen--) {
- if (HLL_SPARSE_IS_XZERO(p)) {
- p += 2;
- continue;
- } else if (HLL_SPARSE_IS_ZERO(p)) {
- p++;
- continue;
- }
- /* We need two adjacent VAL opcodes to try a merge, having
- * the same value, and a len that fits the VAL opcode max len. */
- if (p+1 < end && HLL_SPARSE_IS_VAL(p+1)) {
- int v1 = HLL_SPARSE_VAL_VALUE(p);
- int v2 = HLL_SPARSE_VAL_VALUE(p+1);
- if (v1 == v2) {
- int len = HLL_SPARSE_VAL_LEN(p)+HLL_SPARSE_VAL_LEN(p+1);
- if (len <= HLL_SPARSE_VAL_MAX_LEN) {
- HLL_SPARSE_VAL_SET(p+1,v1,len);
- memmove(p,p+1,end-p);
- sdsIncrLen(o->ptr,-1);
- end--;
- /* After a merge we reiterate without incrementing 'p'
- * in order to try to merge the just merged value with
- * a value on its right. */
- continue;
- }
- }
- }
- p++;
- }
-
- /* Invalidate the cached cardinality. */
- hdr = o->ptr;
- HLL_INVALIDATE_CACHE(hdr);
- return 1;
-
-promote: /* Promote to dense representation. */
- if (hllSparseToDense(o) == C_ERR) return -1; /* Corrupted HLL. */
- hdr = o->ptr;
-
- /* We need to call hllDenseAdd() to perform the operation after the
- * conversion. However the result must be 1, since if we need to
- * convert from sparse to dense a register requires to be updated.
- *
- * Note that this in turn means that PFADD will make sure the command
- * is propagated to slaves / AOF, so if there is a sparse -> dense
- * conversion, it will be performed in all the slaves as well. */
- int dense_retval = hllDenseSet(hdr->registers,index,count);
- serverAssert(dense_retval == 1);
- return dense_retval;
-}
-
-/* "Add" the element in the sparse hyperloglog data structure.
- * Actually nothing is added, but the max 0 pattern counter of the subset
- * the element belongs to is incremented if needed.
- *
- * This function is actually a wrapper for hllSparseSet(), it only performs
- * the hashing of the element to obtain the index and zeros run length. */
-int hllSparseAdd(robj *o, unsigned char *ele, size_t elesize) {
- long index;
- uint8_t count = hllPatLen(ele,elesize,&index);
- /* Update the register if this element produced a longer run of zeroes. */
- return hllSparseSet(o,index,count);
-}
-
-/* Compute the register histogram in the sparse representation. */
-void hllSparseRegHisto(uint8_t *sparse, int sparselen, int *invalid, int* reghisto) {
- int idx = 0, runlen, regval;
- uint8_t *end = sparse+sparselen, *p = sparse;
- int valid = 1;
-
- while(p < end) {
- if (HLL_SPARSE_IS_ZERO(p)) {
- runlen = HLL_SPARSE_ZERO_LEN(p);
- if ((runlen + idx) > HLL_REGISTERS) { /* Overflow. */
- valid = 0;
- break;
- }
- idx += runlen;
- reghisto[0] += runlen;
- p++;
- } else if (HLL_SPARSE_IS_XZERO(p)) {
- runlen = HLL_SPARSE_XZERO_LEN(p);
- if ((runlen + idx) > HLL_REGISTERS) { /* Overflow. */
- valid = 0;
- break;
- }
- idx += runlen;
- reghisto[0] += runlen;
- p += 2;
- } else {
- runlen = HLL_SPARSE_VAL_LEN(p);
- regval = HLL_SPARSE_VAL_VALUE(p);
- if ((runlen + idx) > HLL_REGISTERS) { /* Overflow. */
- valid = 0;
- break;
- }
- idx += runlen;
- reghisto[regval] += runlen;
- p++;
- }
- }
- if ((!valid || idx != HLL_REGISTERS) && invalid) *invalid = 1;
-}
-
-/* ========================= HyperLogLog Count ==============================
- * This is the core of the algorithm where the approximated count is computed.
- * The function uses the lower level hllDenseRegHisto() and hllSparseRegHisto()
- * functions as helpers to compute histogram of register values part of the
- * computation, which is representation-specific, while all the rest is common. */
-
-/* Implements the register histogram calculation for uint8_t data type
- * which is only used internally as speedup for PFCOUNT with multiple keys. */
-void hllRawRegHisto(uint8_t *registers, int* reghisto) {
- uint64_t *word = (uint64_t*) registers;
- uint8_t *bytes;
- int j;
-
- for (j = 0; j < HLL_REGISTERS/8; j++) {
- if (*word == 0) {
- reghisto[0] += 8;
- } else {
- bytes = (uint8_t*) word;
- reghisto[bytes[0]]++;
- reghisto[bytes[1]]++;
- reghisto[bytes[2]]++;
- reghisto[bytes[3]]++;
- reghisto[bytes[4]]++;
- reghisto[bytes[5]]++;
- reghisto[bytes[6]]++;
- reghisto[bytes[7]]++;
- }
- word++;
- }
-}
-
-/* Helper function sigma as defined in
- * "New cardinality estimation algorithms for HyperLogLog sketches"
- * Otmar Ertl, arXiv:1702.01284 */
-double hllSigma(double x) {
- if (x == 1.) return INFINITY;
- double zPrime;
- double y = 1;
- double z = x;
- do {
- x *= x;
- zPrime = z;
- z += x * y;
- y += y;
- } while(zPrime != z);
- return z;
-}
-
-/* Helper function tau as defined in
- * "New cardinality estimation algorithms for HyperLogLog sketches"
- * Otmar Ertl, arXiv:1702.01284 */
-double hllTau(double x) {
- if (x == 0. || x == 1.) return 0.;
- double zPrime;
- double y = 1.0;
- double z = 1 - x;
- do {
- x = sqrt(x);
- zPrime = z;
- y *= 0.5;
- z -= pow(1 - x, 2)*y;
- } while(zPrime != z);
- return z / 3;
-}
-
-/* Return the approximated cardinality of the set based on the harmonic
- * mean of the registers values. 'hdr' points to the start of the SDS
- * representing the String object holding the HLL representation.
- *
- * If the sparse representation of the HLL object is not valid, the integer
- * pointed by 'invalid' is set to non-zero, otherwise it is left untouched.
- *
- * hllCount() supports a special internal-only encoding of HLL_RAW, that
- * is, hdr->registers will point to an uint8_t array of HLL_REGISTERS element.
- * This is useful in order to speedup PFCOUNT when called against multiple
- * keys (no need to work with 6-bit integers encoding). */
-uint64_t hllCount(struct hllhdr *hdr, int *invalid) {
- double m = HLL_REGISTERS;
- double E;
- int j;
- /* Note that reghisto size could be just HLL_Q+2, because HLL_Q+1 is
- * the maximum frequency of the "000...1" sequence the hash function is
- * able to return. However it is slow to check for sanity of the
- * input: instead we history array at a safe size: overflows will
- * just write data to wrong, but correctly allocated, places. */
- int reghisto[64] = {0};
-
- /* Compute register histogram */
- if (hdr->encoding == HLL_DENSE) {
- hllDenseRegHisto(hdr->registers,reghisto);
- } else if (hdr->encoding == HLL_SPARSE) {
- hllSparseRegHisto(hdr->registers,
- sdslen((sds)hdr)-HLL_HDR_SIZE,invalid,reghisto);
- } else if (hdr->encoding == HLL_RAW) {
- hllRawRegHisto(hdr->registers,reghisto);
- } else {
- serverPanic("Unknown HyperLogLog encoding in hllCount()");
- }
-
- /* Estimate cardinality from register histogram. See:
- * "New cardinality estimation algorithms for HyperLogLog sketches"
- * Otmar Ertl, arXiv:1702.01284 */
- double z = m * hllTau((m-reghisto[HLL_Q+1])/(double)m);
- for (j = HLL_Q; j >= 1; --j) {
- z += reghisto[j];
- z *= 0.5;
- }
- z += m * hllSigma(reghisto[0]/(double)m);
- E = llroundl(HLL_ALPHA_INF*m*m/z);
-
- return (uint64_t) E;
-}
-
-/* Call hllDenseAdd() or hllSparseAdd() according to the HLL encoding. */
-int hllAdd(robj *o, unsigned char *ele, size_t elesize) {
- struct hllhdr *hdr = o->ptr;
- switch(hdr->encoding) {
- case HLL_DENSE: return hllDenseAdd(hdr->registers,ele,elesize);
- case HLL_SPARSE: return hllSparseAdd(o,ele,elesize);
- default: return -1; /* Invalid representation. */
- }
-}
-
-#ifdef HAVE_AVX2
-/* A specialized version of hllMergeDense, optimized for default configurations.
- *
- * Requirements:
- * 1) HLL_REGISTERS == 16384 && HLL_BITS == 6
- * 2) The CPU supports AVX2 (checked at runtime in hllMergeDense)
- *
- * reg_raw: pointer to the raw representation array (16384 bytes, one byte per register)
- * reg_dense: pointer to the dense representation array (12288 bytes, 6 bits per register)
- */
-ATTRIBUTE_TARGET_AVX2
-void hllMergeDenseAVX2(uint8_t *reg_raw, const uint8_t *reg_dense) {
- const __m256i shuffle = _mm256_setr_epi8( //
- 4, 5, 6, -1, //
- 7, 8, 9, -1, //
- 10, 11, 12, -1, //
- 13, 14, 15, -1, //
- 0, 1, 2, -1, //
- 3, 4, 5, -1, //
- 6, 7, 8, -1, //
- 9, 10, 11, -1 //
- );
-
- /* Merge the first 8 registers (6 bytes) normally
- * as the AVX2 algorithm needs 4 padding bytes at the start */
- uint8_t val;
- for (int i = 0; i < 8; i++) {
- HLL_DENSE_GET_REGISTER(val, reg_dense, i);
- reg_raw[i] = MAX(reg_raw[i], val);
- }
-
- /* Dense to Raw:
- *
- * 4 registers in 3 bytes:
- * {bbaaaaaa|ccccbbbb|ddddddcc}
- *
- * LOAD 32 bytes (32 registers) per iteration:
- * 4(padding) + 12(16 registers) + 12(16 registers) + 4(padding)
- * {XXXX|AAAB|BBCC|CDDD|EEEF|FFGG|GHHH|XXXX}
- *
- * SHUFFLE to:
- * {AAA0|BBB0|CCC0|DDD0|EEE0|FFF0|GGG0|HHH0}
- * {bbaaaaaa|ccccbbbb|ddddddcc|00000000} x8
- *
- * AVX2 is little endian, each of the 8 groups is a little-endian int32.
- * A group (int32) contains 3 valid bytes (4 registers) and a zero byte.
- *
- * extract registers in each group with AND and SHIFT:
- * {00aaaaaa|00000000|00000000|00000000} x8 (<<0)
- * {00000000|00bbbbbb|00000000|00000000} x8 (<<2)
- * {00000000|00000000|00cccccc|00000000} x8 (<<4)
- * {00000000|00000000|00000000|00dddddd} x8 (<<6)
- *
- * merge the extracted registers with OR:
- * {00aaaaaa|00bbbbbb|00cccccc|00dddddd} x8
- *
- * Finally, compute MAX(reg_raw, merged) and STORE it back to reg_raw
- */
-
- /* Skip 8 registers (6 bytes) */
- const uint8_t *r = reg_dense + 6 - 4;
- uint8_t *t = reg_raw + 8;
-
- for (int i = 0; i < HLL_REGISTERS / 32 - 1; ++i) {
- __m256i x0, x;
- x0 = _mm256_loadu_si256((__m256i *)r);
- x = _mm256_shuffle_epi8(x0, shuffle);
-
- __m256i a1, a2, a3, a4;
- a1 = _mm256_and_si256(x, _mm256_set1_epi32(0x0000003f));
- a2 = _mm256_and_si256(x, _mm256_set1_epi32(0x00000fc0));
- a3 = _mm256_and_si256(x, _mm256_set1_epi32(0x0003f000));
- a4 = _mm256_and_si256(x, _mm256_set1_epi32(0x00fc0000));
-
- a2 = _mm256_slli_epi32(a2, 2);
- a3 = _mm256_slli_epi32(a3, 4);
- a4 = _mm256_slli_epi32(a4, 6);
-
- __m256i y1, y2, y;
- y1 = _mm256_or_si256(a1, a2);
- y2 = _mm256_or_si256(a3, a4);
- y = _mm256_or_si256(y1, y2);
-
- __m256i z = _mm256_loadu_si256((__m256i *)t);
-
- z = _mm256_max_epu8(z, y);
-
- _mm256_storeu_si256((__m256i *)t, z);
-
- r += 24;
- t += 32;
- }
-
- /* Merge the last 24 registers normally
- * as the AVX2 algorithm needs 4 padding bytes at the end */
- for (int i = HLL_REGISTERS - 24; i < HLL_REGISTERS; i++) {
- HLL_DENSE_GET_REGISTER(val, reg_dense, i);
- reg_raw[i] = MAX(reg_raw[i], val);
- }
-}
-#endif
-
-#ifdef HAVE_AARCH64_NEON
-/* A specialized version of hllMergeDense, optimized for default configurations.
- * Based on the AVX2 version.
- *
- * Requirements:
- * 1) HLL_REGISTERS == 16384 && HLL_BITS == 6
- * 2) Aarch64 CPU supports NEON (checked at runtime in hllMergeDense)
- *
- * reg_raw: pointer to the raw representation array (16384 bytes, one byte per register)
- * reg_dense: pointer to the dense representation array (12288 bytes, 6 bits per register)
- */
-void hllMergeDenseAarch64(uint8_t *reg_raw, const uint8_t *reg_dense) {
- const uint8_t *r = reg_dense;
- uint8_t *t = reg_raw;
-
- /* Shuffle pattern to expand each 12-byte packed group (16 regs x 6 bits)
- * to 16 bytes by inserting zeroes at bytes 3, 7, 11 and 15. */
- const uint8x16_t shuffle = {
- 0, 1, 2, -1,
- 3, 4, 5, -1,
- 6, 7, 8, -1,
- 9, 10, 11, -1
- };
-
- for (int i = 0; i < HLL_REGISTERS / 16 - 1; ++i) {
- /* Load 16 bytes (12 meaningful) and apply table; zeros fill pad positions. */
- uint8x16_t x, x0;
- x0 = vld1q_u8(r);
- x = vqtbl1q_u8(x0, shuffle);
-
- /* Treat as 4x32-bit lanes (LE); each lane now holds 3 packed bytes + one zero. */
- uint32x4_t x32 = vreinterpretq_u32_u8(x);
-
- /* Extract the four 6-bit fields per 32-bit lane. */
- uint32x4_t a1, a2, a3, a4;
- a1 = vandq_u32(x32, vdupq_n_u32(0x0000003f));
- a2 = vandq_u32(x32, vdupq_n_u32(0x00000fc0));
- a3 = vandq_u32(x32, vdupq_n_u32(0x0003f000));
- a4 = vandq_u32(x32, vdupq_n_u32(0x00fc0000));
-
- /* Align fields to byte boundaries within each lane. */
- a2 = vshlq_n_u32(a2, 2);
- a3 = vshlq_n_u32(a3, 4);
- a4 = vshlq_n_u32(a4, 6);
-
- /* Combine fields per lane (32-bit). */
- uint32x4_t y32 = vorrq_u32(vorrq_u32(a1, a2), vorrq_u32(a3, a4));
-
- /* Reinterpret to actual 8-bit uints for comparison. */
- uint8x16_t y = vreinterpretq_u8_u32(y32);
-
- /* Max-merge with existing raw registers. */
- uint8x16_t z = vld1q_u8(t);
- z = vmaxq_u8(z, y);
-
- /* Store the results. */
- vst1q_u8(t, z);
-
- r += 12;
- t += 16;
- }
-
- /* Process remaining registers, we do this manually because we don't want to over-read 4 bytes */
- uint8_t val;
- for (int i = HLL_REGISTERS - 16; i < HLL_REGISTERS; i++) {
- HLL_DENSE_GET_REGISTER(val, reg_dense, i);
- reg_raw[i] = MAX(reg_raw[i], val);
- }
-}
-#endif /* HAVE_AARCH64_NEON */
-
-/* Merge dense-encoded registers to raw registers array. */
-void hllMergeDense(uint8_t* reg_raw, const uint8_t* reg_dense) {
-#if HLL_REGISTERS == 16384 && HLL_BITS == 6
-#ifdef HAVE_AVX2
- if (HLL_USE_AVX2) {
- hllMergeDenseAVX2(reg_raw, reg_dense);
- return;
- }
-#endif
-#ifdef HAVE_AARCH64_NEON
- if (HLL_USE_NEON) {
- hllMergeDenseAarch64(reg_raw, reg_dense);
- return;
- }
-#endif
-#endif
-
- uint8_t val;
- for (int i = 0; i < HLL_REGISTERS; i++) {
- HLL_DENSE_GET_REGISTER(val, reg_dense, i);
- reg_raw[i] = MAX(reg_raw[i], val);
- }
-}
-
-/* Merge by computing MAX(registers[i],hll[i]) the HyperLogLog 'hll'
- * with an array of uint8_t HLL_REGISTERS registers pointed by 'max'.
- *
- * The hll object must be already validated via isHLLObjectOrReply()
- * or in some other way.
- *
- * If the HyperLogLog is sparse and is found to be invalid, C_ERR
- * is returned, otherwise the function always succeeds. */
-int hllMerge(uint8_t *max, robj *hll) {
- struct hllhdr *hdr = hll->ptr;
- int i;
-
- if (hdr->encoding == HLL_DENSE) {
- hllMergeDense(max, hdr->registers);
- } else {
- uint8_t *p = hll->ptr, *end = p + sdslen(hll->ptr);
- long runlen, regval;
- int valid = 1;
-
- p += HLL_HDR_SIZE;
- i = 0;
- while(p < end) {
- if (HLL_SPARSE_IS_ZERO(p)) {
- runlen = HLL_SPARSE_ZERO_LEN(p);
- if ((runlen + i) > HLL_REGISTERS) { /* Overflow. */
- valid = 0;
- break;
- }
- i += runlen;
- p++;
- } else if (HLL_SPARSE_IS_XZERO(p)) {
- runlen = HLL_SPARSE_XZERO_LEN(p);
- if ((runlen + i) > HLL_REGISTERS) { /* Overflow. */
- valid = 0;
- break;
- }
- i += runlen;
- p += 2;
- } else {
- runlen = HLL_SPARSE_VAL_LEN(p);
- regval = HLL_SPARSE_VAL_VALUE(p);
- if ((runlen + i) > HLL_REGISTERS) { /* Overflow. */
- valid = 0;
- break;
- }
- while(runlen--) {
- if (regval > max[i]) max[i] = regval;
- i++;
- }
- p++;
- }
- }
- if (!valid || i != HLL_REGISTERS) return C_ERR;
- }
- return C_OK;
-}
-
-#ifdef HAVE_AVX2
-/* A specialized version of hllDenseCompress, optimized for default configurations.
- *
- * Requirements:
- * 1) HLL_REGISTERS == 16384 && HLL_BITS == 6
- * 2) The CPU supports AVX2 (checked at runtime in hllDenseCompress)
- *
- * reg_dense: pointer to the dense representation array (12288 bytes, 6 bits per register)
- * reg_raw: pointer to the raw representation array (16384 bytes, one byte per register)
- */
-ATTRIBUTE_TARGET_AVX2
-void hllDenseCompressAVX2(uint8_t *reg_dense, const uint8_t *reg_raw) {
- const __m256i shuffle = _mm256_setr_epi8( //
- 0, 1, 2, //
- 4, 5, 6, //
- 8, 9, 10, //
- 12, 13, 14, //
- -1, -1, -1, -1, //
- 0, 1, 2, //
- 4, 5, 6, //
- 8, 9, 10, //
- 12, 13, 14, //
- -1, -1, -1, -1 //
- );
-
- /* Raw to Dense:
- *
- * LOAD 32 bytes (32 registers) per iteration:
- * {00aaaaaa|00bbbbbb|00cccccc|00dddddd} x8
- *
- * AVX2 is little endian, each of the 8 groups is a little-endian int32.
- * A group (int32) contains 4 registers.
- *
- * move the registers to correct positions with AND and SHIFT:
- * {00aaaaaa|00000000|00000000|00000000} x8 (>>0)
- * {bb000000|0000bbbb|00000000|00000000} x8 (>>2)
- * {00000000|cccc0000|000000cc|00000000} x8 (>>4)
- * {00000000|00000000|dddddd00|00000000} x8 (>>6)
- *
- * merge the registers with OR:
- * {bbaaaaaa|ccccbbbb|ddddddcc|00000000} x8
- * {AAA0|BBB0|CCC0|DDD0|EEE0|FFF0|GGG0|HHH0}
- *
- * SHUFFLE to:
- * {AAAB|BBCC|CDDD|0000|EEEF|FFGG|GHHH|0000}
- *
- * STORE the lower half and higher half respectively:
- * AAABBBCCCDDD0000
- * EEEFFFGGGHHH0000
- * AAABBBCCCDDDEEEFFFGGGHHH0000
- *
- * Note that the last 4 bytes are padding bytes.
- */
-
- const uint8_t *r = reg_raw;
- uint8_t *t = reg_dense;
-
- for (int i = 0; i < HLL_REGISTERS / 32 - 1; ++i) {
- __m256i x = _mm256_loadu_si256((__m256i *)r);
-
- __m256i a1, a2, a3, a4;
- a1 = _mm256_and_si256(x, _mm256_set1_epi32(0x0000003f));
- a2 = _mm256_and_si256(x, _mm256_set1_epi32(0x00003f00));
- a3 = _mm256_and_si256(x, _mm256_set1_epi32(0x003f0000));
- a4 = _mm256_and_si256(x, _mm256_set1_epi32(0x3f000000));
-
- a2 = _mm256_srli_epi32(a2, 2);
- a3 = _mm256_srli_epi32(a3, 4);
- a4 = _mm256_srli_epi32(a4, 6);
-
- __m256i y1, y2, y;
- y1 = _mm256_or_si256(a1, a2);
- y2 = _mm256_or_si256(a3, a4);
- y = _mm256_or_si256(y1, y2);
- y = _mm256_shuffle_epi8(y, shuffle);
-
- __m128i lower, higher;
- lower = _mm256_castsi256_si128(y);
- higher = _mm256_extracti128_si256(y, 1);
-
- _mm_storeu_si128((__m128i *)t, lower);
- _mm_storeu_si128((__m128i *)(t + 12), higher);
-
- r += 32;
- t += 24;
- }
-
- /* Merge the last 32 registers normally
- * as the AVX2 algorithm needs 4 padding bytes at the end */
- for (int i = HLL_REGISTERS - 32; i < HLL_REGISTERS; i++) {
- HLL_DENSE_SET_REGISTER(reg_dense, i, reg_raw[i]);
- }
-}
-#endif
-
-#ifdef HAVE_AARCH64_NEON
-/* A specialized version of hllDenseCompress, optimized for default configurations.
- * Based on the AVX2 version.
- *
- * Requirements:
- * 1) HLL_REGISTERS == 16384 && HLL_BITS == 6
- * 2) Aarch64 CPU supports NEON (checked at runtime in hllDenseCompress)
- *
- * reg_dense: pointer to the dense representation array (12288 bytes, 6 bits per register)
- * reg_raw: pointer to the raw representation array (16384 bytes, one byte per register)
- */
-void hllDenseCompressAarch64(uint8_t *reg_dense, const uint8_t *reg_raw) {
- const uint8_t *r = reg_raw;
- uint8_t *t = reg_dense;
-
- /* Shuffle pattern to collapse 16 raw bytes (16 regs x 8 bits)
- * into 12 bytes (16 regs x 6 bits) by dropping padding bytes 3, 7, 11, 15. */
- const uint8x16_t shuffle = {
- 0, 1, 2,
- 4, 5, 6,
- 8, 9, 10,
- 12, 13, 14,
- -1, -1, -1
- };
-
- for (int i = 0; i < HLL_REGISTERS / 16 - 1; ++i) {
- /* Load 16 raw registers as four 32-bit lanes (LE). */
- const uint32x4_t x = vld1q_u32((const uint32_t *)r);
-
- /* Extract the four 6-bit fields per 32-bit lane. */
- uint32x4_t a1, a2, a3, a4;
- a1 = vandq_u32(x, vdupq_n_u32(0x0000003f));
- a2 = vandq_u32(x, vdupq_n_u32(0x00003f00));
- a3 = vandq_u32(x, vdupq_n_u32(0x003f0000));
- a4 = vandq_u32(x, vdupq_n_u32(0x3f000000));
-
- /* Align fields to packed positions within each lane. */
- a2 = vshrq_n_u32(a2, 2);
- a3 = vshrq_n_u32(a3, 4);
- a4 = vshrq_n_u32(a4, 6);
-
- /* Combine fields per lane (32-bit). */
- uint32x4_t y32 = vorrq_u32(vorrq_u32(a1, a2), vorrq_u32(a3, a4));
-
- /* Reinterpret to 8-bit uints; each lane now holds 3 packed bytes + one pad. */
- uint8x16_t y = vreinterpretq_u8_u32(y32);
-
- /* Compact to 12 bytes by removing each lane's pad byte. */
- y = vqtbl1q_u8(y, shuffle);
-
- /* Store the results. */
- vst1q_u8(t, y);
-
- r += 16;
- t += 12;
- }
-
- /* Merge the last 16 registers normally
- * as the NEON algorithm needs 4 padding bytes at the end */
- for (int i = HLL_REGISTERS - 16; i < HLL_REGISTERS; i++) {
- HLL_DENSE_SET_REGISTER(reg_dense, i, reg_raw[i]);
- }
-}
-#endif
-
-/* Compress raw registers to dense representation. */
-void hllDenseCompress(uint8_t *reg_dense, const uint8_t *reg_raw) {
-#if HLL_REGISTERS == 16384 && HLL_BITS == 6
-#ifdef HAVE_AVX2
- if (HLL_USE_AVX2) {
- hllDenseCompressAVX2(reg_dense, reg_raw);
- return;
- }
-#endif
-
-#ifdef HAVE_AARCH64_NEON
- if (HLL_USE_NEON) {
- hllDenseCompressAarch64(reg_dense, reg_raw);
- return;
- }
-#endif
-#endif
-
- for (int i = 0; i < HLL_REGISTERS; i++) {
- HLL_DENSE_SET_REGISTER(reg_dense, i, reg_raw[i]);
- }
-}
-
-/* ========================== HyperLogLog commands ========================== */
-
-/* Create an HLL object. We always create the HLL using sparse encoding.
- * This will be upgraded to the dense representation as needed. */
-robj *createHLLObject(void) {
- robj *o;
- struct hllhdr *hdr;
- sds s;
- uint8_t *p;
- int sparselen = HLL_HDR_SIZE +
- (((HLL_REGISTERS+(HLL_SPARSE_XZERO_MAX_LEN-1)) /
- HLL_SPARSE_XZERO_MAX_LEN)*2);
- int aux;
-
- /* Populate the sparse representation with as many XZERO opcodes as
- * needed to represent all the registers. */
- aux = HLL_REGISTERS;
- s = sdsnewlen(NULL,sparselen);
- p = (uint8_t*)s + HLL_HDR_SIZE;
- while(aux) {
- int xzero = HLL_SPARSE_XZERO_MAX_LEN;
- if (xzero > aux) xzero = aux;
- HLL_SPARSE_XZERO_SET(p,xzero);
- p += 2;
- aux -= xzero;
- }
- serverAssert((p-(uint8_t*)s) == sparselen);
-
- /* Create the actual object. */
- o = createObject(OBJ_STRING,s);
- hdr = o->ptr;
- memcpy(hdr->magic,"HYLL",4);
- hdr->encoding = HLL_SPARSE;
- return o;
-}
-
-/* Check if the object is a String with a valid HLL representation.
- * Return C_OK if this is true, otherwise reply to the client
- * with an error and return C_ERR. */
-int isHLLObjectOrReply(client *c, robj *o) {
- struct hllhdr *hdr;
-
- /* Key exists, check type */
- if (checkType(c,o,OBJ_STRING))
- return C_ERR; /* Error already sent. */
-
- if (!sdsEncodedObject(o)) goto invalid;
- if (stringObjectLen(o) < sizeof(*hdr)) goto invalid;
- hdr = o->ptr;
-
- /* Magic should be "HYLL". */
- if (hdr->magic[0] != 'H' || hdr->magic[1] != 'Y' ||
- hdr->magic[2] != 'L' || hdr->magic[3] != 'L') goto invalid;
-
- if (hdr->encoding > HLL_MAX_ENCODING) goto invalid;
-
- /* Dense representation string length should match exactly. */
- if (hdr->encoding == HLL_DENSE &&
- stringObjectLen(o) != HLL_DENSE_SIZE) goto invalid;
-
- /* All tests passed. */
- return C_OK;
-
-invalid:
- addReplyError(c,"-WRONGTYPE Key is not a valid "
- "HyperLogLog string value.");
- return C_ERR;
-}
-
-/* PFADD var ele ele ele ... ele => :0 or :1 */
-void pfaddCommand(client *c) {
- uint64_t oldlen;
- dictEntryLink link;
- size_t oldsize = 0;
- kvobj *kv = lookupKeyWriteWithLink(c->db,c->argv[1], &link);
-
- struct hllhdr *hdr;
- int updated = 0, j;
-
- if (kv == NULL) {
- /* Create the key with a string value of the exact length to
- * hold our HLL data structure. sdsnewlen() when NULL is passed
- * is guaranteed to return bytes initialized to zero. */
- robj *o = createHLLObject();
- kv = dbAddByLink(c->db, c->argv[1], &o, &link);
- updated++;
- } else {
- if (isHLLObjectOrReply(c,kv) != C_OK) return;
- kv = dbUnshareStringValue(c->db,c->argv[1],kv);
- }
- oldlen = stringObjectLen(kv);
- if (server.memory_tracking_per_slot)
- oldsize = stringObjectAllocSize(kv);
-
- /* Perform the low level ADD operation for every element. */
- for (j = 2; j < c->argc; j++) {
- int retval = hllAdd(kv, (unsigned char*)c->argv[j]->ptr,
- sdslen(c->argv[j]->ptr));
- switch(retval) {
- case 1:
- updated++;
- break;
- case -1:
- addReplyError(c,invalid_hll_err);
- if (server.memory_tracking_per_slot)
- updateSlotAllocSize(c->db, getKeySlot(c->argv[1]->ptr), oldsize, stringObjectAllocSize(kv));
- return;
- }
- }
-
- hdr = kv->ptr;
- updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING, oldlen, stringObjectLen(kv));
- if (server.memory_tracking_per_slot)
- updateSlotAllocSize(c->db, getKeySlot(c->argv[1]->ptr), oldsize, stringObjectAllocSize(kv));
- if (updated) {
- HLL_INVALIDATE_CACHE(hdr);
- keyModified(c,c->db,c->argv[1],kv,1);
- notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id);
- server.dirty += updated;
- }
- addReply(c, updated ? shared.cone : shared.czero);
-}
-
-/* PFCOUNT var -> approximated cardinality of set. */
-void pfcountCommand(client *c) {
- struct hllhdr *hdr;
- uint64_t card;
-
- /* Case 1: multi-key keys, cardinality of the union.
- *
- * When multiple keys are specified, PFCOUNT actually computes
- * the cardinality of the merge of the N HLLs specified. */
- if (c->argc > 2) {
- uint8_t max[HLL_HDR_SIZE+HLL_REGISTERS], *registers;
- int j;
-
- /* Compute an HLL with M[i] = MAX(M[i]_j). */
- memset(max,0,sizeof(max));
- hdr = (struct hllhdr*) max;
- hdr->encoding = HLL_RAW; /* Special internal-only encoding. */
- registers = max + HLL_HDR_SIZE;
- for (j = 1; j < c->argc; j++) {
- /* Check type and size. */
- kvobj *o = lookupKeyRead(c->db,c->argv[j]);
- if (o == NULL) continue; /* Assume empty HLL for non existing var.*/
- if (isHLLObjectOrReply(c,o) != C_OK) return;
-
- /* Merge with this HLL with our 'max' HLL by setting max[i]
- * to MAX(max[i],hll[i]). */
- if (hllMerge(registers,o) == C_ERR) {
- addReplyError(c,invalid_hll_err);
- return;
- }
- }
-
- /* Compute cardinality of the resulting set. */
- addReplyLongLong(c,hllCount(hdr,NULL));
- return;
- }
-
- /* Case 2: cardinality of the single HLL.
- *
- * The user specified a single key. Either return the cached value
- * or compute one and update the cache.
- *
- * Since a HLL is a regular Redis string type value, updating the cache does
- * modify the value. We do a lookupKeyRead anyway since this is flagged as a
- * read-only command. The difference is that with lookupKeyWrite, a
- * logically expired key on a replica is deleted, while with lookupKeyRead
- * it isn't, but the lookup returns NULL either way if the key is logically
- * expired, which is what matters here. */
- kvobj *o = lookupKeyRead(c->db, c->argv[1]);
- if (o == NULL) {
- /* No key? Cardinality is zero since no element was added, otherwise
- * we would have a key as HLLADD creates it as a side effect. */
- addReply(c,shared.czero);
- } else {
- if (isHLLObjectOrReply(c,o) != C_OK) return;
- o = dbUnshareStringValue(c->db,c->argv[1],o);
-
- /* Check if the cached cardinality is valid. */
- hdr = o->ptr;
- if (HLL_VALID_CACHE(hdr)) {
- /* Just return the cached value. */
- card = (uint64_t)hdr->card[0];
- card |= (uint64_t)hdr->card[1] << 8;
- card |= (uint64_t)hdr->card[2] << 16;
- card |= (uint64_t)hdr->card[3] << 24;
- card |= (uint64_t)hdr->card[4] << 32;
- card |= (uint64_t)hdr->card[5] << 40;
- card |= (uint64_t)hdr->card[6] << 48;
- card |= (uint64_t)hdr->card[7] << 56;
- } else {
- int invalid = 0;
- /* Recompute it and update the cached value. */
- card = hllCount(hdr,&invalid);
- if (invalid) {
- addReplyError(c,invalid_hll_err);
- return;
- }
- hdr->card[0] = card & 0xff;
- hdr->card[1] = (card >> 8) & 0xff;
- hdr->card[2] = (card >> 16) & 0xff;
- hdr->card[3] = (card >> 24) & 0xff;
- hdr->card[4] = (card >> 32) & 0xff;
- hdr->card[5] = (card >> 40) & 0xff;
- hdr->card[6] = (card >> 48) & 0xff;
- hdr->card[7] = (card >> 56) & 0xff;
- /* This is considered a read-only command even if the cached value
- * may be modified and given that the HLL is a Redis string
- * we need to propagate the change. */
- keyModified(c,c->db,c->argv[1],o,1);
- server.dirty++;
- }
- addReplyLongLong(c,card);
- }
-}
-
-/* PFMERGE dest src1 src2 src3 ... srcN => OK */
-void pfmergeCommand(client *c) {
- uint8_t max[HLL_REGISTERS];
- struct hllhdr *hdr;
- int j;
- int use_dense = 0; /* Use dense representation as target? */
- size_t oldsize = 0;
-
- /* Compute an HLL with M[i] = MAX(M[i]_j).
- * We store the maximum into the max array of registers. We'll write
- * it to the target variable later. */
- memset(max,0,sizeof(max));
- for (j = 1; j < c->argc; j++) {
- /* Check type and size. */
- kvobj *o = lookupKeyRead(c->db, c->argv[j]);
- if (o == NULL) continue; /* Assume empty HLL for non existing var. */
- if (isHLLObjectOrReply(c, o) != C_OK) return;
-
- /* If at least one involved HLL is dense, use the dense representation
- * as target ASAP to save time and avoid the conversion step. */
- hdr = o->ptr;
- if (hdr->encoding == HLL_DENSE) use_dense = 1;
-
- /* Merge with this HLL with our 'max' HLL by setting max[i]
- * to MAX(max[i],hll[i]). */
- if (hllMerge(max,o) == C_ERR) {
- addReplyError(c,invalid_hll_err);
- return;
- }
- }
-
- /* Create / unshare the destination key's value if needed. */
- dictEntryLink link;
- kvobj *kv = lookupKeyWriteWithLink(c->db,c->argv[1],&link);
- if (kv == NULL) {
- /* Create the key with a string value of the exact length to
- * hold our HLL data structure. sdsnewlen() when NULL is passed
- * is guaranteed to return bytes initialized to zero. */
- robj *o = createHLLObject();
- kv = dbAddByLink(c->db, c->argv[1], &o, &link);
- } else {
- /* If key exists we are sure it's of the right type/size
- * since we checked when merging the different HLLs, so we
- * don't check again. */
- kv = dbUnshareStringValue(c->db,c->argv[1],kv);
- }
-
- uint64_t oldLen = stringObjectLen(kv);
- if (server.memory_tracking_per_slot)
- oldsize = stringObjectAllocSize(kv);
-
- /* Convert the destination object to dense representation if at least
- * one of the inputs was dense. */
- if (use_dense && hllSparseToDense(kv) == C_ERR) {
- addReplyError(c,invalid_hll_err);
- return;
- }
-
- /* Write the resulting HLL to the destination HLL registers and
- * invalidate the cached value. */
- if (use_dense) {
- hdr = kv->ptr;
- hllDenseCompress(hdr->registers, max);
- } else {
- for (j = 0; j < HLL_REGISTERS; j++) {
- if (max[j] == 0) continue;
- hdr = kv->ptr;
- switch (hdr->encoding) {
- case HLL_DENSE: hllDenseSet(hdr->registers,j,max[j]); break;
- case HLL_SPARSE: hllSparseSet(kv,j,max[j]); break;
- }
- }
- }
- hdr = kv->ptr; /* o->ptr may be different now, as a side effect of
- last hllSparseSet() call. */
- HLL_INVALIDATE_CACHE(hdr);
-
- if (server.memory_tracking_per_slot)
- updateSlotAllocSize(c->db, getKeySlot(c->argv[1]->ptr), oldsize, stringObjectAllocSize(kv));
- keyModified(c,c->db,c->argv[1],kv,1);
- /* We generate a PFADD event for PFMERGE for semantical simplicity
- * since in theory this is a mass-add of elements. */
- notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id);
-
- updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr),
- OBJ_STRING, oldLen, stringObjectLen(kv));
- server.dirty++;
- addReply(c,shared.ok);
-}
-
-/* ========================== Testing / Debugging ========================== */
-
-/* PFSELFTEST
- * This command performs a self-test of the HLL registers implementation.
- * Something that is not easy to test from within the outside. */
-#define HLL_TEST_CYCLES 1000
-void pfselftestCommand(client *c) {
- unsigned int j, i;
- sds bitcounters = sdsnewlen(NULL,HLL_DENSE_SIZE);
- struct hllhdr *hdr = (struct hllhdr*) bitcounters, *hdr2;
- robj *o = NULL;
- uint8_t bytecounters[HLL_REGISTERS];
-
- /* Test 1: access registers.
- * The test is conceived to test that the different counters of our data
- * structure are accessible and that setting their values both result in
- * the correct value to be retained and not affect adjacent values. */
- for (j = 0; j < HLL_TEST_CYCLES; j++) {
- /* Set the HLL counters and an array of unsigned byes of the
- * same size to the same set of random values. */
- for (i = 0; i < HLL_REGISTERS; i++) {
- unsigned int r = rand() & HLL_REGISTER_MAX;
-
- bytecounters[i] = r;
- HLL_DENSE_SET_REGISTER(hdr->registers,i,r);
- }
- /* Check that we are able to retrieve the same values. */
- for (i = 0; i < HLL_REGISTERS; i++) {
- unsigned int val;
-
- HLL_DENSE_GET_REGISTER(val,hdr->registers,i);
- if (val != bytecounters[i]) {
- addReplyErrorFormat(c,
- "TESTFAILED Register %d should be %d but is %d",
- i, (int) bytecounters[i], (int) val);
- goto cleanup;
- }
- }
- }
-
- /* Test 2: approximation error.
- * The test adds unique elements and check that the estimated value
- * is always reasonable bounds.
- *
- * We check that the error is smaller than a few times than the expected
- * standard error, to make it very unlikely for the test to fail because
- * of a "bad" run.
- *
- * The test is performed with both dense and sparse HLLs at the same
- * time also verifying that the computed cardinality is the same. */
- memset(hdr->registers,0,HLL_DENSE_SIZE-HLL_HDR_SIZE);
- o = createHLLObject();
- double relerr = 1.04/sqrt(HLL_REGISTERS);
- int64_t checkpoint = 1;
- uint64_t seed = (uint64_t)rand() | (uint64_t)rand() << 32;
- uint64_t ele;
- for (j = 1; j <= 10000000; j++) {
- ele = j ^ seed;
- hllDenseAdd(hdr->registers,(unsigned char*)&ele,sizeof(ele));
- hllAdd(o,(unsigned char*)&ele,sizeof(ele));
-
- /* Make sure that for small cardinalities we use sparse
- * encoding. */
- if (j == checkpoint && j < server.hll_sparse_max_bytes/2) {
- hdr2 = o->ptr;
- if (hdr2->encoding != HLL_SPARSE) {
- addReplyError(c, "TESTFAILED sparse encoding not used");
- goto cleanup;
- }
- }
-
- /* Check that dense and sparse representations agree. */
- if (j == checkpoint && hllCount(hdr,NULL) != hllCount(o->ptr,NULL)) {
- addReplyError(c, "TESTFAILED dense/sparse disagree");
- goto cleanup;
- }
-
- /* Check error. */
- if (j == checkpoint) {
- int64_t abserr = checkpoint - (int64_t)hllCount(hdr,NULL);
- uint64_t maxerr = ceil(relerr*6*checkpoint);
-
- /* Adjust the max error we expect for cardinality 10
- * since from time to time it is statistically likely to get
- * much higher error due to collision, resulting into a false
- * positive. */
- if (j == 10) maxerr = 1;
-
- if (abserr < 0) abserr = -abserr;
- if (abserr > (int64_t)maxerr) {
- addReplyErrorFormat(c,
- "TESTFAILED Too big error. card:%llu abserr:%llu",
- (unsigned long long) checkpoint,
- (unsigned long long) abserr);
- goto cleanup;
- }
- checkpoint *= 10;
- }
- }
-
- /* Success! */
- addReply(c,shared.ok);
-
-cleanup:
- sdsfree(bitcounters);
- if (o) decrRefCount(o);
-}
-
-/* Different debugging related operations about the HLL implementation.
- *
- * PFDEBUG GETREG <key>
- * PFDEBUG DECODE <key>
- * PFDEBUG ENCODING <key>
- * PFDEBUG TODENSE <key>
- * PFDEBUG SIMD (ON|OFF)
- */
-void pfdebugCommand(client *c) {
- char *cmd = c->argv[1]->ptr;
- struct hllhdr *hdr;
- kvobj *o;
- int j;
- size_t oldsize = 0;
-
- if (!strcasecmp(cmd, "simd")) {
- if (c->argc != 3) goto arityerr;
-
- if (!strcasecmp(c->argv[2]->ptr, "on")) {
-#if defined(HAVE_AVX2) || defined(HAVE_AARCH64_NEON)
- simd_enabled = 1;
-#endif
- } else if (!strcasecmp(c->argv[2]->ptr, "off")) {
-#if defined(HAVE_AVX2) || defined(HAVE_AARCH64_NEON)
- simd_enabled = 0;
-#endif
- } else {
- addReplyError(c, "Argument must be ON or OFF");
- }
-
- addReplyStatus(c, HLL_USE_AVX2 || HLL_USE_NEON ? "enabled" : "disabled");
-
- return;
- }
-
- o = lookupKeyWrite(c->db,c->argv[2]);
- if (o == NULL) {
- addReplyError(c,"The specified key does not exist");
- return;
- }
- if (isHLLObjectOrReply(c,o) != C_OK) return;
- o = dbUnshareStringValue(c->db,c->argv[2],o);
- hdr = o->ptr;
- if (server.memory_tracking_per_slot)
- oldsize = stringObjectAllocSize(o);
-
- /* PFDEBUG GETREG <key> */
- if (!strcasecmp(cmd,"getreg")) {
- if (c->argc != 3) goto arityerr;
-
- if (hdr->encoding == HLL_SPARSE) {
- uint64_t oldlen = (uint64_t) stringObjectLen(o);
- if (hllSparseToDense(o) == C_ERR) {
- addReplyError(c,invalid_hll_err);
- return;
- }
- updateKeysizesHist(c->db, getKeySlot(c->argv[2]->ptr), OBJ_STRING, oldlen, stringObjectLen(o));
- if (server.memory_tracking_per_slot)
- updateSlotAllocSize(c->db, getKeySlot(c->argv[2]->ptr), oldsize, stringObjectAllocSize(o));
- server.dirty++; /* Force propagation on encoding change. */
- }
-
- hdr = o->ptr;
- addReplyArrayLen(c,HLL_REGISTERS);
- for (j = 0; j < HLL_REGISTERS; j++) {
- uint8_t val;
-
- HLL_DENSE_GET_REGISTER(val,hdr->registers,j);
- addReplyLongLong(c,val);
- }
- }
- /* PFDEBUG DECODE <key> */
- else if (!strcasecmp(cmd,"decode")) {
- if (c->argc != 3) goto arityerr;
-
- uint8_t *p = o->ptr, *end = p+sdslen(o->ptr);
- sds decoded = sdsempty();
-
- if (hdr->encoding != HLL_SPARSE) {
- sdsfree(decoded);
- addReplyError(c,"HLL encoding is not sparse");
- return;
- }
-
- p += HLL_HDR_SIZE;
- while(p < end) {
- int runlen, regval;
-
- if (HLL_SPARSE_IS_ZERO(p)) {
- runlen = HLL_SPARSE_ZERO_LEN(p);
- p++;
- decoded = sdscatprintf(decoded,"z:%d ",runlen);
- } else if (HLL_SPARSE_IS_XZERO(p)) {
- runlen = HLL_SPARSE_XZERO_LEN(p);
- p += 2;
- decoded = sdscatprintf(decoded,"Z:%d ",runlen);
- } else {
- runlen = HLL_SPARSE_VAL_LEN(p);
- regval = HLL_SPARSE_VAL_VALUE(p);
- p++;
- decoded = sdscatprintf(decoded,"v:%d,%d ",regval,runlen);
- }
- }
- decoded = sdstrim(decoded," ");
- addReplyBulkCBuffer(c,decoded,sdslen(decoded));
- sdsfree(decoded);
- }
- /* PFDEBUG ENCODING <key> */
- else if (!strcasecmp(cmd,"encoding")) {
- char *encodingstr[2] = {"dense","sparse"};
- if (c->argc != 3) goto arityerr;
-
- addReplyStatus(c,encodingstr[hdr->encoding]);
- }
- /* PFDEBUG TODENSE <key> */
- else if (!strcasecmp(cmd,"todense")) {
- int conv = 0;
- if (c->argc != 3) goto arityerr;
-
- if (hdr->encoding == HLL_SPARSE) {
- int64_t oldlen = (int64_t) stringObjectLen(o);
- if (hllSparseToDense(o) == C_ERR) {
- addReplyError(c,invalid_hll_err);
- return;
- }
- updateKeysizesHist(c->db, getKeySlot(c->argv[2]->ptr), OBJ_STRING, oldlen, stringObjectLen(o));
- if (server.memory_tracking_per_slot)
- updateSlotAllocSize(c->db, getKeySlot(c->argv[2]->ptr), oldsize, stringObjectAllocSize(o));
- conv = 1;
- server.dirty++; /* Force propagation on encoding change. */
- }
- addReply(c,conv ? shared.cone : shared.czero);
- } else {
- addReplyErrorFormat(c,"Unknown PFDEBUG subcommand '%s'", cmd);
- }
- return;
-
-arityerr:
- addReplyErrorFormat(c,
- "Wrong number of arguments for the '%s' subcommand",cmd);
-}
-