aboutsummaryrefslogtreecommitdiff
path: root/examples/redis-unstable/utils
diff options
context:
space:
mode:
authorMitja Felicijan <mitja.felicijan@gmail.com>2026-01-21 22:40:55 +0100
committerMitja Felicijan <mitja.felicijan@gmail.com>2026-01-21 22:40:55 +0100
commit5d8dfe892a2ea89f706ee140c3bdcfd89fe03fda (patch)
tree1acdfa5220cd13b7be43a2a01368e80d306473ca /examples/redis-unstable/utils
parentc7ab12bba64d9c20ccd79b132dac475f7bc3923e (diff)
downloadcrep-5d8dfe892a2ea89f706ee140c3bdcfd89fe03fda.tar.gz
Add Redis source code for testing
Diffstat (limited to 'examples/redis-unstable/utils')
-rw-r--r--examples/redis-unstable/utils/build-static-symbols.tcl26
-rw-r--r--examples/redis-unstable/utils/cluster_fail_time.tcl50
-rw-r--r--examples/redis-unstable/utils/corrupt_rdb.c49
-rw-r--r--examples/redis-unstable/utils/create-cluster/.gitignore6
-rw-r--r--examples/redis-unstable/utils/create-cluster/README27
-rwxr-xr-xexamples/redis-unstable/utils/create-cluster/create-cluster143
-rwxr-xr-xexamples/redis-unstable/utils/gen-test-certs.sh58
-rwxr-xr-xexamples/redis-unstable/utils/generate-command-code.py629
-rwxr-xr-xexamples/redis-unstable/utils/generate-commands-json.py148
-rwxr-xr-xexamples/redis-unstable/utils/generate-fmtargs.py22
-rwxr-xr-xexamples/redis-unstable/utils/generate-module-api-doc.rb195
-rw-r--r--examples/redis-unstable/utils/graphs/commits-over-time/README.md16
-rwxr-xr-xexamples/redis-unstable/utils/graphs/commits-over-time/genhtml.tcl96
-rw-r--r--examples/redis-unstable/utils/hyperloglog/.gitignore1
-rw-r--r--examples/redis-unstable/utils/hyperloglog/hll-err.rb30
-rw-r--r--examples/redis-unstable/utils/hyperloglog/hll-gnuplot-graph.rb91
-rwxr-xr-xexamples/redis-unstable/utils/install_server.sh291
-rw-r--r--examples/redis-unstable/utils/lru/README19
-rw-r--r--examples/redis-unstable/utils/lru/lfu-simulation.c158
-rw-r--r--examples/redis-unstable/utils/lru/test-lru.rb223
-rw-r--r--examples/redis-unstable/utils/redis-copy.rb38
-rw-r--r--examples/redis-unstable/utils/redis-sha1.rb55
-rwxr-xr-xexamples/redis-unstable/utils/redis_init_script50
-rwxr-xr-xexamples/redis-unstable/utils/redis_init_script.tpl44
-rwxr-xr-xexamples/redis-unstable/utils/releasetools/01_create_tarball.sh14
-rwxr-xr-xexamples/redis-unstable/utils/releasetools/02_upload_tarball.sh23
-rwxr-xr-xexamples/redis-unstable/utils/releasetools/03_test_release.sh28
-rwxr-xr-xexamples/redis-unstable/utils/releasetools/04_release_hash.sh13
-rwxr-xr-xexamples/redis-unstable/utils/releasetools/changelog.tcl35
-rw-r--r--examples/redis-unstable/utils/reply_schema_linter.js31
-rwxr-xr-xexamples/redis-unstable/utils/req-res-log-validator.py364
-rw-r--r--examples/redis-unstable/utils/req-res-validator/requirements.txt2
-rwxr-xr-xexamples/redis-unstable/utils/speed-regression.tcl133
-rw-r--r--examples/redis-unstable/utils/srandmember/README.md14
-rw-r--r--examples/redis-unstable/utils/srandmember/showdist.rb33
-rw-r--r--examples/redis-unstable/utils/srandmember/showfreq.rb23
-rw-r--r--examples/redis-unstable/utils/systemd-redis_multiple_servers@.service37
-rw-r--r--examples/redis-unstable/utils/systemd-redis_server.service43
-rw-r--r--examples/redis-unstable/utils/tracking_collisions.c79
-rwxr-xr-xexamples/redis-unstable/utils/whatisdoing.sh24
40 files changed, 3361 insertions, 0 deletions
diff --git a/examples/redis-unstable/utils/build-static-symbols.tcl b/examples/redis-unstable/utils/build-static-symbols.tcl
new file mode 100644
index 0000000..eb0e218
--- /dev/null
+++ b/examples/redis-unstable/utils/build-static-symbols.tcl
@@ -0,0 +1,26 @@
1# Build a symbol table for static symbols of redis.c
2# Useful to get stack traces on segfault without a debugger. See redis.c
3# for more information.
4#
5# Copyright(C) 2009-Present Redis Ltd. All rights reserved.
6#
7# Licensed under your choice of (a) the Redis Source Available License 2.0
8# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
9# GNU Affero General Public License v3 (AGPLv3).
10
11set fd [open redis.c]
12set symlist {}
13while {[gets $fd line] != -1} {
14 if {[regexp {^static +[A-z0-9]+[ *]+([A-z0-9]*)\(} $line - sym]} {
15 lappend symlist $sym
16 }
17}
18set symlist [lsort -unique $symlist]
19puts "static struct redisFunctionSym symsTable\[\] = {"
20foreach sym $symlist {
21 puts "{\"$sym\",(unsigned long)$sym},"
22}
23puts "{NULL,0}"
24puts "};"
25
26close $fd
diff --git a/examples/redis-unstable/utils/cluster_fail_time.tcl b/examples/redis-unstable/utils/cluster_fail_time.tcl
new file mode 100644
index 0000000..8739949
--- /dev/null
+++ b/examples/redis-unstable/utils/cluster_fail_time.tcl
@@ -0,0 +1,50 @@
1# This simple script is used in order to estimate the average PFAIL->FAIL
2# state switch after a failure.
3
4set ::sleep_time 10 ; # How much to sleep to trigger PFAIL.
5set ::fail_port 30016 ; # Node to put in sleep.
6set ::other_port 30001 ; # Node to use to monitor the flag switch.
7
8proc avg vector {
9 set sum 0.0
10 foreach x $vector {
11 set sum [expr {$sum+$x}]
12 }
13 expr {$sum/[llength $vector]}
14}
15
16set samples {}
17while 1 {
18 exec redis-cli -p $::fail_port debug sleep $::sleep_time > /dev/null &
19
20 # Wait for fail? to appear.
21 while 1 {
22 set output [exec redis-cli -p $::other_port cluster nodes]
23 if {[string match {*fail\?*} $output]} break
24 after 100
25 }
26
27 puts "FAIL?"
28 set start [clock milliseconds]
29
30 # Wait for fail? to disappear.
31 while 1 {
32 set output [exec redis-cli -p $::other_port cluster nodes]
33 if {![string match {*fail\?*} $output]} break
34 after 100
35 }
36
37 puts "FAIL"
38 set now [clock milliseconds]
39 set elapsed [expr {$now-$start}]
40 puts $elapsed
41 lappend samples $elapsed
42
43 puts "AVG([llength $samples]): [avg $samples]"
44
45 # Wait for the instance to be available again.
46 exec redis-cli -p $::fail_port ping
47
48 # Wait for the fail flag to be cleared.
49 after 2000
50}
diff --git a/examples/redis-unstable/utils/corrupt_rdb.c b/examples/redis-unstable/utils/corrupt_rdb.c
new file mode 100644
index 0000000..d1a093c
--- /dev/null
+++ b/examples/redis-unstable/utils/corrupt_rdb.c
@@ -0,0 +1,49 @@
1/* Trivia program to corrupt an RDB file in order to check the RDB check
2 * program behavior and effectiveness.
3 *
4 * Copyright (C) 2016-Present Redis Ltd. All rights reserved.
5 *
6 * Licensed under your choice of (a) the Redis Source Available License 2.0
7 * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
8 * GNU Affero General Public License v3 (AGPLv3).
9 */
10
11#include <stdio.h>
12#include <fcntl.h>
13#include <sys/stat.h>
14#include <stdlib.h>
15#include <unistd.h>
16#include <time.h>
17
18int main(int argc, char **argv) {
19 struct stat stat;
20 int fd, cycles;
21
22 if (argc != 3) {
23 fprintf(stderr,"Usage: <filename> <cycles>\n");
24 exit(1);
25 }
26
27 srand(time(NULL));
28 char *filename = argv[1];
29 cycles = atoi(argv[2]);
30 fd = open(filename,O_RDWR);
31 if (fd == -1) {
32 perror("open");
33 exit(1);
34 }
35 fstat(fd,&stat);
36
37 while(cycles--) {
38 unsigned char buf[32];
39 unsigned long offset = rand()%stat.st_size;
40 int writelen = 1+rand()%31;
41 int j;
42
43 for (j = 0; j < writelen; j++) buf[j] = (char)rand();
44 lseek(fd,offset,SEEK_SET);
45 printf("Writing %d bytes at offset %lu\n", writelen, offset);
46 write(fd,buf,writelen);
47 }
48 return 0;
49}
diff --git a/examples/redis-unstable/utils/create-cluster/.gitignore b/examples/redis-unstable/utils/create-cluster/.gitignore
new file mode 100644
index 0000000..a34b639
--- /dev/null
+++ b/examples/redis-unstable/utils/create-cluster/.gitignore
@@ -0,0 +1,6 @@
1config.sh
2*.rdb
3*.aof
4*.conf
5*.log
6appendonlydir-*
diff --git a/examples/redis-unstable/utils/create-cluster/README b/examples/redis-unstable/utils/create-cluster/README
new file mode 100644
index 0000000..bcd7459
--- /dev/null
+++ b/examples/redis-unstable/utils/create-cluster/README
@@ -0,0 +1,27 @@
1create-cluster is a small script used to easily start a big number of Redis
2instances configured to run in cluster mode. Its main goal is to allow manual
3testing in a condition which is not easy to replicate with the Redis cluster
4unit tests, for example when a lot of instances are needed in order to trigger
5a given bug.
6
7The tool can also be used just to easily create a number of instances in a
8Redis Cluster in order to experiment a bit with the system.
9
10USAGE
11---
12
13To create a cluster, follow these steps:
14
151. Edit create-cluster and change the start / end port, depending on the
16number of instances you want to create.
172. Use "./create-cluster start" in order to run the instances.
183. Use "./create-cluster create" in order to execute redis-cli --cluster create, so that
19an actual Redis cluster will be created. (If you're accessing your setup via a local container, ensure that the CLUSTER_HOST value is changed to your local IP)
204. Now you are ready to play with the cluster. AOF files and logs for each instances are created in the current directory.
21
22In order to stop a cluster:
23
241. Use "./create-cluster stop" to stop all the instances. After you stopped the instances you can use "./create-cluster start" to restart them if you change your mind.
252. Use "./create-cluster clean" to remove all the AOF / log files to restart with a clean environment.
26
27Use the command "./create-cluster help" to get the full list of features.
diff --git a/examples/redis-unstable/utils/create-cluster/create-cluster b/examples/redis-unstable/utils/create-cluster/create-cluster
new file mode 100755
index 0000000..dd20083
--- /dev/null
+++ b/examples/redis-unstable/utils/create-cluster/create-cluster
@@ -0,0 +1,143 @@
1#!/bin/bash
2
3SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
4
5# Settings
6BIN_PATH="$SCRIPT_DIR/../../src/"
7CLUSTER_HOST=127.0.0.1
8PORT=30000
9TIMEOUT=2000
10NODES=6
11REPLICAS=1
12PROTECTED_MODE=yes
13ADDITIONAL_OPTIONS=""
14
15# You may want to put the above config parameters into config.sh in order to
16# override the defaults without modifying this script.
17
18if [ -a config.sh ]
19then
20 source "config.sh"
21fi
22
23# Computed vars
24ENDPORT=$((PORT+NODES))
25
26if [ "$1" == "start" ]
27then
28 while [ $((PORT < ENDPORT)) != "0" ]; do
29 PORT=$((PORT+1))
30 echo "Starting $PORT"
31 $BIN_PATH/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --appenddirname appendonlydir-${PORT} --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS}
32 done
33 exit 0
34fi
35
36if [ "$1" == "create" ]
37then
38 HOSTS=""
39 while [ $((PORT < ENDPORT)) != "0" ]; do
40 PORT=$((PORT+1))
41 HOSTS="$HOSTS $CLUSTER_HOST:$PORT"
42 done
43 OPT_ARG=""
44 if [ "$2" == "-f" ]; then
45 OPT_ARG="--cluster-yes"
46 fi
47 $BIN_PATH/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS $OPT_ARG
48 exit 0
49fi
50
51if [ "$1" == "stop" ]
52then
53 while [ $((PORT < ENDPORT)) != "0" ]; do
54 PORT=$((PORT+1))
55 echo "Stopping $PORT"
56 $BIN_PATH/redis-cli -p $PORT shutdown nosave
57 done
58 exit 0
59fi
60
61if [ "$1" == "restart" ]
62then
63 OLD_PORT=$PORT
64 while [ $((PORT < ENDPORT)) != "0" ]; do
65 PORT=$((PORT+1))
66 echo "Stopping $PORT"
67 $BIN_PATH/redis-cli -p $PORT shutdown nosave
68 done
69 PORT=$OLD_PORT
70 while [ $((PORT < ENDPORT)) != "0" ]; do
71 PORT=$((PORT+1))
72 echo "Starting $PORT"
73 $BIN_PATH/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --appenddirname appendonlydir-${PORT} --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS}
74 done
75 exit 0
76fi
77
78if [ "$1" == "watch" ]
79then
80 PORT=$((PORT+1))
81 while [ 1 ]; do
82 clear
83 date
84 $BIN_PATH/redis-cli -p $PORT cluster nodes | head -30
85 sleep 1
86 done
87 exit 0
88fi
89
90if [ "$1" == "tail" ]
91then
92 INSTANCE=$2
93 PORT=$((PORT+INSTANCE))
94 tail -f ${PORT}.log
95 exit 0
96fi
97
98if [ "$1" == "tailall" ]
99then
100 tail -f *.log
101 exit 0
102fi
103
104if [ "$1" == "call" ]
105then
106 while [ $((PORT < ENDPORT)) != "0" ]; do
107 PORT=$((PORT+1))
108 $BIN_PATH/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9
109 done
110 exit 0
111fi
112
113if [ "$1" == "clean" ]
114then
115 echo "Cleaning *.log"
116 rm -rf *.log
117 echo "Cleaning appendonlydir-*"
118 rm -rf appendonlydir-*
119 echo "Cleaning dump-*.rdb"
120 rm -rf dump-*.rdb
121 echo "Cleaning nodes-*.conf"
122 rm -rf nodes-*.conf
123 exit 0
124fi
125
126if [ "$1" == "clean-logs" ]
127then
128 echo "Cleaning *.log"
129 rm -rf *.log
130 exit 0
131fi
132
133echo "Usage: $0 [start|create|stop|restart|watch|tail|tailall|clean|clean-logs|call]"
134echo "start -- Launch Redis Cluster instances."
135echo "create [-f] -- Create a cluster using redis-cli --cluster create."
136echo "stop -- Stop Redis Cluster instances."
137echo "restart -- Restart Redis Cluster instances."
138echo "watch -- Show CLUSTER NODES output (first 30 lines) of first node."
139echo "tail <id> -- Run tail -f of instance at base port + ID."
140echo "tailall -- Run tail -f for all the log files at once."
141echo "clean -- Remove all instances data, logs, configs."
142echo "clean-logs -- Remove just instances logs."
143echo "call <cmd> -- Call a command (up to 7 arguments) on all nodes."
diff --git a/examples/redis-unstable/utils/gen-test-certs.sh b/examples/redis-unstable/utils/gen-test-certs.sh
new file mode 100755
index 0000000..6bc9d86
--- /dev/null
+++ b/examples/redis-unstable/utils/gen-test-certs.sh
@@ -0,0 +1,58 @@
1#!/bin/bash
2
3# Generate some test certificates which are used by the regression test suite:
4#
5# tests/tls/ca.{crt,key} Self signed CA certificate.
6# tests/tls/redis.{crt,key} A certificate with no key usage/policy restrictions.
7# tests/tls/client.{crt,key} A certificate restricted for SSL client usage.
8# tests/tls/server.{crt,key} A certificate restricted for SSL server usage.
9# tests/tls/redis.dh DH Params file.
10
11generate_cert() {
12 local name=$1
13 local cn="$2"
14 local opts="$3"
15
16 local keyfile=tests/tls/${name}.key
17 local certfile=tests/tls/${name}.crt
18
19 [ -f $keyfile ] || openssl genrsa -out $keyfile 2048
20 openssl req \
21 -new -sha256 \
22 -subj "/O=Redis Test/CN=$cn" \
23 -key $keyfile | \
24 openssl x509 \
25 -req -sha256 \
26 -CA tests/tls/ca.crt \
27 -CAkey tests/tls/ca.key \
28 -CAserial tests/tls/ca.txt \
29 -CAcreateserial \
30 -days 365 \
31 $opts \
32 -out $certfile
33}
34
35mkdir -p tests/tls
36[ -f tests/tls/ca.key ] || openssl genrsa -out tests/tls/ca.key 4096
37openssl req \
38 -x509 -new -nodes -sha256 \
39 -key tests/tls/ca.key \
40 -days 3650 \
41 -subj '/O=Redis Test/CN=Certificate Authority' \
42 -out tests/tls/ca.crt
43
44cat > tests/tls/openssl.cnf <<_END_
45[ server_cert ]
46keyUsage = digitalSignature, keyEncipherment
47nsCertType = server
48
49[ client_cert ]
50keyUsage = digitalSignature, keyEncipherment
51nsCertType = client
52_END_
53
54generate_cert server "Server-only" "-extfile tests/tls/openssl.cnf -extensions server_cert"
55generate_cert client "Client-only" "-extfile tests/tls/openssl.cnf -extensions client_cert"
56generate_cert redis "Generic-cert"
57
58[ -f tests/tls/redis.dh ] || openssl dhparam -out tests/tls/redis.dh 2048
diff --git a/examples/redis-unstable/utils/generate-command-code.py b/examples/redis-unstable/utils/generate-command-code.py
new file mode 100755
index 0000000..76c8c3b
--- /dev/null
+++ b/examples/redis-unstable/utils/generate-command-code.py
@@ -0,0 +1,629 @@
1#!/usr/bin/env python3
2import glob
3import json
4import os
5import argparse
6
7ARG_TYPES = {
8 "string": "ARG_TYPE_STRING",
9 "integer": "ARG_TYPE_INTEGER",
10 "double": "ARG_TYPE_DOUBLE",
11 "key": "ARG_TYPE_KEY",
12 "pattern": "ARG_TYPE_PATTERN",
13 "unix-time": "ARG_TYPE_UNIX_TIME",
14 "pure-token": "ARG_TYPE_PURE_TOKEN",
15 "oneof": "ARG_TYPE_ONEOF",
16 "block": "ARG_TYPE_BLOCK",
17}
18
19GROUPS = {
20 "generic": "COMMAND_GROUP_GENERIC",
21 "string": "COMMAND_GROUP_STRING",
22 "list": "COMMAND_GROUP_LIST",
23 "set": "COMMAND_GROUP_SET",
24 "sorted_set": "COMMAND_GROUP_SORTED_SET",
25 "hash": "COMMAND_GROUP_HASH",
26 "pubsub": "COMMAND_GROUP_PUBSUB",
27 "transactions": "COMMAND_GROUP_TRANSACTIONS",
28 "connection": "COMMAND_GROUP_CONNECTION",
29 "server": "COMMAND_GROUP_SERVER",
30 "scripting": "COMMAND_GROUP_SCRIPTING",
31 "hyperloglog": "COMMAND_GROUP_HYPERLOGLOG",
32 "cluster": "COMMAND_GROUP_CLUSTER",
33 "sentinel": "COMMAND_GROUP_SENTINEL",
34 "geo": "COMMAND_GROUP_GEO",
35 "stream": "COMMAND_GROUP_STREAM",
36 "bitmap": "COMMAND_GROUP_BITMAP",
37}
38
39
40def get_optional_desc_string(desc, field, force_uppercase=False):
41 v = desc.get(field, None)
42 if v and force_uppercase:
43 v = v.upper()
44 ret = "\"%s\"" % v if v else "NULL"
45 return ret.replace("\n", "\\n")
46
47
48def check_command_args_key_specs(args, command_key_specs_index_set, command_arg_key_specs_index_set):
49 if not args:
50 return True
51
52 for arg in args:
53 if arg.key_spec_index is not None:
54 assert isinstance(arg.key_spec_index, int)
55
56 if arg.key_spec_index not in command_key_specs_index_set:
57 print("command: %s arg: %s key_spec_index error" % (command.fullname(), arg.name))
58 return False
59
60 command_arg_key_specs_index_set.add(arg.key_spec_index)
61
62 if not check_command_args_key_specs(arg.subargs, command_key_specs_index_set, command_arg_key_specs_index_set):
63 return False
64
65 return True
66
67def check_command_key_specs(command):
68 if not command.key_specs:
69 return True
70
71 assert isinstance(command.key_specs, list)
72
73 for cmd_key_spec in command.key_specs:
74 if "flags" not in cmd_key_spec:
75 print("command: %s key_specs missing flags" % command.fullname())
76 return False
77
78 if "NOT_KEY" in cmd_key_spec["flags"]:
79 # Like SUNSUBSCRIBE / SPUBLISH / SSUBSCRIBE
80 return True
81
82 command_key_specs_index_set = set(range(len(command.key_specs)))
83 command_arg_key_specs_index_set = set()
84
85 # Collect key_spec used for each arg, including arg.subarg
86 if not check_command_args_key_specs(command.args, command_key_specs_index_set, command_arg_key_specs_index_set):
87 return False
88
89 # Check if we have key_specs not used
90 if command_key_specs_index_set != command_arg_key_specs_index_set:
91 print("command: %s may have unused key_spec" % command.fullname())
92 return False
93
94 return True
95
96
97# Globals
98subcommands = {} # container_name -> dict(subcommand_name -> Subcommand) - Only subcommands
99commands = {} # command_name -> Command - Only commands
100
101
102class KeySpec(object):
103 def __init__(self, spec):
104 self.spec = spec
105
106 def struct_code(self):
107 def _flags_code():
108 s = ""
109 for flag in self.spec.get("flags", []):
110 s += "CMD_KEY_%s|" % flag
111 return s[:-1] if s else 0
112
113 def _begin_search_code():
114 if self.spec["begin_search"].get("index"):
115 return "KSPEC_BS_INDEX,.bs.index={%d}" % (
116 self.spec["begin_search"]["index"]["pos"]
117 )
118 elif self.spec["begin_search"].get("keyword"):
119 return "KSPEC_BS_KEYWORD,.bs.keyword={\"%s\",%d}" % (
120 self.spec["begin_search"]["keyword"]["keyword"],
121 self.spec["begin_search"]["keyword"]["startfrom"],
122 )
123 elif "unknown" in self.spec["begin_search"]:
124 return "KSPEC_BS_UNKNOWN,{{0}}"
125 else:
126 print("Invalid begin_search! value=%s" % self.spec["begin_search"])
127 exit(1)
128
129 def _find_keys_code():
130 if self.spec["find_keys"].get("range"):
131 return "KSPEC_FK_RANGE,.fk.range={%d,%d,%d}" % (
132 self.spec["find_keys"]["range"]["lastkey"],
133 self.spec["find_keys"]["range"]["step"],
134 self.spec["find_keys"]["range"]["limit"]
135 )
136 elif self.spec["find_keys"].get("keynum"):
137 return "KSPEC_FK_KEYNUM,.fk.keynum={%d,%d,%d}" % (
138 self.spec["find_keys"]["keynum"]["keynumidx"],
139 self.spec["find_keys"]["keynum"]["firstkey"],
140 self.spec["find_keys"]["keynum"]["step"]
141 )
142 elif "unknown" in self.spec["find_keys"]:
143 return "KSPEC_FK_UNKNOWN,{{0}}"
144 else:
145 print("Invalid find_keys! value=%s" % self.spec["find_keys"])
146 exit(1)
147
148 return "%s,%s,%s,%s" % (
149 get_optional_desc_string(self.spec, "notes"),
150 _flags_code(),
151 _begin_search_code(),
152 _find_keys_code()
153 )
154
155
156def verify_no_dup_names(container_fullname, args):
157 name_list = [arg.name for arg in args]
158 name_set = set(name_list)
159 if len(name_list) != len(name_set):
160 print("{}: Dup argument names: {}".format(container_fullname, name_list))
161 exit(1)
162
163
164class Argument(object):
165 def __init__(self, parent_name, desc):
166 self.parent_name = parent_name
167 self.desc = desc
168 self.name = self.desc["name"].lower()
169 if "_" in self.name:
170 print("{}: name ({}) should not contain underscores".format(self.fullname(), self.name))
171 exit(1)
172 self.type = self.desc["type"]
173 self.key_spec_index = self.desc.get("key_spec_index", None)
174 self.subargs = []
175 if self.type in ["oneof", "block"]:
176 self.display = None
177 for subdesc in self.desc["arguments"]:
178 self.subargs.append(Argument(self.fullname(), subdesc))
179 if len(self.subargs) < 2:
180 print("{}: oneof or block arg contains less than two subargs".format(self.fullname()))
181 exit(1)
182 verify_no_dup_names(self.fullname(), self.subargs)
183 else:
184 self.display = self.desc.get("display")
185
186 def fullname(self):
187 return ("%s %s" % (self.parent_name, self.name)).replace("-", "_")
188
189 def struct_name(self):
190 return "%s_Arg" % (self.fullname().replace(" ", "_"))
191
192 def subarg_table_name(self):
193 assert self.subargs
194 return "%s_Subargs" % (self.fullname().replace(" ", "_"))
195
196 def struct_code(self):
197 """
198 Output example:
199 MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=GETEX_expiration_Subargs
200 """
201
202 def _flags_code():
203 s = ""
204 if self.desc.get("optional", False):
205 s += "CMD_ARG_OPTIONAL|"
206 if self.desc.get("multiple", False):
207 s += "CMD_ARG_MULTIPLE|"
208 if self.desc.get("multiple_token", False):
209 assert self.desc.get("multiple", False) # Sanity
210 s += "CMD_ARG_MULTIPLE_TOKEN|"
211 return s[:-1] if s else "CMD_ARG_NONE"
212
213 s = "MAKE_ARG(\"%s\",%s,%d,%s,%s,%s,%s,%d,%s)" % (
214 self.name,
215 ARG_TYPES[self.type],
216 self.desc.get("key_spec_index", -1),
217 get_optional_desc_string(self.desc, "token", force_uppercase=True),
218 get_optional_desc_string(self.desc, "summary"),
219 get_optional_desc_string(self.desc, "since"),
220 _flags_code(),
221 len(self.subargs),
222 get_optional_desc_string(self.desc, "deprecated_since"),
223 )
224 if "display" in self.desc:
225 s += ",.display_text=\"%s\"" % self.desc["display"].lower()
226 if self.subargs:
227 s += ",.subargs=%s" % self.subarg_table_name()
228
229 return s
230
231 def write_internal_structs(self, f):
232 if self.subargs:
233 for subarg in self.subargs:
234 subarg.write_internal_structs(f)
235
236 f.write("/* %s argument table */\n" % self.fullname())
237 f.write("struct COMMAND_ARG %s[] = {\n" % self.subarg_table_name())
238 for subarg in self.subargs:
239 f.write("{%s},\n" % subarg.struct_code())
240 f.write("};\n\n")
241
242
243def to_c_name(str):
244 return str.replace(":", "").replace(".", "_").replace("$", "_")\
245 .replace("^", "_").replace("*", "_").replace("-", "_") \
246 .replace("\\", "_").replace("+", "_")
247
248
249class ReplySchema(object):
250 def __init__(self, name, desc):
251 self.name = to_c_name(name)
252 self.schema = {}
253 if desc.get("type") == "object":
254 if desc.get("properties") and desc.get("additionalProperties") is None:
255 print("%s: Any object that has properties should have the additionalProperties field" % self.name)
256 exit(1)
257 elif desc.get("type") == "array":
258 if desc.get("items") and isinstance(desc["items"], list) and any([desc.get(k) is None for k in ["minItems", "maxItems"]]):
259 print("%s: Any array that has items should have the minItems and maxItems fields" % self.name)
260 exit(1)
261 for k, v in desc.items():
262 if isinstance(v, dict):
263 self.schema[k] = ReplySchema("%s_%s" % (self.name, k), v)
264 elif isinstance(v, list):
265 self.schema[k] = []
266 for i, subdesc in enumerate(v):
267 self.schema[k].append(ReplySchema("%s_%s_%i" % (self.name, k,i), subdesc))
268 else:
269 self.schema[k] = v
270
271 def write(self, f):
272 def struct_code(name, k, v):
273 if isinstance(v, ReplySchema):
274 t = "JSON_TYPE_OBJECT"
275 vstr = ".value.object=&%s" % name
276 elif isinstance(v, list):
277 t = "JSON_TYPE_ARRAY"
278 vstr = ".value.array={.objects=%s,.length=%d}" % (name, len(v))
279 elif isinstance(v, bool):
280 t = "JSON_TYPE_BOOLEAN"
281 vstr = ".value.boolean=%d" % int(v)
282 elif isinstance(v, str):
283 t = "JSON_TYPE_STRING"
284 vstr = ".value.string=\"%s\"" % v
285 elif isinstance(v, int):
286 t = "JSON_TYPE_INTEGER"
287 vstr = ".value.integer=%d" % v
288
289 return "%s,%s,%s" % (t, json.dumps(k), vstr)
290
291 for k, v in self.schema.items():
292 if isinstance(v, ReplySchema):
293 v.write(f)
294 elif isinstance(v, list):
295 for i, schema in enumerate(v):
296 schema.write(f)
297 name = to_c_name("%s_%s" % (self.name, k))
298 f.write("/* %s array reply schema */\n" % name)
299 f.write("struct jsonObject *%s[] = {\n" % name)
300 for i, schema in enumerate(v):
301 f.write("&%s,\n" % schema.name)
302 f.write("};\n\n")
303
304 f.write("/* %s reply schema */\n" % self.name)
305 f.write("struct jsonObjectElement %s_elements[] = {\n" % self.name)
306 for k, v in self.schema.items():
307 name = to_c_name("%s_%s" % (self.name, k))
308 f.write("{%s},\n" % struct_code(name, k, v))
309 f.write("};\n\n")
310 f.write("struct jsonObject %s = {%s_elements,.length=%d};\n\n" % (self.name, self.name, len(self.schema)))
311
312
313class Command(object):
314 def __init__(self, name, desc):
315 self.name = name.upper()
316 self.desc = desc
317 self.group = self.desc["group"]
318 self.key_specs = self.desc.get("key_specs", [])
319 self.subcommands = []
320 self.args = []
321 for arg_desc in self.desc.get("arguments", []):
322 self.args.append(Argument(self.fullname(), arg_desc))
323 verify_no_dup_names(self.fullname(), self.args)
324 self.reply_schema = None
325 if "reply_schema" in self.desc:
326 self.reply_schema = ReplySchema(self.reply_schema_name(), self.desc["reply_schema"])
327
328 def fullname(self):
329 return self.name.replace("-", "_").replace(":", "")
330
331 def return_types_table_name(self):
332 return "%s_ReturnInfo" % self.fullname().replace(" ", "_")
333
334 def subcommand_table_name(self):
335 assert self.subcommands
336 return "%s_Subcommands" % self.name
337
338 def history_table_name(self):
339 return "%s_History" % (self.fullname().replace(" ", "_"))
340
341 def tips_table_name(self):
342 return "%s_Tips" % (self.fullname().replace(" ", "_"))
343
344 def arg_table_name(self):
345 return "%s_Args" % (self.fullname().replace(" ", "_"))
346
347 def key_specs_table_name(self):
348 return "%s_Keyspecs" % (self.fullname().replace(" ", "_"))
349
350 def reply_schema_name(self):
351 return "%s_ReplySchema" % (self.fullname().replace(" ", "_"))
352
353 def struct_name(self):
354 return "%s_Command" % (self.fullname().replace(" ", "_"))
355
356 def history_code(self):
357 if not self.desc.get("history"):
358 return ""
359 s = ""
360 for tupl in self.desc["history"]:
361 s += "{\"%s\",\"%s\"},\n" % (tupl[0], tupl[1])
362 return s
363
364 def num_history(self):
365 if not self.desc.get("history"):
366 return 0
367 return len(self.desc["history"])
368
369 def tips_code(self):
370 if not self.desc.get("command_tips"):
371 return ""
372 s = ""
373 for hint in self.desc["command_tips"]:
374 s += "\"%s\",\n" % hint.lower()
375 return s
376
377 def num_tips(self):
378 if not self.desc.get("command_tips"):
379 return 0
380 return len(self.desc["command_tips"])
381
382 def key_specs_code(self):
383 s = ""
384 for spec in self.key_specs:
385 s += "{%s}," % KeySpec(spec).struct_code()
386 return s[:-1]
387
388
389 def struct_code(self):
390 """
391 Output example:
392 MAKE_CMD("set","Set the string value of a key","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SET_History,4,SET_Tips,0,setCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_STRING,SET_Keyspecs,1,setGetKeys,5),.args=SET_Args
393 """
394
395 def _flags_code():
396 s = ""
397 for flag in self.desc.get("command_flags", []):
398 s += "CMD_%s|" % flag
399 return s[:-1] if s else 0
400
401 def _acl_categories_code():
402 s = ""
403 for cat in self.desc.get("acl_categories", []):
404 s += "ACL_CATEGORY_%s|" % cat
405 return s[:-1] if s else 0
406
407 def _doc_flags_code():
408 s = ""
409 for flag in self.desc.get("doc_flags", []):
410 s += "CMD_DOC_%s|" % flag
411 return s[:-1] if s else "CMD_DOC_NONE"
412
413 s = "MAKE_CMD(\"%s\",%s,%s,%s,%s,%s,%s,%s,%s,%s,%d,%s,%d,%s,%d,%s,%s,%s,%d,%s,%d)," % (
414 self.name.lower(),
415 get_optional_desc_string(self.desc, "summary"),
416 get_optional_desc_string(self.desc, "complexity"),
417 get_optional_desc_string(self.desc, "since"),
418 _doc_flags_code(),
419 get_optional_desc_string(self.desc, "replaced_by"),
420 get_optional_desc_string(self.desc, "deprecated_since"),
421 "\"%s\"" % self.group,
422 GROUPS[self.group],
423 self.history_table_name(),
424 self.num_history(),
425 self.tips_table_name(),
426 self.num_tips(),
427 self.desc.get("function", "NULL"),
428 self.desc["arity"],
429 _flags_code(),
430 _acl_categories_code(),
431 self.key_specs_table_name(),
432 len(self.key_specs),
433 self.desc.get("get_keys_function", "NULL"),
434 len(self.args),
435 )
436
437 if self.subcommands:
438 s += ".subcommands=%s," % self.subcommand_table_name()
439
440 if self.args:
441 s += ".args=%s," % self.arg_table_name()
442
443 if self.reply_schema and args.with_reply_schema:
444 s += ".reply_schema=&%s," % self.reply_schema_name()
445
446 return s[:-1]
447
448 def write_internal_structs(self, f):
449 if self.subcommands:
450 subcommand_list = sorted(self.subcommands, key=lambda cmd: cmd.name)
451 for subcommand in subcommand_list:
452 subcommand.write_internal_structs(f)
453
454 f.write("/* %s command table */\n" % self.fullname())
455 f.write("struct COMMAND_STRUCT %s[] = {\n" % self.subcommand_table_name())
456 for subcommand in subcommand_list:
457 f.write("{%s},\n" % subcommand.struct_code())
458 f.write("{0}\n")
459 f.write("};\n\n")
460
461 f.write("/********** %s ********************/\n\n" % self.fullname())
462
463 f.write("#ifndef SKIP_CMD_HISTORY_TABLE\n")
464 f.write("/* %s history */\n" % self.fullname())
465 code = self.history_code()
466 if code:
467 f.write("commandHistory %s[] = {\n" % self.history_table_name())
468 f.write("%s" % code)
469 f.write("};\n")
470 else:
471 f.write("#define %s NULL\n" % self.history_table_name())
472 f.write("#endif\n\n")
473
474 f.write("#ifndef SKIP_CMD_TIPS_TABLE\n")
475 f.write("/* %s tips */\n" % self.fullname())
476 code = self.tips_code()
477 if code:
478 f.write("const char *%s[] = {\n" % self.tips_table_name())
479 f.write("%s" % code)
480 f.write("};\n")
481 else:
482 f.write("#define %s NULL\n" % self.tips_table_name())
483 f.write("#endif\n\n")
484
485 f.write("#ifndef SKIP_CMD_KEY_SPECS_TABLE\n")
486 f.write("/* %s key specs */\n" % self.fullname())
487 code = self.key_specs_code()
488 if code:
489 f.write("keySpec %s[%d] = {\n" % (self.key_specs_table_name(), len(self.key_specs)))
490 f.write("%s\n" % code)
491 f.write("};\n")
492 else:
493 f.write("#define %s NULL\n" % self.key_specs_table_name())
494 f.write("#endif\n\n")
495
496 if self.args:
497 for arg in self.args:
498 arg.write_internal_structs(f)
499
500 f.write("/* %s argument table */\n" % self.fullname())
501 f.write("struct COMMAND_ARG %s[] = {\n" % self.arg_table_name())
502 for arg in self.args:
503 f.write("{%s},\n" % arg.struct_code())
504 f.write("};\n\n")
505
506 if self.reply_schema and args.with_reply_schema:
507 self.reply_schema.write(f)
508
509
510class Subcommand(Command):
511 def __init__(self, name, desc):
512 self.container_name = desc["container"].upper()
513 super(Subcommand, self).__init__(name, desc)
514
515 def fullname(self):
516 return "%s %s" % (self.container_name, self.name.replace("-", "_").replace(":", ""))
517
518
519def create_command(name, desc):
520 flags = desc.get("command_flags")
521 if flags and "EXPERIMENTAL" in flags:
522 print("Command %s is experimental, skipping..." % name)
523 return
524
525 if desc.get("container"):
526 cmd = Subcommand(name.upper(), desc)
527 subcommands.setdefault(desc["container"].upper(), {})[name] = cmd
528 else:
529 cmd = Command(name.upper(), desc)
530 commands[name.upper()] = cmd
531
532
533# MAIN
534
535# Figure out where the sources are
536srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src")
537
538parser = argparse.ArgumentParser()
539parser.add_argument('--with-reply-schema', action='store_true')
540args = parser.parse_args()
541
542# Create all command objects
543print("Processing json files...")
544for filename in glob.glob('%s/commands/*.json' % srcdir):
545 with open(filename, "r") as f:
546 try:
547 d = json.load(f)
548 for name, desc in d.items():
549 create_command(name, desc)
550 except json.decoder.JSONDecodeError as err:
551 print("Error processing %s: %s" % (filename, err))
552 exit(1)
553
554# Link subcommands to containers
555print("Linking container command to subcommands...")
556for command in commands.values():
557 assert command.group
558 if command.name not in subcommands:
559 continue
560 for subcommand in subcommands[command.name].values():
561 assert not subcommand.group or subcommand.group == command.group
562 subcommand.group = command.group
563 command.subcommands.append(subcommand)
564
565check_command_error_counter = 0 # An error counter is used to count errors in command checking.
566
567print("Checking all commands...")
568for command in commands.values():
569 if not check_command_key_specs(command):
570 check_command_error_counter += 1
571
572if check_command_error_counter != 0:
573 print("Error: There are errors in the commands check, please check the above logs.")
574 exit(1)
575
576commands_filename = "commands_with_reply_schema" if args.with_reply_schema else "commands"
577print("Generating %s.def..." % commands_filename)
578with open("%s/%s.def" % (srcdir, commands_filename), "w") as f:
579 f.write("/* Automatically generated by %s, do not edit. */\n\n" % os.path.basename(__file__))
580 f.write(
581"""
582/* We have fabulous commands from
583 * the fantastic
584 * Redis Command Table! */
585
586/* Must match redisCommandGroup */
587const char *COMMAND_GROUP_STR[] = {
588 "generic",
589 "string",
590 "list",
591 "set",
592 "sorted-set",
593 "hash",
594 "pubsub",
595 "transactions",
596 "connection",
597 "server",
598 "scripting",
599 "hyperloglog",
600 "cluster",
601 "sentinel",
602 "geo",
603 "stream",
604 "bitmap",
605 "module"
606};
607
608const char *commandGroupStr(int index) {
609 return COMMAND_GROUP_STR[index];
610}
611"""
612 )
613
614 command_list = sorted(commands.values(), key=lambda cmd: (cmd.group, cmd.name))
615 for command in command_list:
616 command.write_internal_structs(f)
617
618 f.write("/* Main command table */\n")
619 f.write("struct COMMAND_STRUCT redisCommandTable[] = {\n")
620 curr_group = None
621 for command in command_list:
622 if curr_group != command.group:
623 curr_group = command.group
624 f.write("/* %s */\n" % curr_group)
625 f.write("{%s},\n" % command.struct_code())
626 f.write("{0}\n")
627 f.write("};\n")
628
629print("All done, exiting.")
diff --git a/examples/redis-unstable/utils/generate-commands-json.py b/examples/redis-unstable/utils/generate-commands-json.py
new file mode 100755
index 0000000..ea18a79
--- /dev/null
+++ b/examples/redis-unstable/utils/generate-commands-json.py
@@ -0,0 +1,148 @@
1#!/usr/bin/env python3
2import argparse
3import json
4import os
5import sys
6import subprocess
7from collections import OrderedDict
8
9
10def convert_flags_to_boolean_dict(flags):
11 """Return a dict with a key set to `True` per element in the flags list."""
12 return {f: True for f in flags}
13
14
15def set_if_not_none_or_empty(dst, key, value):
16 """Set 'key' in 'dst' if 'value' is not `None` or an empty list."""
17 if value is not None and (type(value) is not list or len(value)):
18 dst[key] = value
19
20
21def convert_argument(arg):
22 """Transform an argument."""
23 arg.update(convert_flags_to_boolean_dict(arg.pop('flags', [])))
24 set_if_not_none_or_empty(arg, 'arguments',
25 [convert_argument(x) for x in arg.pop('arguments', [])])
26 return arg
27
28
29def convert_keyspec(spec):
30 """Transform a key spec."""
31 spec.update(convert_flags_to_boolean_dict(spec.pop('flags', [])))
32 return spec
33
34
35def convert_entry_to_objects_array(cmd, docs):
36 """Transform the JSON output of `COMMAND` to a friendlier format.
37
38 cmd is the output of `COMMAND` as follows:
39 1. Name (lower case, e.g. "lolwut")
40 2. Arity
41 3. Flags
42 4-6. First/last/step key specification (deprecated as of Redis v7.0)
43 7. ACL categories
44 8. hints (as of Redis 7.0)
45 9. key-specs (as of Redis 7.0)
46 10. subcommands (as of Redis 7.0)
47
48 docs is the output of `COMMAND DOCS`, which holds a map of additional metadata
49
50 This returns a list with a dict for the command and per each of its
51 subcommands. Each dict contains one key, the command's full name, with a
52 value of a dict that's set with the command's properties and meta
53 information."""
54 assert len(cmd) >= 9
55 obj = {}
56 rep = [obj]
57 name = cmd[0].upper()
58 arity = cmd[1]
59 command_flags = cmd[2]
60 acl_categories = cmd[6]
61 hints = cmd[7]
62 keyspecs = cmd[8]
63 subcommands = cmd[9] if len(cmd) > 9 else []
64 key = name.replace('|', ' ')
65
66 subcommand_docs = docs.pop('subcommands', [])
67 rep.extend([convert_entry_to_objects_array(x, subcommand_docs[x[0]])[0] for x in subcommands])
68
69 # The command's value is ordered so the interesting stuff that we care about
70 # is at the start. Optional `None` and empty list values are filtered out.
71 value = OrderedDict()
72 group = docs.pop('group')
73 if group == 'module':
74 set_if_not_none_or_empty(value, 'summary', docs.pop('summary', None))
75 set_if_not_none_or_empty(value, 'since', docs.pop('since', None))
76 else:
77 # "summary" and "since" are required for all non-module commands
78 value['summary'] = docs.pop('summary')
79 value['since'] = docs.pop('since')
80 value['group'] = group
81 set_if_not_none_or_empty(value, 'complexity', docs.pop('complexity', None))
82 set_if_not_none_or_empty(value, 'deprecated_since', docs.pop('deprecated_since', None))
83 set_if_not_none_or_empty(value, 'replaced_by', docs.pop('replaced_by', None))
84 set_if_not_none_or_empty(value, 'history', docs.pop('history', []))
85 set_if_not_none_or_empty(value, 'acl_categories', acl_categories)
86 value['arity'] = arity
87 set_if_not_none_or_empty(value, 'key_specs',
88 [convert_keyspec(x) for x in keyspecs])
89 set_if_not_none_or_empty(value, 'arguments',
90 [convert_argument(x) for x in docs.pop('arguments', [])])
91 set_if_not_none_or_empty(value, 'command_flags', command_flags)
92 set_if_not_none_or_empty(value, 'doc_flags', docs.pop('doc_flags', []))
93 set_if_not_none_or_empty(value, 'hints', hints)
94
95 # All remaining docs key-value tuples, if any, are appended to the command
96 # to be future-proof.
97 while len(docs) > 0:
98 (k, v) = docs.popitem()
99 value[k] = v
100
101 obj[key] = value
102 return rep
103
104
105# Figure out where the sources are
106srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src")
107
108# MAIN
109if __name__ == '__main__':
110 opts = {
111 'description': 'Transform the output from `redis-cli --json` using COMMAND and COMMAND DOCS to a single commands.json format.',
112 'epilog': f'Usage example: {sys.argv[0]} --cli src/redis-cli --port 6379 > commands.json'
113 }
114 parser = argparse.ArgumentParser(**opts)
115 parser.add_argument('--host', type=str, default='localhost')
116 parser.add_argument('--port', type=int, default=6379)
117 parser.add_argument('--cli', type=str, default='%s/redis-cli' % srcdir)
118 args = parser.parse_args()
119
120 payload = OrderedDict()
121 cmds = []
122
123 p = subprocess.Popen([args.cli, '-h', args.host, '-p', str(args.port), '--json', 'command'], stdout=subprocess.PIPE)
124 stdout, stderr = p.communicate()
125 commands = json.loads(stdout)
126
127 p = subprocess.Popen([args.cli, '-h', args.host, '-p', str(args.port), '--json', 'command', 'docs'],
128 stdout=subprocess.PIPE)
129 stdout, stderr = p.communicate()
130 docs = json.loads(stdout)
131
132 for entry in commands:
133 cmd = convert_entry_to_objects_array(entry, docs[entry[0]])
134 cmds.extend(cmd)
135
136 # The final output is a dict of all commands, ordered by name.
137 cmds.sort(key=lambda x: list(x.keys())[0])
138 for cmd in cmds:
139 name = list(cmd.keys())[0]
140 payload[name] = cmd[name]
141
142 # Print the final JSON output. If the output is piped and the pipe is closed (e.g., by 'less' or 'head'),
143 # catch BrokenPipeError to prevent a traceback and exit gracefully.
144 try:
145 print(json.dumps(payload, indent=4))
146 except BrokenPipeError:
147 sys.stderr.close()
148 sys.exit(0)
diff --git a/examples/redis-unstable/utils/generate-fmtargs.py b/examples/redis-unstable/utils/generate-fmtargs.py
new file mode 100755
index 0000000..1eced02
--- /dev/null
+++ b/examples/redis-unstable/utils/generate-fmtargs.py
@@ -0,0 +1,22 @@
1#!/usr/bin/env python3
2
3# Outputs the generated part of src/fmtargs.h
4MAX_ARGS = 160
5
6import os
7print("/* Everything below this line is automatically generated by")
8print(" * %s. Do not manually edit. */\n" % os.path.basename(__file__))
9
10print('#define ARG_N(' + ', '.join(['_' + str(i) for i in range(1, MAX_ARGS + 1, 1)]) + ', N, ...) N')
11
12print('\n#define RSEQ_N() ' + ', '.join([str(i) for i in range(MAX_ARGS, -1, -1)]))
13
14print('\n#define COMPACT_FMT_2(fmt, value) fmt')
15for i in range(4, MAX_ARGS + 1, 2):
16 print('#define COMPACT_FMT_{}(fmt, value, ...) fmt COMPACT_FMT_{}(__VA_ARGS__)'.format(i, i - 2))
17
18print('\n#define COMPACT_VALUES_2(fmt, value) value')
19for i in range(4, MAX_ARGS + 1, 2):
20 print('#define COMPACT_VALUES_{}(fmt, value, ...) value, COMPACT_VALUES_{}(__VA_ARGS__)'.format(i, i - 2))
21
22print("\n#endif")
diff --git a/examples/redis-unstable/utils/generate-module-api-doc.rb b/examples/redis-unstable/utils/generate-module-api-doc.rb
new file mode 100755
index 0000000..cd45c93
--- /dev/null
+++ b/examples/redis-unstable/utils/generate-module-api-doc.rb
@@ -0,0 +1,195 @@
1#!/usr/bin/env ruby
2# coding: utf-8
3# gendoc.rb -- Converts the top-comments inside module.c to modules API
4# reference documentation in markdown format.
5
6# Convert the C comment to markdown
7def markdown(s)
8 s = s.gsub(/\*\/$/,"")
9 s = s.gsub(/^ ?\* ?/,"")
10 s = s.gsub(/^\/\*\*? ?/,"")
11 s.chop! while s[-1] == "\n" || s[-1] == " "
12 lines = s.split("\n")
13 newlines = []
14 # Fix some markdown
15 lines.each{|l|
16 # Rewrite RM_Xyz() to RedisModule_Xyz().
17 l = l.gsub(/(?<![A-Z_])RM_(?=[A-Z])/, 'RedisModule_')
18 # Fix more markdown, except in code blocks indented by 4 spaces, which we
19 # don't want to mess with.
20 if not l.start_with?(' ')
21 # Add backquotes around RedisModule functions and type where missing.
22 l = l.gsub(/(?<!`)RedisModule[A-z]+(?:\*?\(\))?/){|x| "`#{x}`"}
23 # Add backquotes around c functions like malloc() where missing.
24 l = l.gsub(/(?<![`A-z.])[a-z_]+\(\)/, '`\0`')
25 # Add backquotes around macro and var names containing underscores.
26 l = l.gsub(/(?<![`A-z\*])[A-Za-z]+_[A-Za-z0-9_]+/){|x| "`#{x}`"}
27 # Link URLs preceded by space or newline (not already linked)
28 l = l.gsub(/(^| )(https?:\/\/[A-Za-z0-9_\/\.\-]+[A-Za-z0-9\/])/,
29 '\1[\2](\2)')
30 # Replace double-dash with unicode ndash
31 l = l.gsub(/ -- /, ' – ')
32 end
33 # Link function names to their definition within the page
34 l = l.gsub(/`(RedisModule_[A-z0-9]+)[()]*`/) {|x|
35 $index[$1] ? "[#{x}](\##{$1})" : x
36 }
37 newlines << l
38 }
39 return newlines.join("\n")
40end
41
42# Linebreak a prototype longer than 80 characters on the commas, but only
43# between balanced parentheses so that we don't linebreak args which are
44# function pointers, and then aligning each arg under each other.
45def linebreak_proto(proto, indent)
46 if proto.bytesize <= 80
47 return proto
48 end
49 parts = proto.split(/,\s*/);
50 if parts.length == 1
51 return proto;
52 end
53 align_pos = proto.index("(") + 1;
54 align = " " * align_pos
55 result = parts.shift;
56 bracket_balance = 0;
57 parts.each{|part|
58 if bracket_balance == 0
59 result += ",\n" + indent + align
60 else
61 result += ", "
62 end
63 result += part
64 bracket_balance += part.count("(") - part.count(")")
65 }
66 return result;
67end
68
69# Given the source code array and the index at which an exported symbol was
70# detected, extracts and outputs the documentation.
71def docufy(src,i)
72 m = /RM_[A-z0-9]+/.match(src[i])
73 name = m[0]
74 name = name.sub("RM_","RedisModule_")
75 proto = src[i].sub("{","").strip+";\n"
76 proto = proto.sub("RM_","RedisModule_")
77 proto = linebreak_proto(proto, " ");
78 # Add a link target with the function name. (We don't trust the exact id of
79 # the generated one, which depends on the Markdown implementation.)
80 puts "<span id=\"#{name}\"></span>\n\n"
81 puts "### `#{name}`\n\n"
82 puts " #{proto}\n"
83 puts "**Available since:** #{$since[name] or "unreleased"}\n\n"
84 comment = ""
85 while true
86 i = i-1
87 comment = src[i]+comment
88 break if src[i] =~ /\/\*/
89 end
90 comment = markdown(comment)
91 puts comment+"\n\n"
92end
93
94# Print a comment from line until */ is found, as markdown.
95def section_doc(src, i)
96 name = get_section_heading(src, i)
97 comment = "<span id=\"#{section_name_to_id(name)}\"></span>\n\n"
98 while true
99 # append line, except if it's a horizontal divider
100 comment = comment + src[i] if src[i] !~ /^[\/ ]?\*{1,2} ?-{50,}/
101 break if src[i] =~ /\*\//
102 i = i+1
103 end
104 comment = markdown(comment)
105 puts comment+"\n\n"
106end
107
108# generates an id suitable for links within the page
109def section_name_to_id(name)
110 return "section-" +
111 name.strip.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/^-+|-+$/, '')
112end
113
114# Returns the name of the first section heading in the comment block for which
115# is_section_doc(src, i) is true
116def get_section_heading(src, i)
117 if src[i] =~ /^\/\*\*? \#+ *(.*)/
118 heading = $1
119 elsif src[i+1] =~ /^ ?\* \#+ *(.*)/
120 heading = $1
121 end
122 return heading.gsub(' -- ', ' – ')
123end
124
125# Returns true if the line is the start of a generic documentation section. Such
126# section must start with the # symbol, i.e. a markdown heading, on the first or
127# the second line.
128def is_section_doc(src, i)
129 return src[i] =~ /^\/\*\*? \#/ ||
130 (src[i] =~ /^\/\*/ && src[i+1] =~ /^ ?\* \#/)
131end
132
133def is_func_line(src, i)
134 line = src[i]
135 return line =~ /RM_/ &&
136 line[0] != ' ' && line[0] != '#' && line[0] != '/' &&
137 src[i-1] =~ /\*\//
138end
139
140puts "<!-- This file is generated from module.c using\n"
141puts " redis/redis:utils/generate-module-api-doc.rb -->\n\n"
142src = File.open(File.dirname(__FILE__) ++ "/../src/module.c").to_a
143
144# Build function index
145$index = {}
146src.each_with_index do |line,i|
147 if is_func_line(src, i)
148 line =~ /RM_([A-z0-9]+)/
149 name = "RedisModule_#{$1}"
150 $index[name] = true
151 end
152end
153
154# Populate the 'since' map (name => version) if we're in a git repo.
155$since = {}
156git_dir = File.dirname(__FILE__) ++ "/../.git"
157if File.directory?(git_dir) && `which git` != ""
158 `git --git-dir="#{git_dir}" tag --sort=v:refname`.each_line do |version|
159 next if version !~ /^(\d+)\.\d+\.\d+?$/ || $1.to_i < 4
160 version.chomp!
161 `git --git-dir="#{git_dir}" cat-file blob "#{version}:src/module.c"`.each_line do |line|
162 if line =~ /^\w.*[ \*]RM_([A-z0-9]+)/
163 name = "RedisModule_#{$1}"
164 if ! $since[name]
165 $since[name] = version
166 end
167 end
168 end
169 end
170end
171
172# Print TOC
173puts "## Sections\n\n"
174src.each_with_index do |_line,i|
175 if is_section_doc(src, i)
176 name = get_section_heading(src, i)
177 puts "* [#{name}](\##{section_name_to_id(name)})\n"
178 end
179end
180puts "* [Function index](#section-function-index)\n\n"
181
182# Docufy: Print function prototype and markdown docs
183src.each_with_index do |_line,i|
184 if is_func_line(src, i)
185 docufy(src, i)
186 elsif is_section_doc(src, i)
187 section_doc(src, i)
188 end
189end
190
191# Print function index
192puts "<span id=\"section-function-index\"></span>\n\n"
193puts "## Function index\n\n"
194$index.keys.sort.each{|x| puts "* [`#{x}`](\##{x})\n"}
195puts "\n"
diff --git a/examples/redis-unstable/utils/graphs/commits-over-time/README.md b/examples/redis-unstable/utils/graphs/commits-over-time/README.md
new file mode 100644
index 0000000..b28019e
--- /dev/null
+++ b/examples/redis-unstable/utils/graphs/commits-over-time/README.md
@@ -0,0 +1,16 @@
1This Tcl script is what I used in order to generate the graph you
2can find at http://antirez.com/news/98. It's really quick & dirty, more
3a trow away program than anything else, but probably could be reused or
4modified in the future in order to visualize other similar data or an
5updated version of the same data.
6
7The usage is trivial:
8
9 ./genhtml.tcl > output.html
10
11The generated HTML is quite broken but good enough to grab a screenshot
12from the browser. Feel free to improve it if you got time / interest.
13
14Note that the code filtering the tags, and the hardcoded branch name, does
15not make the script, as it is, able to analyze a different repository.
16However the changes needed are trivial.
diff --git a/examples/redis-unstable/utils/graphs/commits-over-time/genhtml.tcl b/examples/redis-unstable/utils/graphs/commits-over-time/genhtml.tcl
new file mode 100755
index 0000000..c4b4e09
--- /dev/null
+++ b/examples/redis-unstable/utils/graphs/commits-over-time/genhtml.tcl
@@ -0,0 +1,96 @@
1#!/usr/bin/env tclsh
2
3# Load commits history as "sha1 unixtime".
4set commits [exec git log unstable {--pretty="%H %at"}]
5set raw_tags [exec git tag]
6
7# Load all the tags that are about stable releases.
8foreach tag $raw_tags {
9 if {[string match v*-stable $tag]} {
10 set tag [string range $tag 1 end-7]
11 puts $tag
12 }
13 if {[regexp {^[0-9]+.[0-9]+.[0-9]+$} $tag]} {
14 lappend tags $tag
15 }
16}
17
18# For each tag, create a list of "name unixtime"
19foreach tag $tags {
20 set taginfo [exec git log $tag -n 1 "--pretty=\"$tag %at\""]
21 set taginfo [string trim $taginfo {"}]
22 lappend labels $taginfo
23}
24
25# For each commit, check the amount of code changed and create an array
26# mapping the commit to the number of lines affected.
27foreach c $commits {
28 set stat [exec git show --oneline --numstat [lindex $c 0]]
29 set linenum 0
30 set affected 0
31 foreach line [split $stat "\n"] {
32 incr linenum
33 if {$linenum == 1 || [string match *deps/* $line]} continue
34 if {[catch {llength $line} numfields]} continue
35 if {$numfields == 0} continue
36 catch {
37 incr affected [lindex $line 0]
38 incr affected [lindex $line 1]
39 }
40 }
41 set commit_to_affected([lindex $c 0]) $affected
42}
43
44set base_time [lindex [lindex $commits end] 1]
45puts [clock format $base_time]
46
47# Generate a graph made of HTML DIVs.
48puts {<html>
49<style>
50.box {
51 position:absolute;
52 width:10px;
53 height:5px;
54 border:1px black solid;
55 background-color:#44aa33;
56 opacity: 0.04;
57}
58.label {
59 position:absolute;
60 background-color:#dddddd;
61 font-family:helvetica;
62 font-size:12px;
63 padding:2px;
64 color:#666;
65 border:1px #aaa solid;
66 border-radius: 5px;
67}
68#outer {
69 position:relative;
70 width:1500;
71 height:500;
72 border:1px #aaa solid;
73}
74</style>
75<div id="outer">
76}
77foreach c $commits {
78 set sha [lindex $c 0]
79 set t [expr {([lindex $c 1]-$base_time)/(3600*24*2)}]
80 set affected [expr $commit_to_affected($sha)]
81 set left $t
82 set height [expr {log($affected)*20}]
83 puts "<div class=\"box\" style=\"left:$left; bottom:0; height:$height\"></div>"
84}
85
86set bottom -30
87foreach l $labels {
88 set name [lindex $l 0]
89 set t [expr {([lindex $l 1]-$base_time)/(3600*24*2)}]
90 set left $t
91 if {$left < 0} continue
92 incr bottom -20
93 if {$bottom == -210} {set bottom -30}
94 puts "<div class=\"label\" style=\"left:$left; bottom:$bottom\">$name</div>"
95}
96puts {</div></html>}
diff --git a/examples/redis-unstable/utils/hyperloglog/.gitignore b/examples/redis-unstable/utils/hyperloglog/.gitignore
new file mode 100644
index 0000000..2211df6
--- /dev/null
+++ b/examples/redis-unstable/utils/hyperloglog/.gitignore
@@ -0,0 +1 @@
*.txt
diff --git a/examples/redis-unstable/utils/hyperloglog/hll-err.rb b/examples/redis-unstable/utils/hyperloglog/hll-err.rb
new file mode 100644
index 0000000..1278397
--- /dev/null
+++ b/examples/redis-unstable/utils/hyperloglog/hll-err.rb
@@ -0,0 +1,30 @@
1# hll-err.rb - Copyright (C) 2014-Present Redis Ltd.
2#
3# Licensed under your choice of (a) the Redis Source Available License 2.0
4# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
5# GNU Affero General Public License v3 (AGPLv3).
6#
7# Check error of HyperLogLog Redis implementation for different set sizes.
8
9require 'rubygems'
10require 'redis'
11require 'digest/sha1'
12
13r = Redis.new
14r.del('hll')
15i = 0
16while true do
17 100.times {
18 elements = []
19 1000.times {
20 ele = Digest::SHA1.hexdigest(i.to_s)
21 elements << ele
22 i += 1
23 }
24 r.pfadd('hll',elements)
25 }
26 approx = r.pfcount('hll')
27 abs_err = (approx-i).abs
28 rel_err = 100.to_f*abs_err/i
29 puts "#{i} vs #{approx}: #{rel_err}%"
30end
diff --git a/examples/redis-unstable/utils/hyperloglog/hll-gnuplot-graph.rb b/examples/redis-unstable/utils/hyperloglog/hll-gnuplot-graph.rb
new file mode 100644
index 0000000..1e48e6c
--- /dev/null
+++ b/examples/redis-unstable/utils/hyperloglog/hll-gnuplot-graph.rb
@@ -0,0 +1,91 @@
1# hll-err.rb - Copyright (C) 2014-Present Redis Ltd.
2#
3# Licensed under your choice of (a) the Redis Source Available License 2.0
4# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
5# GNU Affero General Public License v3 (AGPLv3).
6#
7# This program is suited to output average and maximum errors of
8# the Redis HyperLogLog implementation in a format suitable to print
9# graphs using gnuplot.
10
11require 'rubygems'
12require 'redis'
13require 'digest/sha1'
14
15# Generate an array of [cardinality,relative_error] pairs
16# in the 0 - max range, with the specified step.
17#
18# 'r' is the Redis object used to perform the queries.
19# 'seed' must be different every time you want a test performed
20# with a different set. The function guarantees that if 'seed' is the
21# same, exactly the same dataset is used, and when it is different,
22# a totally unrelated different data set is used (without any common
23# element in practice).
24def run_experiment(r,seed,max,step)
25 r.del('hll')
26 i = 0
27 samples = []
28 step = 1000 if step > 1000
29 while i < max do
30 elements = []
31 step.times {
32 ele = Digest::SHA1.hexdigest(i.to_s+seed.to_s)
33 elements << ele
34 i += 1
35 }
36 r.pfadd('hll',elements)
37 approx = r.pfcount('hll')
38 err = approx-i
39 rel_err = 100.to_f*err/i
40 samples << [i,rel_err]
41 end
42 samples
43end
44
45def filter_samples(numsets,max,step,filter)
46 r = Redis.new
47 dataset = {}
48 (0...numsets).each{|i|
49 dataset[i] = run_experiment(r,i,max,step)
50 STDERR.puts "Set #{i}"
51 }
52 dataset[0].each_with_index{|ele,index|
53 if filter == :max
54 card=ele[0]
55 err=ele[1].abs
56 (1...numsets).each{|i|
57 err = dataset[i][index][1] if err < dataset[i][index][1]
58 }
59 puts "#{card} #{err}"
60 elsif filter == :avg
61 card=ele[0]
62 err = 0
63 (0...numsets).each{|i|
64 err += dataset[i][index][1]
65 }
66 err /= numsets
67 puts "#{card} #{err}"
68 elsif filter == :absavg
69 card=ele[0]
70 err = 0
71 (0...numsets).each{|i|
72 err += dataset[i][index][1].abs
73 }
74 err /= numsets
75 puts "#{card} #{err}"
76 elsif filter == :all
77 (0...numsets).each{|i|
78 card,err = dataset[i][index]
79 puts "#{card} #{err}"
80 }
81 else
82 raise "Unknown filter #{filter}"
83 end
84 }
85end
86
87if ARGV.length != 4
88 puts "Usage: hll-gnuplot-graph <samples> <max> <step> (max|avg|absavg|all)"
89 exit 1
90end
91filter_samples(ARGV[0].to_i,ARGV[1].to_i,ARGV[2].to_i,ARGV[3].to_sym)
diff --git a/examples/redis-unstable/utils/install_server.sh b/examples/redis-unstable/utils/install_server.sh
new file mode 100755
index 0000000..efda7da
--- /dev/null
+++ b/examples/redis-unstable/utils/install_server.sh
@@ -0,0 +1,291 @@
1#!/bin/sh
2
3# Copyright 2011 Dvir Volk <dvirsk at gmail dot com>. All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are met:
7#
8# 1. Redistributions of source code must retain the above copyright notice,
9# this list of conditions and the following disclaimer.
10#
11# 2. Redistributions in binary form must reproduce the above copyright
12# notice, this list of conditions and the following disclaimer in the
13# documentation and/or other materials provided with the distribution.
14#
15# THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
16# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
17# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
18# EVENT SHALL Dvir Volk OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
19# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
20# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
21# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
22# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
23# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25#
26################################################################################
27#
28# Service installer for redis server, runs interactively by default.
29#
30# To run this script non-interactively (for automation/provisioning purposes),
31# feed the variables into the script. Any missing variables will be prompted!
32# Tip: Environment variables also support command substitution (see REDIS_EXECUTABLE)
33#
34# Example:
35#
36# sudo REDIS_PORT=1234 \
37# REDIS_CONFIG_FILE=/etc/redis/1234.conf \
38# REDIS_LOG_FILE=/var/log/redis_1234.log \
39# REDIS_DATA_DIR=/var/lib/redis/1234 \
40# REDIS_EXECUTABLE=`command -v redis-server` ./utils/install_server.sh
41#
42# This generates a redis config file and an /etc/init.d script, and installs them.
43#
44# /!\ This script should be run as root
45#
46# NOTE: This script will not work on Mac OSX.
47# It supports Debian and Ubuntu Linux.
48#
49################################################################################
50
51die () {
52 echo "ERROR: $1. Aborting!"
53 exit 1
54}
55
56
57#Absolute path to this script
58SCRIPT=$(readlink -f $0)
59#Absolute path this script is in
60SCRIPTPATH=$(dirname $SCRIPT)
61
62#Initial defaults
63_REDIS_PORT=6379
64_MANUAL_EXECUTION=false
65
66echo "Welcome to the redis service installer"
67echo "This script will help you easily set up a running redis server"
68echo
69
70#check for root user
71if [ "$(id -u)" -ne 0 ] ; then
72 echo "You must run this script as root. Sorry!"
73 exit 1
74fi
75
76#bail if this system is managed by systemd
77_pid_1_exe="$(readlink -f /proc/1/exe)"
78if [ "${_pid_1_exe##*/}" = systemd ]
79then
80 echo "This systems seems to use systemd."
81 echo "Please take a look at the provided example service unit files in this directory, and adapt and install them. Sorry!"
82 exit 1
83fi
84unset _pid_1_exe
85
86if ! echo $REDIS_PORT | egrep -q '^[0-9]+$' ; then
87 _MANUAL_EXECUTION=true
88 #Read the redis port
89 read -p "Please select the redis port for this instance: [$_REDIS_PORT] " REDIS_PORT
90 if ! echo $REDIS_PORT | egrep -q '^[0-9]+$' ; then
91 echo "Selecting default: $_REDIS_PORT"
92 REDIS_PORT=$_REDIS_PORT
93 fi
94fi
95
96if [ -z "$REDIS_CONFIG_FILE" ] ; then
97 _MANUAL_EXECUTION=true
98 #read the redis config file
99 _REDIS_CONFIG_FILE="/etc/redis/$REDIS_PORT.conf"
100 read -p "Please select the redis config file name [$_REDIS_CONFIG_FILE] " REDIS_CONFIG_FILE
101 if [ -z "$REDIS_CONFIG_FILE" ] ; then
102 REDIS_CONFIG_FILE=$_REDIS_CONFIG_FILE
103 echo "Selected default - $REDIS_CONFIG_FILE"
104 fi
105fi
106
107if [ -z "$REDIS_LOG_FILE" ] ; then
108 _MANUAL_EXECUTION=true
109 #read the redis log file path
110 _REDIS_LOG_FILE="/var/log/redis_$REDIS_PORT.log"
111 read -p "Please select the redis log file name [$_REDIS_LOG_FILE] " REDIS_LOG_FILE
112 if [ -z "$REDIS_LOG_FILE" ] ; then
113 REDIS_LOG_FILE=$_REDIS_LOG_FILE
114 echo "Selected default - $REDIS_LOG_FILE"
115 fi
116fi
117
118if [ -z "$REDIS_DATA_DIR" ] ; then
119 _MANUAL_EXECUTION=true
120 #get the redis data directory
121 _REDIS_DATA_DIR="/var/lib/redis/$REDIS_PORT"
122 read -p "Please select the data directory for this instance [$_REDIS_DATA_DIR] " REDIS_DATA_DIR
123 if [ -z "$REDIS_DATA_DIR" ] ; then
124 REDIS_DATA_DIR=$_REDIS_DATA_DIR
125 echo "Selected default - $REDIS_DATA_DIR"
126 fi
127fi
128
129if [ ! -x "$REDIS_EXECUTABLE" ] ; then
130 _MANUAL_EXECUTION=true
131 #get the redis executable path
132 _REDIS_EXECUTABLE=`command -v redis-server`
133 read -p "Please select the redis executable path [$_REDIS_EXECUTABLE] " REDIS_EXECUTABLE
134 if [ ! -x "$REDIS_EXECUTABLE" ] ; then
135 REDIS_EXECUTABLE=$_REDIS_EXECUTABLE
136
137 if [ ! -x "$REDIS_EXECUTABLE" ] ; then
138 echo "Mmmmm... it seems like you don't have a redis executable. Did you run make install yet?"
139 exit 1
140 fi
141 fi
142fi
143
144#check the default for redis cli
145CLI_EXEC=`command -v redis-cli`
146if [ -z "$CLI_EXEC" ] ; then
147 CLI_EXEC=`dirname $REDIS_EXECUTABLE`"/redis-cli"
148fi
149
150echo "Selected config:"
151
152echo "Port : $REDIS_PORT"
153echo "Config file : $REDIS_CONFIG_FILE"
154echo "Log file : $REDIS_LOG_FILE"
155echo "Data dir : $REDIS_DATA_DIR"
156echo "Executable : $REDIS_EXECUTABLE"
157echo "Cli Executable : $CLI_EXEC"
158
159if $_MANUAL_EXECUTION == true ; then
160 read -p "Is this ok? Then press ENTER to go on or Ctrl-C to abort." _UNUSED_
161fi
162
163mkdir -p `dirname "$REDIS_CONFIG_FILE"` || die "Could not create redis config directory"
164mkdir -p `dirname "$REDIS_LOG_FILE"` || die "Could not create redis log dir"
165mkdir -p "$REDIS_DATA_DIR" || die "Could not create redis data directory"
166
167#render the templates
168TMP_FILE="/tmp/${REDIS_PORT}.conf"
169DEFAULT_CONFIG="${SCRIPTPATH}/../redis.conf"
170INIT_TPL_FILE="${SCRIPTPATH}/redis_init_script.tpl"
171INIT_SCRIPT_DEST="/etc/init.d/redis_${REDIS_PORT}"
172PIDFILE="/var/run/redis_${REDIS_PORT}.pid"
173
174if [ ! -f "$DEFAULT_CONFIG" ]; then
175 echo "Mmmmm... the default config is missing. Did you switch to the utils directory?"
176 exit 1
177fi
178
179#Generate config file from the default config file as template
180#changing only the stuff we're controlling from this script
181echo "## Generated by install_server.sh ##" > $TMP_FILE
182
183read -r SED_EXPR <<-EOF
184s#^port .\+#port ${REDIS_PORT}#; \
185s#^logfile .\+#logfile ${REDIS_LOG_FILE}#; \
186s#^dir .\+#dir ${REDIS_DATA_DIR}#; \
187s#^pidfile .\+#pidfile ${PIDFILE}#; \
188s#^daemonize no#daemonize yes#;
189EOF
190sed "$SED_EXPR" $DEFAULT_CONFIG >> $TMP_FILE
191
192#cat $TPL_FILE | while read line; do eval "echo \"$line\"" >> $TMP_FILE; done
193cp $TMP_FILE $REDIS_CONFIG_FILE || die "Could not write redis config file $REDIS_CONFIG_FILE"
194
195#Generate sample script from template file
196rm -f $TMP_FILE
197
198#we hard code the configs here to avoid issues with templates containing env vars
199#kinda lame but works!
200REDIS_INIT_HEADER=\
201"#!/bin/sh\n
202#Configurations injected by install_server below....\n\n
203EXEC=$REDIS_EXECUTABLE\n
204CLIEXEC=$CLI_EXEC\n
205PIDFILE=\"$PIDFILE\"\n
206CONF=\"$REDIS_CONFIG_FILE\"\n\n
207REDISPORT=\"$REDIS_PORT\"\n\n
208###############\n\n"
209
210REDIS_CHKCONFIG_INFO=\
211"# REDHAT chkconfig header\n\n
212# chkconfig: - 58 74\n
213# description: redis_${REDIS_PORT} is the redis daemon.\n
214### BEGIN INIT INFO\n
215# Provides: redis_6379\n
216# Required-Start: \$network \$local_fs \$remote_fs\n
217# Required-Stop: \$network \$local_fs \$remote_fs\n
218# Default-Start: 2 3 4 5\n
219# Default-Stop: 0 1 6\n
220# Should-Start: \$syslog \$named\n
221# Should-Stop: \$syslog \$named\n
222# Short-Description: start and stop redis_${REDIS_PORT}\n
223# Description: Redis daemon\n
224### END INIT INFO\n\n"
225
226if command -v chkconfig >/dev/null; then
227 #if we're a box with chkconfig on it we want to include info for chkconfig
228 echo "$REDIS_INIT_HEADER" "$REDIS_CHKCONFIG_INFO" > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE"
229else
230 #combine the header and the template (which is actually a static footer)
231 echo "$REDIS_INIT_HEADER" > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE"
232fi
233
234###
235# Generate sample script from template file
236# - No need to check which system we are on. The init info are comments and
237# do not interfere with update_rc.d systems. Additionally:
238# Ubuntu/debian by default does not come with chkconfig, but does issue a
239# warning if init info is not available.
240
241cat > ${TMP_FILE} <<EOT
242#!/bin/sh
243#Configurations injected by install_server below....
244
245EXEC=$REDIS_EXECUTABLE
246CLIEXEC=$CLI_EXEC
247PIDFILE=$PIDFILE
248CONF="$REDIS_CONFIG_FILE"
249REDISPORT="$REDIS_PORT"
250###############
251# SysV Init Information
252# chkconfig: - 58 74
253# description: redis_${REDIS_PORT} is the redis daemon.
254### BEGIN INIT INFO
255# Provides: redis_${REDIS_PORT}
256# Required-Start: \$network \$local_fs \$remote_fs
257# Required-Stop: \$network \$local_fs \$remote_fs
258# Default-Start: 2 3 4 5
259# Default-Stop: 0 1 6
260# Should-Start: \$syslog \$named
261# Should-Stop: \$syslog \$named
262# Short-Description: start and stop redis_${REDIS_PORT}
263# Description: Redis daemon
264### END INIT INFO
265
266EOT
267cat ${INIT_TPL_FILE} >> ${TMP_FILE}
268
269#copy to /etc/init.d
270cp $TMP_FILE $INIT_SCRIPT_DEST && \
271 chmod +x $INIT_SCRIPT_DEST || die "Could not copy redis init script to $INIT_SCRIPT_DEST"
272echo "Copied $TMP_FILE => $INIT_SCRIPT_DEST"
273
274#Install the service
275echo "Installing service..."
276if command -v chkconfig >/dev/null 2>&1; then
277 # we're chkconfig, so lets add to chkconfig and put in runlevel 345
278 chkconfig --add redis_${REDIS_PORT} && echo "Successfully added to chkconfig!"
279 chkconfig --level 345 redis_${REDIS_PORT} on && echo "Successfully added to runlevels 345!"
280elif command -v update-rc.d >/dev/null 2>&1; then
281 #if we're not a chkconfig box assume we're able to use update-rc.d
282 update-rc.d redis_${REDIS_PORT} defaults && echo "Success!"
283else
284 echo "No supported init tool found."
285fi
286
287/etc/init.d/redis_$REDIS_PORT start || die "Failed starting service..."
288
289#tada
290echo "Installation successful!"
291exit 0
diff --git a/examples/redis-unstable/utils/lru/README b/examples/redis-unstable/utils/lru/README
new file mode 100644
index 0000000..f043b29
--- /dev/null
+++ b/examples/redis-unstable/utils/lru/README
@@ -0,0 +1,19 @@
1The test-lru.rb program can be used in order to check the behavior of the
2Redis approximated LRU algorithm against the theoretical output of true
3LRU algorithm.
4
5In order to use the program you need to recompile Redis setting the define
6REDIS_LRU_CLOCK_RESOLUTION to 1, by editing the file server.h.
7This allows to execute the program in a fast way since the 1 ms resolution
8is enough for all the objects to have a different enough time stamp during
9the test.
10
11The program is executed like this:
12
13 ruby test-lru.rb /tmp/lru.html
14
15You can optionally specify a number of times to run, so that the program
16will output averages of different runs, by adding an additional argument.
17For instance in order to run the test 10 times use:
18
19 ruby test-lru.rb /tmp/lru.html 10
diff --git a/examples/redis-unstable/utils/lru/lfu-simulation.c b/examples/redis-unstable/utils/lru/lfu-simulation.c
new file mode 100644
index 0000000..60105e5
--- /dev/null
+++ b/examples/redis-unstable/utils/lru/lfu-simulation.c
@@ -0,0 +1,158 @@
1#include <stdio.h>
2#include <time.h>
3#include <stdint.h>
4#include <stdlib.h>
5
6int decr_every = 1;
7int keyspace_size = 1000000;
8time_t switch_after = 30; /* Switch access pattern after N seconds. */
9
10struct entry {
11 /* Field that the LFU Redis implementation will have (we have
12 * 24 bits of total space in the object->lru field). */
13 uint8_t counter; /* Logarithmic counter. */
14 uint16_t decrtime; /* (Reduced precision) time of last decrement. */
15
16 /* Fields only useful for visualization. */
17 uint64_t hits; /* Number of real accesses. */
18 time_t ctime; /* Key creation time. */
19};
20
21#define to_16bit_minutes(x) ((x/60) & 65535)
22#define LFU_INIT_VAL 5
23
24/* Compute the difference in minutes between two 16 bit minutes times
25 * obtained with to_16bit_minutes(). Since they can wrap around if
26 * we detect the overflow we account for it as if the counter wrapped
27 * a single time. */
28uint16_t minutes_diff(uint16_t now, uint16_t prev) {
29 if (now >= prev) return now-prev;
30 return 65535-prev+now;
31}
32
33/* Increment a counter logarithmically: the greatest is its value, the
34 * less likely is that the counter is really incremented.
35 * The maximum value of the counter is saturated at 255. */
36uint8_t log_incr(uint8_t counter) {
37 if (counter == 255) return counter;
38 double r = (double)rand()/RAND_MAX;
39 double baseval = counter-LFU_INIT_VAL;
40 if (baseval < 0) baseval = 0;
41 double limit = 1.0/(baseval*10+1);
42 if (r < limit) counter++;
43 return counter;
44}
45
46/* Simulate an access to an entry. */
47void access_entry(struct entry *e) {
48 e->counter = log_incr(e->counter);
49 e->hits++;
50}
51
52/* Return the entry LFU value and as a side effect decrement the
53 * entry value if the decrement time was reached. */
54uint8_t scan_entry(struct entry *e) {
55 if (minutes_diff(to_16bit_minutes(time(NULL)),e->decrtime)
56 >= decr_every)
57 {
58 if (e->counter) {
59 if (e->counter > LFU_INIT_VAL*2) {
60 e->counter /= 2;
61 } else {
62 e->counter--;
63 }
64 }
65 e->decrtime = to_16bit_minutes(time(NULL));
66 }
67 return e->counter;
68}
69
70/* Print the entry info. */
71void show_entry(long pos, struct entry *e) {
72 char *tag = "normal ";
73
74 if (pos >= 10 && pos <= 14) tag = "new no access";
75 if (pos >= 15 && pos <= 19) tag = "new accessed ";
76 if (pos >= keyspace_size -5) tag= "old no access";
77
78 printf("%ld] <%s> frequency:%d decrtime:%d [%lu hits | age:%ld sec]\n",
79 pos, tag, e->counter, e->decrtime, (unsigned long)e->hits,
80 time(NULL) - e->ctime);
81}
82
83int main(void) {
84 time_t start = time(NULL);
85 time_t new_entry_time = start;
86 time_t display_time = start;
87 struct entry *entries = malloc(sizeof(*entries)*keyspace_size);
88 long j;
89
90 /* Initialize. */
91 for (j = 0; j < keyspace_size; j++) {
92 entries[j].counter = LFU_INIT_VAL;
93 entries[j].decrtime = to_16bit_minutes(start);
94 entries[j].hits = 0;
95 entries[j].ctime = time(NULL);
96 }
97
98 while(1) {
99 time_t now = time(NULL);
100 long idx;
101
102 /* Scan N random entries (simulates the eviction under maxmemory). */
103 for (j = 0; j < 3; j++) {
104 scan_entry(entries+(rand()%keyspace_size));
105 }
106
107 /* Access a random entry: use a power-law access pattern up to
108 * 'switch_after' seconds. Then revert to flat access pattern. */
109 if (now-start < switch_after) {
110 /* Power law. */
111 idx = 1;
112 while((rand() % 21) != 0 && idx < keyspace_size) idx *= 2;
113 if (idx > keyspace_size) idx = keyspace_size;
114 idx = rand() % idx;
115 } else {
116 /* Flat. */
117 idx = rand() % keyspace_size;
118 }
119
120 /* Never access entries between position 10 and 14, so that
121 * we simulate what happens to new entries that are never
122 * accessed VS new entries which are accessed in positions
123 * 15-19.
124 *
125 * Also never access last 5 entry, so that we have keys which
126 * are never recreated (old), and never accessed. */
127 if ((idx < 10 || idx > 14) && (idx < keyspace_size-5))
128 access_entry(entries+idx);
129
130 /* Simulate the addition of new entries at positions between
131 * 10 and 19, a random one every 10 seconds. */
132 if (new_entry_time <= now) {
133 idx = 10+(rand()%10);
134 entries[idx].counter = LFU_INIT_VAL;
135 entries[idx].decrtime = to_16bit_minutes(time(NULL));
136 entries[idx].hits = 0;
137 entries[idx].ctime = time(NULL);
138 new_entry_time = now+10;
139 }
140
141 /* Show the first 20 entries and the last 20 entries. */
142 if (display_time != now) {
143 printf("=============================\n");
144 printf("Current minutes time: %d\n", (int)to_16bit_minutes(now));
145 printf("Access method: %s\n",
146 (now-start < switch_after) ? "power-law" : "flat");
147
148 for (j = 0; j < 20; j++)
149 show_entry(j,entries+j);
150
151 for (j = keyspace_size-20; j < keyspace_size; j++)
152 show_entry(j,entries+j);
153 display_time = now;
154 }
155 }
156 return 0;
157}
158
diff --git a/examples/redis-unstable/utils/lru/test-lru.rb b/examples/redis-unstable/utils/lru/test-lru.rb
new file mode 100644
index 0000000..d511e20
--- /dev/null
+++ b/examples/redis-unstable/utils/lru/test-lru.rb
@@ -0,0 +1,223 @@
1require 'rubygems'
2require 'redis'
3
4$runs = []; # Remember the error rate of each run for average purposes.
5$o = {}; # Options set parsing arguments
6
7def testit(filename)
8 r = Redis.new
9 r.config("SET","maxmemory","2000000")
10 if $o[:ttl]
11 r.config("SET","maxmemory-policy","volatile-ttl")
12 else
13 r.config("SET","maxmemory-policy","allkeys-lru")
14 end
15 r.config("SET","maxmemory-samples",5)
16 r.config("RESETSTAT")
17 r.flushall
18
19 html = ""
20 html << <<EOF
21 <html>
22 <body>
23 <style>
24 .box {
25 width:5px;
26 height:5px;
27 float:left;
28 margin: 1px;
29 }
30
31 .old {
32 border: 1px black solid;
33 }
34
35 .new {
36 border: 1px green solid;
37 }
38
39 .otherdb {
40 border: 1px red solid;
41 }
42
43 .ex {
44 background-color: #666;
45 }
46 </style>
47 <pre>
48EOF
49
50 # Fill the DB up to the first eviction.
51 oldsize = r.dbsize
52 id = 0
53 while true
54 id += 1
55 begin
56 r.set(id,"foo")
57 rescue
58 break
59 end
60 newsize = r.dbsize
61 break if newsize == oldsize # A key was evicted? Stop.
62 oldsize = newsize
63 end
64
65 inserted = r.dbsize
66 first_set_max_id = id
67 html << "#{r.dbsize} keys inserted.\n"
68
69 # Access keys sequentially, so that in theory the first part will be expired
70 # and the latter part will not, according to perfect LRU.
71
72 if $o[:ttl]
73 STDERR.puts "Set increasing expire value"
74 (1..first_set_max_id).each{|id|
75 r.expire(id,1000+id)
76 STDERR.print(".") if (id % 150) == 0
77 }
78 else
79 STDERR.puts "Access keys sequentially"
80 (1..first_set_max_id).each{|id|
81 r.get(id)
82 sleep 0.001
83 STDERR.print(".") if (id % 150) == 0
84 }
85 end
86 STDERR.puts
87
88 # Insert more 50% keys. We expect that the new keys will rarely be expired
89 # since their last access time is recent compared to the others.
90 #
91 # Note that we insert the first 100 keys of the new set into DB1 instead
92 # of DB0, so that we can try how cross-DB eviction works.
93 half = inserted/2
94 html << "Insert enough keys to evict half the keys we inserted.\n"
95 add = 0
96
97 otherdb_start_idx = id+1
98 otherdb_end_idx = id+100
99 while true
100 add += 1
101 id += 1
102 if id >= otherdb_start_idx && id <= otherdb_end_idx
103 r.select(1)
104 r.set(id,"foo")
105 r.select(0)
106 else
107 r.set(id,"foo")
108 end
109 break if r.info['evicted_keys'].to_i >= half
110 end
111
112 html << "#{add} additional keys added.\n"
113 html << "#{r.dbsize} keys in DB.\n"
114
115 # Check if evicted keys respect LRU
116 # We consider errors from 1 to N progressively more serious as they violate
117 # more the access pattern.
118
119 errors = 0
120 e = 1
121 error_per_key = 100000.0/first_set_max_id
122 half_set_size = first_set_max_id/2
123 maxerr = 0
124 (1..(first_set_max_id/2)).each{|id|
125 if id >= otherdb_start_idx && id <= otherdb_end_idx
126 r.select(1)
127 exists = r.exists(id)
128 r.select(0)
129 else
130 exists = r.exists(id)
131 end
132 if id < first_set_max_id/2
133 thiserr = error_per_key * ((half_set_size-id).to_f/half_set_size)
134 maxerr += thiserr
135 errors += thiserr if exists
136 elsif id >= first_set_max_id/2
137 thiserr = error_per_key * ((id-half_set_size).to_f/half_set_size)
138 maxerr += thiserr
139 errors += thiserr if !exists
140 end
141 }
142 errors = errors*100/maxerr
143
144 STDERR.puts "Test finished with #{errors}% error! Generating HTML on stdout."
145
146 html << "#{errors}% error!\n"
147 html << "</pre>"
148 $runs << errors
149
150 # Generate the graphical representation
151 (1..id).each{|id|
152 # Mark first set and added items in a different way.
153 c = "box"
154 if id >= otherdb_start_idx && id <= otherdb_end_idx
155 c << " otherdb"
156 elsif id <= first_set_max_id
157 c << " old"
158 else
159 c << " new"
160 end
161
162 # Add class if exists
163 if id >= otherdb_start_idx && id <= otherdb_end_idx
164 r.select(1)
165 exists = r.exists(id)
166 r.select(0)
167 else
168 exists = r.exists(id)
169 end
170
171 c << " ex" if exists
172 html << "<div title=\"#{id}\" class=\"#{c}\"></div>"
173 }
174
175 # Close HTML page
176
177 html << <<EOF
178 </body>
179 </html>
180EOF
181
182 f = File.open(filename,"w")
183 f.write(html)
184 f.close
185end
186
187def print_avg
188 avg = ($runs.reduce {|a,b| a+b}) / $runs.length
189 puts "#{$runs.length} runs, AVG is #{avg}"
190end
191
192if ARGV.length < 1
193 STDERR.puts "Usage: ruby test-lru.rb <html-output-filename> [--runs <count>] [--ttl]"
194 STDERR.puts "Options:"
195 STDERR.puts " --runs <count> Execute the test <count> times."
196 STDERR.puts " --ttl Set keys with increasing TTL values"
197 STDERR.puts " (starting from 1000 seconds) in order to"
198 STDERR.puts " test the volatile-lru policy."
199 exit 1
200end
201
202filename = ARGV[0]
203$o[:numruns] = 1
204
205# Options parsing
206i = 1
207while i < ARGV.length
208 if ARGV[i] == '--runs'
209 $o[:numruns] = ARGV[i+1].to_i
210 i+= 1
211 elsif ARGV[i] == '--ttl'
212 $o[:ttl] = true
213 else
214 STDERR.puts "Unknown option #{ARGV[i]}"
215 exit 1
216 end
217 i+= 1
218end
219
220$o[:numruns].times {
221 testit(filename)
222 print_avg if $o[:numruns] != 1
223}
diff --git a/examples/redis-unstable/utils/redis-copy.rb b/examples/redis-unstable/utils/redis-copy.rb
new file mode 100644
index 0000000..816b1e4
--- /dev/null
+++ b/examples/redis-unstable/utils/redis-copy.rb
@@ -0,0 +1,38 @@
1# redis-copy.rb - Copyright (C) 2009-Present Redis Ltd. All rights reserved.
2#
3# Licensed under your choice of (a) the Redis Source Available License 2.0
4# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
5# GNU Affero General Public License v3 (AGPLv3).
6#
7# Copy the whole dataset from one Redis instance to another one
8#
9# WARNING: this utility is deprecated and serves as a legacy adapter
10# for the more-robust redis-copy gem.
11
12require 'shellwords'
13
14def redisCopy(opts={})
15 src = "#{opts[:srchost]}:#{opts[:srcport]}"
16 dst = "#{opts[:dsthost]}:#{opts[:dstport]}"
17 `redis-copy #{src.shellescape} #{dst.shellescape}`
18rescue Errno::ENOENT
19 $stderr.puts 'This utility requires the redis-copy executable',
20 'from the redis-copy gem on https://rubygems.org',
21 'To install it, run `gem install redis-copy`.'
22 exit 1
23end
24
25$stderr.puts "This utility is deprecated. Use the redis-copy gem instead."
26if ARGV.length != 4
27 puts "Usage: redis-copy.rb <srchost> <srcport> <dsthost> <dstport>"
28 exit 1
29end
30puts "WARNING: it's up to you to FLUSHDB the destination host before to continue, press any key when ready."
31STDIN.gets
32srchost = ARGV[0]
33srcport = ARGV[1]
34dsthost = ARGV[2]
35dstport = ARGV[3]
36puts "Copying #{srchost}:#{srcport} into #{dsthost}:#{dstport}"
37redisCopy(:srchost => srchost, :srcport => srcport.to_i,
38 :dsthost => dsthost, :dstport => dstport.to_i)
diff --git a/examples/redis-unstable/utils/redis-sha1.rb b/examples/redis-unstable/utils/redis-sha1.rb
new file mode 100644
index 0000000..dc87c77
--- /dev/null
+++ b/examples/redis-unstable/utils/redis-sha1.rb
@@ -0,0 +1,55 @@
1# redis-sha1.rb - Copyright (C) 2009-Present Redis Ltd. All rights reserved.
2#
3# Licensed under your choice of (a) the Redis Source Available License 2.0
4# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
5# GNU Affero General Public License v3 (AGPLv3).
6#
7# Performs the SHA1 sum of the whole dataset.
8# This is useful to spot bugs in persistence related code and to make sure
9# Slaves and Masters are in SYNC.
10#
11# If you hack this code make sure to sort keys and set elements as this are
12# unsorted elements. Otherwise the sum may differ with equal dataset.
13
14require 'rubygems'
15require 'redis'
16require 'digest/sha1'
17
18def redisSha1(opts={})
19 sha1=""
20 r = Redis.new(opts)
21 r.keys('*').sort.each{|k|
22 vtype = r.type?(k)
23 if vtype == "string"
24 len = 1
25 sha1 = Digest::SHA1.hexdigest(sha1+k)
26 sha1 = Digest::SHA1.hexdigest(sha1+r.get(k))
27 elsif vtype == "list"
28 len = r.llen(k)
29 if len != 0
30 sha1 = Digest::SHA1.hexdigest(sha1+k)
31 sha1 = Digest::SHA1.hexdigest(sha1+r.list_range(k,0,-1).join("\x01"))
32 end
33 elsif vtype == "set"
34 len = r.scard(k)
35 if len != 0
36 sha1 = Digest::SHA1.hexdigest(sha1+k)
37 sha1 = Digest::SHA1.hexdigest(sha1+r.set_members(k).to_a.sort.join("\x02"))
38 end
39 elsif vtype == "zset"
40 len = r.zcard(k)
41 if len != 0
42 sha1 = Digest::SHA1.hexdigest(sha1+k)
43 sha1 = Digest::SHA1.hexdigest(sha1+r.zrange(k,0,-1).join("\x01"))
44 end
45 end
46 # puts "#{k} => #{sha1}" if len != 0
47 }
48 sha1
49end
50
51host = ARGV[0] || "127.0.0.1"
52port = ARGV[1] || "6379"
53db = ARGV[2] || "0"
54puts "Performing SHA1 of Redis server #{host} #{port} DB: #{db}"
55p "Dataset SHA1: #{redisSha1(:host => host, :port => port.to_i, :db => db)}"
diff --git a/examples/redis-unstable/utils/redis_init_script b/examples/redis-unstable/utils/redis_init_script
new file mode 100755
index 0000000..006db87
--- /dev/null
+++ b/examples/redis-unstable/utils/redis_init_script
@@ -0,0 +1,50 @@
1#!/bin/sh
2#
3# Simple Redis init.d script conceived to work on Linux systems
4# as it does use of the /proc filesystem.
5
6### BEGIN INIT INFO
7# Provides: redis_6379
8# Default-Start: 2 3 4 5
9# Default-Stop: 0 1 6
10# Short-Description: Redis data structure server
11# Description: Redis data structure server. See https://redis.io
12### END INIT INFO
13
14REDISPORT=6379
15EXEC=/usr/local/bin/redis-server
16CLIEXEC=/usr/local/bin/redis-cli
17
18PIDFILE=/var/run/redis_${REDISPORT}.pid
19CONF="/etc/redis/${REDISPORT}.conf"
20
21case "$1" in
22 start)
23 if [ -f $PIDFILE ]
24 then
25 echo "$PIDFILE exists, process is already running or crashed"
26 else
27 echo "Starting Redis server..."
28 $EXEC $CONF
29 fi
30 ;;
31 stop)
32 if [ ! -f $PIDFILE ]
33 then
34 echo "$PIDFILE does not exist, process is not running"
35 else
36 PID=$(cat $PIDFILE)
37 echo "Stopping ..."
38 $CLIEXEC -p $REDISPORT shutdown
39 while [ -x /proc/${PID} ]
40 do
41 echo "Waiting for Redis to shutdown ..."
42 sleep 1
43 done
44 echo "Redis stopped"
45 fi
46 ;;
47 *)
48 echo "Please use start or stop as first argument"
49 ;;
50esac
diff --git a/examples/redis-unstable/utils/redis_init_script.tpl b/examples/redis-unstable/utils/redis_init_script.tpl
new file mode 100755
index 0000000..2e5b613
--- /dev/null
+++ b/examples/redis-unstable/utils/redis_init_script.tpl
@@ -0,0 +1,44 @@
1
2case "$1" in
3 start)
4 if [ -f $PIDFILE ]
5 then
6 echo "$PIDFILE exists, process is already running or crashed"
7 else
8 echo "Starting Redis server..."
9 $EXEC $CONF
10 fi
11 ;;
12 stop)
13 if [ ! -f $PIDFILE ]
14 then
15 echo "$PIDFILE does not exist, process is not running"
16 else
17 PID=$(cat $PIDFILE)
18 echo "Stopping ..."
19 $CLIEXEC -p $REDISPORT shutdown
20 while [ -x /proc/${PID} ]
21 do
22 echo "Waiting for Redis to shutdown ..."
23 sleep 1
24 done
25 echo "Redis stopped"
26 fi
27 ;;
28 status)
29 PID=$(cat $PIDFILE)
30 if [ ! -x /proc/${PID} ]
31 then
32 echo 'Redis is not running'
33 else
34 echo "Redis is running ($PID)"
35 fi
36 ;;
37 restart)
38 $0 stop
39 $0 start
40 ;;
41 *)
42 echo "Please use start, stop, restart or status as first argument"
43 ;;
44esac
diff --git a/examples/redis-unstable/utils/releasetools/01_create_tarball.sh b/examples/redis-unstable/utils/releasetools/01_create_tarball.sh
new file mode 100755
index 0000000..366a61e
--- /dev/null
+++ b/examples/redis-unstable/utils/releasetools/01_create_tarball.sh
@@ -0,0 +1,14 @@
1#!/bin/sh
2if [ $# != "1" ]
3then
4 echo "Usage: ./utils/releasetools/01_create_tarball.sh <version_tag>"
5 exit 1
6fi
7
8TAG=$1
9TARNAME="redis-${TAG}.tar"
10echo "Generating /tmp/${TARNAME}"
11git archive $TAG --prefix redis-${TAG}/ > /tmp/$TARNAME || exit 1
12echo "Gizipping the archive"
13rm -f /tmp/$TARNAME.gz
14gzip -9 /tmp/$TARNAME
diff --git a/examples/redis-unstable/utils/releasetools/02_upload_tarball.sh b/examples/redis-unstable/utils/releasetools/02_upload_tarball.sh
new file mode 100755
index 0000000..ef1e777
--- /dev/null
+++ b/examples/redis-unstable/utils/releasetools/02_upload_tarball.sh
@@ -0,0 +1,23 @@
1#!/bin/bash
2if [ $# != "1" ]
3then
4 echo "Usage: ./utils/releasetools/02_upload_tarball.sh <version_tag>"
5 exit 1
6fi
7
8echo "Uploading..."
9scp /tmp/redis-${1}.tar.gz ubuntu@host.redis.io:/var/www/download/releases/
10echo "Updating web site... "
11echo "Please check the github action tests for the release."
12echo "Press any key if it is a stable release, or Ctrl+C to abort"
13read x
14ssh ubuntu@host.redis.io "cd /var/www/download;
15 rm -rf redis-${1}.tar.gz;
16 wget http://download.redis.io/releases/redis-${1}.tar.gz;
17 tar xvzf redis-${1}.tar.gz;
18 rm -rf redis-stable;
19 mv redis-${1} redis-stable;
20 tar cvzf redis-stable.tar.gz redis-stable;
21 rm -rf redis-${1}.tar.gz;
22 shasum -a 256 redis-stable.tar.gz > redis-stable.tar.gz.SHA256SUM;
23 "
diff --git a/examples/redis-unstable/utils/releasetools/03_test_release.sh b/examples/redis-unstable/utils/releasetools/03_test_release.sh
new file mode 100755
index 0000000..493d0b7
--- /dev/null
+++ b/examples/redis-unstable/utils/releasetools/03_test_release.sh
@@ -0,0 +1,28 @@
1#!/bin/sh
2set -e
3if [ $# != "1" ]
4then
5 echo "Usage: ./utils/releasetools/03_test_release.sh <version_tag>"
6 exit 1
7fi
8
9TAG=$1
10TARNAME="redis-${TAG}.tar.gz"
11DOWNLOADURL="http://download.redis.io/releases/${TARNAME}"
12
13echo "Doing sanity test on the actual tarball"
14
15cd /tmp
16rm -rf test_release_tmp_dir
17mkdir test_release_tmp_dir
18cd test_release_tmp_dir
19rm -f $TARNAME
20rm -rf redis-${TAG}
21wget $DOWNLOADURL
22tar xvzf $TARNAME
23cd redis-${TAG}
24make
25./runtest
26./runtest-sentinel
27./runtest-cluster
28./runtest-moduleapi
diff --git a/examples/redis-unstable/utils/releasetools/04_release_hash.sh b/examples/redis-unstable/utils/releasetools/04_release_hash.sh
new file mode 100755
index 0000000..d932928
--- /dev/null
+++ b/examples/redis-unstable/utils/releasetools/04_release_hash.sh
@@ -0,0 +1,13 @@
1#!/bin/bash
2if [ $# != "1" ]
3then
4 echo "Usage: ./utils/releasetools/04_release_hash.sh <version_tag>"
5 exit 1
6fi
7
8SHA=$(curl -s http://download.redis.io/releases/redis-${1}.tar.gz | shasum -a 256 | cut -f 1 -d' ')
9ENTRY="hash redis-${1}.tar.gz sha256 $SHA http://download.redis.io/releases/redis-${1}.tar.gz"
10echo $ENTRY >> ../redis-hashes/README
11echo "Press any key to commit, Ctrl-C to abort)."
12read yes
13(cd ../redis-hashes; git commit -a -m "${1} hash."; git push)
diff --git a/examples/redis-unstable/utils/releasetools/changelog.tcl b/examples/redis-unstable/utils/releasetools/changelog.tcl
new file mode 100755
index 0000000..2288794
--- /dev/null
+++ b/examples/redis-unstable/utils/releasetools/changelog.tcl
@@ -0,0 +1,35 @@
1#!/usr/bin/env tclsh
2
3if {[llength $::argv] != 2 && [llength $::argv] != 3} {
4 puts "Usage: $::argv0 <branch> <version> \[<num-commits>\]"
5 exit 1
6}
7
8set branch [lindex $::argv 0]
9set ver [lindex $::argv 1]
10if {[llength $::argv] == 3} {
11 set count [lindex ::$argv 2]
12} else {
13 set count 100
14}
15
16set template {
17================================================================================
18Redis %ver% Released %date%
19================================================================================
20
21Upgrade urgency <URGENCY>: <DESCRIPTION>
22}
23
24set template [string trim $template]
25append template "\n\n"
26set date [clock format [clock seconds]]
27set template [string map [list %ver% $ver %date% $date] $template]
28
29append template [exec git log $branch~$count..$branch "--format=format:%an in commit %h:%n %s" --shortstat]
30
31#Older, more verbose version.
32#
33#append template [exec git log $branch~30..$branch "--format=format:+-------------------------------------------------------------------------------%n| %s%n| By %an, %ai%n+--------------------------------------------------------------------------------%nhttps://github.com/redis/redis/commit/%H%n%n%b" --stat]
34
35puts $template
diff --git a/examples/redis-unstable/utils/reply_schema_linter.js b/examples/redis-unstable/utils/reply_schema_linter.js
new file mode 100644
index 0000000..e2358d4
--- /dev/null
+++ b/examples/redis-unstable/utils/reply_schema_linter.js
@@ -0,0 +1,31 @@
1function validate_schema(command_schema) {
2 var error_status = false
3 const Ajv = require("ajv/dist/2019")
4 const ajv = new Ajv({strict: true, strictTuples: false})
5 let json = require('../src/commands/'+ command_schema);
6 for (var item in json) {
7 const schema = json[item].reply_schema
8 if (schema == undefined)
9 continue;
10 try {
11 ajv.compile(schema)
12 } catch (error) {
13 console.error(command_schema + " : " + error.toString())
14 error_status = true
15 }
16 }
17 return error_status
18}
19
20const schema_directory_path = './src/commands'
21const path = require('path')
22var fs = require('fs');
23var files = fs.readdirSync(schema_directory_path);
24jsonFiles = files.filter(el => path.extname(el) === '.json')
25var error_status = false
26jsonFiles.forEach(function(file){
27 if (validate_schema(file))
28 error_status = true
29})
30if (error_status)
31 process.exit(1)
diff --git a/examples/redis-unstable/utils/req-res-log-validator.py b/examples/redis-unstable/utils/req-res-log-validator.py
new file mode 100755
index 0000000..0f00935
--- /dev/null
+++ b/examples/redis-unstable/utils/req-res-log-validator.py
@@ -0,0 +1,364 @@
1#!/usr/bin/env python3
2import os
3import glob
4import json
5import sys
6
7import jsonschema
8import subprocess
9import redis
10import time
11import argparse
12import multiprocessing
13import collections
14import io
15import traceback
16from datetime import timedelta
17from functools import partial
18try:
19 from jsonschema import Draft201909Validator as schema_validator
20except ImportError:
21 from jsonschema import Draft7Validator as schema_validator
22
23"""
24The purpose of this file is to validate the reply_schema values of COMMAND DOCS.
25Basically, this is what it does:
261. Goes over req-res files, generated by redis-servers, spawned by the testsuite (see logreqres.c)
272. For each request-response pair, it validates the response against the request's reply_schema (obtained from COMMAND DOCS)
28
29This script spins up a redis-server and a redis-cli in order to obtain COMMAND DOCS.
30
31In order to use this file you must run the redis testsuite with the following flags:
32./runtest --dont-clean --force-resp3 --log-req-res
33
34And then:
35./utils/req-res-log-validator.py
36
37The script will fail only if:
381. One or more of the replies doesn't comply with its schema.
392. One or more of the commands in COMMANDS DOCS doesn't have the reply_schema field (with --fail-missing-reply-schemas)
403. The testsuite didn't execute all of the commands (with --fail-commands-not-all-hit)
41
42Future validations:
431. Fail the script if one or more of the branches of the reply schema (e.g. oneOf, anyOf) was not hit.
44"""
45
46IGNORED_COMMANDS = {
47 # Commands that don't work in a req-res manner (see logreqres.c)
48 "debug", # because of DEBUG SEGFAULT
49 "sync",
50 "psync",
51 "monitor",
52 "subscribe",
53 "unsubscribe",
54 "ssubscribe",
55 "sunsubscribe",
56 "psubscribe",
57 "punsubscribe",
58 # Commands to which we decided not write a reply schema
59 "pfdebug",
60 "lolwut",
61 # TODO: for vector-sets module
62 "VADD",
63 "VCARD",
64 "VDIM",
65 "VEMB",
66 "VGETATTR",
67 "VINFO",
68 "VLINKS",
69 "VRANDMEMBER",
70 "VREM",
71 "VSETATTR",
72 "VSIM",
73 "VISMEMBER",
74 "VRANGE",
75}
76
77class Request(object):
78 """
79 This class represents a Redis request (AKA command, argv)
80 """
81 def __init__(self, f, docs, line_counter):
82 """
83 Read lines from `f` (generated by logreqres.c) and populates the argv array
84 """
85 self.command = None
86 self.schema = None
87 self.argv = []
88
89 while True:
90 line = f.readline()
91 line_counter[0] += 1
92 if not line:
93 break
94 length = int(line)
95 arg = str(f.read(length))
96 f.read(2) # read \r\n
97 line_counter[0] += 1
98 if arg == "__argv_end__":
99 break
100 self.argv.append(arg)
101
102 if not self.argv:
103 return
104
105 self.command = self.argv[0].lower()
106 doc = docs.get(self.command, {})
107 if not doc and len(self.argv) > 1:
108 self.command = f"{self.argv[0].lower()}|{self.argv[1].lower()}"
109 doc = docs.get(self.command, {})
110
111 if not doc:
112 self.command = None
113 return
114
115 self.schema = doc.get("reply_schema")
116
117 def __str__(self):
118 return json.dumps(self.argv)
119
120
121class Response(object):
122 """
123 This class represents a Redis response in RESP3
124 """
125 def __init__(self, f, line_counter):
126 """
127 Read lines from `f` (generated by logreqres.c) and build the JSON representing the response in RESP3
128 """
129 self.error = False
130 self.queued = False
131 self.json = None
132
133 line = f.readline()[:-2]
134 line_counter[0] += 1
135 if line[0] == '+':
136 self.json = line[1:]
137 if self.json == "QUEUED":
138 self.queued = True
139 elif line[0] == '-':
140 self.json = line[1:]
141 self.error = True
142 elif line[0] == '$':
143 self.json = str(f.read(int(line[1:])))
144 f.read(2) # read \r\n
145 line_counter[0] += 1
146 elif line[0] == ':':
147 self.json = int(line[1:])
148 elif line[0] == ',':
149 self.json = float(line[1:])
150 elif line[0] == '_':
151 self.json = None
152 elif line[0] == '#':
153 self.json = line[1] == 't'
154 elif line[0] == '!':
155 self.json = str(f.read(int(line[1:])))
156 f.read(2) # read \r\n
157 line_counter[0] += 1
158 self.error = True
159 elif line[0] == '=':
160 self.json = str(f.read(int(line[1:])))[4:] # skip "txt:" or "mkd:"
161 f.read(2) # read \r\n
162 line_counter[0] += 1 + self.json.count("\r\n")
163 elif line[0] == '(':
164 self.json = line[1:] # big-number is actually a string
165 elif line[0] in ['*', '~', '>']: # unfortunately JSON doesn't tell the difference between a list and a set
166 self.json = []
167 count = int(line[1:])
168 for i in range(count):
169 ele = Response(f, line_counter)
170 self.json.append(ele.json)
171 elif line[0] in ['%', '|']:
172 self.json = {}
173 count = int(line[1:])
174 for i in range(count):
175 field = Response(f, line_counter)
176 # Redis allows fields to be non-strings but JSON doesn't.
177 # Luckily, for any kind of response we can validate, the fields are
178 # always strings (example: XINFO STREAM)
179 # The reason we can't always convert to string is because of DEBUG PROTOCOL MAP
180 # which anyway doesn't have a schema
181 if isinstance(field.json, str):
182 field = field.json
183 value = Response(f, line_counter)
184 self.json[field] = value.json
185 if line[0] == '|':
186 # We don't care about the attributes, read the real response
187 real_res = Response(f, line_counter)
188 self.__dict__.update(real_res.__dict__)
189
190
191 def __str__(self):
192 return json.dumps(self.json)
193
194
195def process_file(docs, path):
196 """
197 This function processes a single file generated by logreqres.c
198 """
199 line_counter = [0] # A list with one integer: to force python to pass it by reference
200 command_counter = dict()
201
202 print(f"Processing {path} ...")
203
204 # Convert file to StringIO in order to minimize IO operations
205 with open(path, "r", newline="\r\n", encoding="latin-1") as f:
206 content = f.read()
207
208 with io.StringIO(content) as fakefile:
209 while True:
210 try:
211 req = Request(fakefile, docs, line_counter)
212 if not req.argv:
213 # EOF
214 break
215 res = Response(fakefile, line_counter)
216 except json.decoder.JSONDecodeError as err:
217 print(f"JSON decoder error while processing {path}:{line_counter[0]}: {err}")
218 print(traceback.format_exc())
219 raise
220 except Exception as err:
221 print(f"General error while processing {path}:{line_counter[0]}: {err}")
222 print(traceback.format_exc())
223 raise
224
225 if not req.command:
226 # Unknown command
227 continue
228
229 command_counter[req.command] = command_counter.get(req.command, 0) + 1
230
231 if res.error or res.queued:
232 continue
233
234 if req.command in IGNORED_COMMANDS:
235 continue
236
237 try:
238 jsonschema.validate(instance=res.json, schema=req.schema, cls=schema_validator)
239 except (jsonschema.ValidationError, jsonschema.exceptions.SchemaError) as err:
240 print(f"JSON schema validation error on {path}: {err}")
241 print(f"argv: {req.argv}")
242 try:
243 print(f"Response: {res}")
244 except UnicodeDecodeError as err:
245 print("Response: (unprintable)")
246 print(f"Schema: {json.dumps(req.schema, indent=2)}")
247 print(traceback.format_exc())
248 raise
249
250 return command_counter
251
252
253def fetch_schemas(cli, port, args, docs):
254 redis_proc = subprocess.Popen(args, stdout=subprocess.PIPE)
255
256 while True:
257 try:
258 print('Connecting to Redis...')
259 r = redis.Redis(port=port)
260 r.ping()
261 break
262 except Exception as e:
263 time.sleep(0.1)
264
265 print('Connected')
266
267 cli_proc = subprocess.Popen([cli, '-p', str(port), '--json', 'command', 'docs'], stdout=subprocess.PIPE)
268 stdout, stderr = cli_proc.communicate()
269 docs_response = json.loads(stdout)
270
271 for name, doc in docs_response.items():
272 if "subcommands" in doc:
273 for subname, subdoc in doc["subcommands"].items():
274 docs[subname] = subdoc
275 else:
276 docs[name] = doc
277
278 redis_proc.terminate()
279 redis_proc.wait()
280
281
282if __name__ == '__main__':
283 # Figure out where the sources are
284 srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src")
285 testdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../tests")
286
287 parser = argparse.ArgumentParser()
288 parser.add_argument('--server', type=str, default='%s/redis-server' % srcdir)
289 parser.add_argument('--port', type=int, default=6534)
290 parser.add_argument('--cli', type=str, default='%s/redis-cli' % srcdir)
291 parser.add_argument('--module', type=str, action='append', default=[])
292 parser.add_argument('--verbose', action='store_true')
293 parser.add_argument('--fail-commands-not-all-hit', action='store_true')
294 parser.add_argument('--fail-missing-reply-schemas', action='store_true')
295 args = parser.parse_args()
296
297 docs = dict()
298
299 # Fetch schemas from a Redis instance
300 print('Starting Redis server')
301 redis_args = [args.server, '--port', str(args.port)]
302 for module in args.module:
303 redis_args += ['--loadmodule', 'tests/modules/%s.so' % module]
304
305 fetch_schemas(args.cli, args.port, redis_args, docs)
306
307 # Fetch schemas from a sentinel
308 print('Starting Redis sentinel')
309
310 # Sentinel needs a config file to start
311 config_file = "tmpsentinel.conf"
312 open(config_file, 'a').close()
313
314 sentinel_args = [args.server, config_file, '--port', str(args.port), "--sentinel"]
315 fetch_schemas(args.cli, args.port, sentinel_args, docs)
316 os.unlink(config_file)
317
318 missing_schema = [k for k, v in docs.items()
319 if "reply_schema" not in v and k not in IGNORED_COMMANDS]
320 if missing_schema:
321 print("WARNING! The following commands are missing a reply_schema:")
322 for k in sorted(missing_schema):
323 print(f" {k}")
324 if args.fail_missing_reply_schemas:
325 print("ERROR! at least one command does not have a reply_schema")
326 sys.exit(1)
327
328 start = time.time()
329
330 # Obtain all the files to processes
331 paths = []
332 for path in glob.glob('%s/tmp/*/*.reqres' % testdir):
333 paths.append(path)
334
335 for path in glob.glob('%s/cluster/tmp/*/*.reqres' % testdir):
336 paths.append(path)
337
338 for path in glob.glob('%s/sentinel/tmp/*/*.reqres' % testdir):
339 paths.append(path)
340
341 counter = collections.Counter()
342 # Spin several processes to handle the files in parallel
343 with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:
344 func = partial(process_file, docs)
345 # pool.map blocks until all the files have been processed
346 for result in pool.map(func, paths):
347 counter.update(result)
348 command_counter = dict(counter)
349
350 elapsed = time.time() - start
351 print(f"Done. ({timedelta(seconds=elapsed)})")
352 print("Hits per command:")
353 for k, v in sorted(command_counter.items()):
354 print(f" {k}: {v}")
355 not_hit = set(set(docs.keys()) - set(command_counter.keys()) - set(IGNORED_COMMANDS))
356 if not_hit:
357 if args.verbose:
358 print("WARNING! The following commands were not hit at all:")
359 for k in sorted(not_hit):
360 print(f" {k}")
361 if args.fail_commands_not_all_hit:
362 print("ERROR! at least one command was not hit by the tests")
363 sys.exit(1)
364
diff --git a/examples/redis-unstable/utils/req-res-validator/requirements.txt b/examples/redis-unstable/utils/req-res-validator/requirements.txt
new file mode 100644
index 0000000..0e3024b
--- /dev/null
+++ b/examples/redis-unstable/utils/req-res-validator/requirements.txt
@@ -0,0 +1,2 @@
1jsonschema==4.17.3
2redis==4.5.1 \ No newline at end of file
diff --git a/examples/redis-unstable/utils/speed-regression.tcl b/examples/redis-unstable/utils/speed-regression.tcl
new file mode 100755
index 0000000..62b5ec8
--- /dev/null
+++ b/examples/redis-unstable/utils/speed-regression.tcl
@@ -0,0 +1,133 @@
1#!/usr/bin/env tclsh8.5
2# Copyright (C) 2011-Present Redis Ltd. All rights reserved.
3#
4# Licensed under your choice of (a) the Redis Source Available License 2.0
5# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
6# GNU Affero General Public License v3 (AGPLv3).
7
8source ../tests/support/redis.tcl
9set ::port 12123
10set ::tests {PING,SET,GET,INCR,LPUSH,LPOP,SADD,SPOP,LRANGE_100,LRANGE_600,MSET}
11set ::datasize 16
12set ::requests 100000
13
14proc run-tests branches {
15 set runs {}
16 set branch_id 0
17 foreach b $branches {
18 cd ../src
19 puts "Benchmarking $b"
20 exec -ignorestderr git checkout $b 2> /dev/null
21 exec -ignorestderr make clean 2> /dev/null
22 puts " compiling..."
23 exec -ignorestderr make 2> /dev/null
24
25 if {$branch_id == 0} {
26 puts " copy redis-benchmark from unstable to /tmp..."
27 exec -ignorestderr cp ./redis-benchmark /tmp
28 incr branch_id
29 continue
30 }
31
32 # Start the Redis server
33 puts " starting the server... [exec ./redis-server -v]"
34 set pids [exec echo "port $::port\nloglevel warning\n" | ./redis-server - > /dev/null 2> /dev/null &]
35 puts " pids: $pids"
36 after 1000
37 puts " running the benchmark"
38
39 set r [redis 127.0.0.1 $::port]
40 set i [$r info]
41 puts " redis INFO shows version: [lindex [split $i] 0]"
42 $r close
43
44 set output [exec /tmp/redis-benchmark -n $::requests -t $::tests -d $::datasize --csv -p $::port]
45 lappend runs $b $output
46 puts " killing server..."
47 catch {exec kill -9 [lindex $pids 0]}
48 catch {exec kill -9 [lindex $pids 1]}
49 incr branch_id
50 }
51 return $runs
52}
53
54proc get-result-with-name {output name} {
55 foreach line [split $output "\n"] {
56 lassign [split $line ","] key value
57 set key [string tolower [string range $key 1 end-1]]
58 set value [string range $value 1 end-1]
59 if {$key eq [string tolower $name]} {
60 return $value
61 }
62 }
63 return "n/a"
64}
65
66proc get-test-names output {
67 set names {}
68 foreach line [split $output "\n"] {
69 lassign [split $line ","] key value
70 set key [string tolower [string range $key 1 end-1]]
71 lappend names $key
72 }
73 return $names
74}
75
76proc combine-results {results} {
77 set tests [get-test-names [lindex $results 1]]
78 foreach test $tests {
79 puts $test
80 foreach {branch output} $results {
81 puts [format "%-20s %s" \
82 $branch [get-result-with-name $output $test]]
83 }
84 puts {}
85 }
86}
87
88proc main {} {
89 # Note: the first branch is only used in order to get the redis-benchmark
90 # executable. Tests are performed starting from the second branch.
91 set branches {
92 slowset 2.2.0 2.4.0 unstable slowset
93 }
94 set results [run-tests $branches]
95 puts "\n"
96 puts "# Test results: datasize=$::datasize requests=$::requests"
97 puts [combine-results $results]
98}
99
100# Force the user to run the script from the 'utils' directory.
101if {![file exists speed-regression.tcl]} {
102 puts "Please make sure to run speed-regression.tcl while inside /utils."
103 puts "Example: cd utils; ./speed-regression.tcl"
104 exit 1
105}
106
107# Make sure there is not already a server running on port 12123
108set is_not_running [catch {set r [redis 127.0.0.1 $::port]}]
109if {!$is_not_running} {
110 puts "Sorry, you have a running server on port $::port"
111 exit 1
112}
113
114# parse arguments
115for {set j 0} {$j < [llength $argv]} {incr j} {
116 set opt [lindex $argv $j]
117 set arg [lindex $argv [expr $j+1]]
118 if {$opt eq {--tests}} {
119 set ::tests $arg
120 incr j
121 } elseif {$opt eq {--datasize}} {
122 set ::datasize $arg
123 incr j
124 } elseif {$opt eq {--requests}} {
125 set ::requests $arg
126 incr j
127 } else {
128 puts "Wrong argument: $opt"
129 exit 1
130 }
131}
132
133main
diff --git a/examples/redis-unstable/utils/srandmember/README.md b/examples/redis-unstable/utils/srandmember/README.md
new file mode 100644
index 0000000..d3da1e8
--- /dev/null
+++ b/examples/redis-unstable/utils/srandmember/README.md
@@ -0,0 +1,14 @@
1The utilities in this directory plot the distribution of SRANDMEMBER to
2evaluate how fair it is.
3
4See http://theshfl.com/redis_sets for more information on the topic that lead
5to such investigation fix.
6
7showdist.rb -- shows the distribution of the frequency elements are returned.
8 The x axis is the number of times elements were returned, and
9 the y axis is how many elements were returned with such
10 frequency.
11
12showfreq.rb -- shows the frequency each element was returned.
13 The x axis is the element number.
14 The y axis is the times it was returned.
diff --git a/examples/redis-unstable/utils/srandmember/showdist.rb b/examples/redis-unstable/utils/srandmember/showdist.rb
new file mode 100644
index 0000000..2435857
--- /dev/null
+++ b/examples/redis-unstable/utils/srandmember/showdist.rb
@@ -0,0 +1,33 @@
1require 'redis'
2
3r = Redis.new
4r.select(9)
5r.del("myset");
6r.sadd("myset",(0..999).to_a)
7freq = {}
8100.times {
9 res = r.pipelined {
10 1000.times {
11 r.srandmember("myset")
12 }
13 }
14 res.each{|ele|
15 freq[ele] = 0 if freq[ele] == nil
16 freq[ele] += 1
17 }
18}
19
20# Convert into frequency distribution
21dist = {}
22freq.each{|item,count|
23 dist[count] = 0 if dist[count] == nil
24 dist[count] += 1
25}
26
27min = dist.keys.min
28max = dist.keys.max
29(min..max).each{|x|
30 count = dist[x]
31 count = 0 if count == nil
32 puts "#{x} -> #{"*"*count}"
33}
diff --git a/examples/redis-unstable/utils/srandmember/showfreq.rb b/examples/redis-unstable/utils/srandmember/showfreq.rb
new file mode 100644
index 0000000..625519c
--- /dev/null
+++ b/examples/redis-unstable/utils/srandmember/showfreq.rb
@@ -0,0 +1,23 @@
1require 'redis'
2
3r = Redis.new
4r.select(9)
5r.del("myset");
6r.sadd("myset",(0..999).to_a)
7freq = {}
8500.times {
9 res = r.pipelined {
10 1000.times {
11 r.srandmember("myset")
12 }
13 }
14 res.each{|ele|
15 freq[ele] = 0 if freq[ele] == nil
16 freq[ele] += 1
17 }
18}
19
20# Print the frequency each element was yield to process it with gnuplot
21freq.each{|item,count|
22 puts "#{item} #{count}"
23}
diff --git a/examples/redis-unstable/utils/systemd-redis_multiple_servers@.service b/examples/redis-unstable/utils/systemd-redis_multiple_servers@.service
new file mode 100644
index 0000000..108ccfc
--- /dev/null
+++ b/examples/redis-unstable/utils/systemd-redis_multiple_servers@.service
@@ -0,0 +1,37 @@
1# example systemd template service unit file for multiple redis-servers
2#
3# You can use this file as a blueprint for your actual template service unit
4# file, if you intend to run multiple independent redis-server instances in
5# parallel using systemd's "template unit files" feature. If you do, you will
6# want to choose a better basename for your service unit by renaming this file
7# when copying it.
8#
9# Please take a look at the provided "systemd-redis_server.service" example
10# service unit file, too, if you choose to use this approach at managing
11# multiple redis-server instances via systemd.
12
13[Unit]
14Description=Redis data structure server - instance %i
15Documentation=https://redis.io/documentation
16# This template unit assumes your redis-server configuration file(s)
17# to live at /etc/redis/redis_server_<INSTANCE_NAME>.conf
18AssertPathExists=/etc/redis/redis_server_%i.conf
19#Before=your_application.service another_example_application.service
20#AssertPathExists=/var/lib/redis
21
22[Service]
23ExecStart=/usr/local/bin/redis-server /etc/redis/redis_server_%i.conf
24LimitNOFILE=10032
25NoNewPrivileges=yes
26#OOMScoreAdjust=-900
27#PrivateTmp=yes
28Type=notify
29TimeoutStartSec=infinity
30TimeoutStopSec=infinity
31UMask=0077
32#User=redis
33#Group=redis
34#WorkingDirectory=/var/lib/redis
35
36[Install]
37WantedBy=multi-user.target
diff --git a/examples/redis-unstable/utils/systemd-redis_server.service b/examples/redis-unstable/utils/systemd-redis_server.service
new file mode 100644
index 0000000..cf15864
--- /dev/null
+++ b/examples/redis-unstable/utils/systemd-redis_server.service
@@ -0,0 +1,43 @@
1# example systemd service unit file for redis-server
2#
3# In order to use this as a template for providing a redis service in your
4# environment, _at the very least_ make sure to adapt the redis configuration
5# file you intend to use as needed (make sure to set "supervised systemd"), and
6# to set sane TimeoutStartSec and TimeoutStopSec property values in the unit's
7# "[Service]" section to fit your needs.
8#
9# Some properties, such as User= and Group=, are highly desirable for virtually
10# all deployments of redis, but cannot be provided in a manner that fits all
11# expectable environments. Some of these properties have been commented out in
12# this example service unit file, but you are highly encouraged to set them to
13# fit your needs.
14#
15# Please refer to systemd.unit(5), systemd.service(5), and systemd.exec(5) for
16# more information.
17
18[Unit]
19Description=Redis data structure server
20Documentation=https://redis.io/documentation
21#Before=your_application.service another_example_application.service
22#AssertPathExists=/var/lib/redis
23Wants=network-online.target
24After=network-online.target
25
26[Service]
27ExecStart=/usr/local/bin/redis-server --supervised systemd --daemonize no
28## Alternatively, have redis-server load a configuration file:
29#ExecStart=/usr/local/bin/redis-server /path/to/your/redis.conf
30LimitNOFILE=10032
31NoNewPrivileges=yes
32#OOMScoreAdjust=-900
33#PrivateTmp=yes
34Type=notify
35TimeoutStartSec=infinity
36TimeoutStopSec=infinity
37UMask=0077
38#User=redis
39#Group=redis
40#WorkingDirectory=/var/lib/redis
41
42[Install]
43WantedBy=multi-user.target
diff --git a/examples/redis-unstable/utils/tracking_collisions.c b/examples/redis-unstable/utils/tracking_collisions.c
new file mode 100644
index 0000000..4273092
--- /dev/null
+++ b/examples/redis-unstable/utils/tracking_collisions.c
@@ -0,0 +1,79 @@
1/* This is a small program used in order to understand the collision rate
2 * of CRC64 (ISO version) VS other stronger hashing functions in the context
3 * of hashing keys for the Redis "tracking" feature (client side caching
4 * assisted by the server).
5 *
6 * The program attempts to hash keys with common names in the form of
7 *
8 * prefix:<counter>
9 *
10 * And counts the resulting collisions generated in the 24 bits of output
11 * needed for the tracking feature invalidation table (16 millions + entries)
12 *
13 * Compile with:
14 *
15 * cc -O2 ./tracking_collisions.c ../src/crc64.c ../src/sha1.c
16 * ./a.out
17 *
18 * --------------------------------------------------------------------------
19 *
20 * Copyright (C) 2019-Present Redis Ltd. All rights reserved.
21 *
22 * Licensed under your choice of (a) the Redis Source Available License 2.0
23 * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the
24 * GNU Affero General Public License v3 (AGPLv3).
25 */
26
27#include <stdlib.h>
28#include <stdint.h>
29#include <string.h>
30#include <stdio.h>
31#include "../src/crc64.h"
32#include "../src/sha1.h"
33
34#define TABLE_SIZE (1<<24)
35int Table[TABLE_SIZE];
36
37uint64_t crc64Hash(char *key, size_t len) {
38 return crc64(0,(unsigned char*)key,len);
39}
40
41uint64_t sha1Hash(char *key, size_t len) {
42 SHA1_CTX ctx;
43 unsigned char hash[20];
44
45 SHA1Init(&ctx);
46 SHA1Update(&ctx,(unsigned char*)key,len);
47 SHA1Final(hash,&ctx);
48 uint64_t hash64;
49 memcpy(&hash64,hash,sizeof(hash64));
50 return hash64;
51}
52
53/* Test the hashing function provided as callback and return the
54 * number of collisions found. */
55unsigned long testHashingFunction(uint64_t (*hash)(char *, size_t)) {
56 unsigned long collisions = 0;
57 memset(Table,0,sizeof(Table));
58 char *prefixes[] = {"object", "message", "user", NULL};
59 for (int i = 0; prefixes[i] != NULL; i++) {
60 for (int j = 0; j < TABLE_SIZE/2; j++) {
61 char keyname[128];
62 size_t keylen = snprintf(keyname,sizeof(keyname),"%s:%d",
63 prefixes[i],j);
64 uint64_t bucket = hash(keyname,keylen) % TABLE_SIZE;
65 if (Table[bucket]) {
66 collisions++;
67 } else {
68 Table[bucket] = 1;
69 }
70 }
71 }
72 return collisions;
73}
74
75int main(void) {
76 printf("SHA1 : %lu\n", testHashingFunction(sha1Hash));
77 printf("CRC64: %lu\n", testHashingFunction(crc64Hash));
78 return 0;
79}
diff --git a/examples/redis-unstable/utils/whatisdoing.sh b/examples/redis-unstable/utils/whatisdoing.sh
new file mode 100755
index 0000000..09c0b08
--- /dev/null
+++ b/examples/redis-unstable/utils/whatisdoing.sh
@@ -0,0 +1,24 @@
1# This script is from http://poormansprofiler.org/
2#
3# NOTE: Instead of using this script, you should use the Redis
4# Software Watchdog, which provides a similar functionality but in
5# a more reliable / easy to use way.
6#
7# Check https://redis.io/docs/latest/operate/oss_and_stack/management/optimization/latency/ for more information.
8
9#!/bin/bash
10nsamples=1
11sleeptime=0
12pid=$(ps auxww | grep '[r]edis-server' | awk '{print $2}')
13
14for x in $(seq 1 $nsamples)
15 do
16 gdb -ex "set pagination 0" -ex "thread apply all bt" -batch -p $pid
17 sleep $sleeptime
18 done | \
19awk '
20 BEGIN { s = ""; }
21 /Thread/ { print s; s = ""; }
22 /^\#/ { if (s != "" ) { s = s "," $4} else { s = $4 } }
23 END { print s }' | \
24sort | uniq -c | sort -r -n -k 1,1
generated by cgit v1.2.3 (git 2.46.0) at 2026-04-30 00:59:09 +0000