diff options
Diffstat (limited to 'examples/redis-unstable/utils')
40 files changed, 0 insertions, 3361 deletions
diff --git a/examples/redis-unstable/utils/build-static-symbols.tcl b/examples/redis-unstable/utils/build-static-symbols.tcl deleted file mode 100644 index eb0e218..0000000 --- a/examples/redis-unstable/utils/build-static-symbols.tcl +++ /dev/null @@ -1,26 +0,0 @@ -# Build a symbol table for static symbols of redis.c -# Useful to get stack traces on segfault without a debugger. See redis.c -# for more information. -# -# Copyright(C) 2009-Present Redis Ltd. All rights reserved. -# -# Licensed under your choice of (a) the Redis Source Available License 2.0 -# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the -# GNU Affero General Public License v3 (AGPLv3). - -set fd [open redis.c] -set symlist {} -while {[gets $fd line] != -1} { - if {[regexp {^static +[A-z0-9]+[ *]+([A-z0-9]*)\(} $line - sym]} { - lappend symlist $sym - } -} -set symlist [lsort -unique $symlist] -puts "static struct redisFunctionSym symsTable\[\] = {" -foreach sym $symlist { - puts "{\"$sym\",(unsigned long)$sym}," -} -puts "{NULL,0}" -puts "};" - -close $fd diff --git a/examples/redis-unstable/utils/cluster_fail_time.tcl b/examples/redis-unstable/utils/cluster_fail_time.tcl deleted file mode 100644 index 8739949..0000000 --- a/examples/redis-unstable/utils/cluster_fail_time.tcl +++ /dev/null @@ -1,50 +0,0 @@ -# This simple script is used in order to estimate the average PFAIL->FAIL -# state switch after a failure. - -set ::sleep_time 10 ; # How much to sleep to trigger PFAIL. -set ::fail_port 30016 ; # Node to put in sleep. -set ::other_port 30001 ; # Node to use to monitor the flag switch. - -proc avg vector { - set sum 0.0 - foreach x $vector { - set sum [expr {$sum+$x}] - } - expr {$sum/[llength $vector]} -} - -set samples {} -while 1 { - exec redis-cli -p $::fail_port debug sleep $::sleep_time > /dev/null & - - # Wait for fail? to appear. - while 1 { - set output [exec redis-cli -p $::other_port cluster nodes] - if {[string match {*fail\?*} $output]} break - after 100 - } - - puts "FAIL?" - set start [clock milliseconds] - - # Wait for fail? to disappear. - while 1 { - set output [exec redis-cli -p $::other_port cluster nodes] - if {![string match {*fail\?*} $output]} break - after 100 - } - - puts "FAIL" - set now [clock milliseconds] - set elapsed [expr {$now-$start}] - puts $elapsed - lappend samples $elapsed - - puts "AVG([llength $samples]): [avg $samples]" - - # Wait for the instance to be available again. - exec redis-cli -p $::fail_port ping - - # Wait for the fail flag to be cleared. - after 2000 -} diff --git a/examples/redis-unstable/utils/corrupt_rdb.c b/examples/redis-unstable/utils/corrupt_rdb.c deleted file mode 100644 index d1a093c..0000000 --- a/examples/redis-unstable/utils/corrupt_rdb.c +++ /dev/null @@ -1,49 +0,0 @@ -/* Trivia program to corrupt an RDB file in order to check the RDB check - * program behavior and effectiveness. - * - * Copyright (C) 2016-Present Redis Ltd. All rights reserved. - * - * Licensed under your choice of (a) the Redis Source Available License 2.0 - * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the - * GNU Affero General Public License v3 (AGPLv3). - */ - -#include <stdio.h> -#include <fcntl.h> -#include <sys/stat.h> -#include <stdlib.h> -#include <unistd.h> -#include <time.h> - -int main(int argc, char **argv) { - struct stat stat; - int fd, cycles; - - if (argc != 3) { - fprintf(stderr,"Usage: <filename> <cycles>\n"); - exit(1); - } - - srand(time(NULL)); - char *filename = argv[1]; - cycles = atoi(argv[2]); - fd = open(filename,O_RDWR); - if (fd == -1) { - perror("open"); - exit(1); - } - fstat(fd,&stat); - - while(cycles--) { - unsigned char buf[32]; - unsigned long offset = rand()%stat.st_size; - int writelen = 1+rand()%31; - int j; - - for (j = 0; j < writelen; j++) buf[j] = (char)rand(); - lseek(fd,offset,SEEK_SET); - printf("Writing %d bytes at offset %lu\n", writelen, offset); - write(fd,buf,writelen); - } - return 0; -} diff --git a/examples/redis-unstable/utils/create-cluster/.gitignore b/examples/redis-unstable/utils/create-cluster/.gitignore deleted file mode 100644 index a34b639..0000000 --- a/examples/redis-unstable/utils/create-cluster/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -config.sh -*.rdb -*.aof -*.conf -*.log -appendonlydir-* diff --git a/examples/redis-unstable/utils/create-cluster/README b/examples/redis-unstable/utils/create-cluster/README deleted file mode 100644 index bcd7459..0000000 --- a/examples/redis-unstable/utils/create-cluster/README +++ /dev/null @@ -1,27 +0,0 @@ -create-cluster is a small script used to easily start a big number of Redis -instances configured to run in cluster mode. Its main goal is to allow manual -testing in a condition which is not easy to replicate with the Redis cluster -unit tests, for example when a lot of instances are needed in order to trigger -a given bug. - -The tool can also be used just to easily create a number of instances in a -Redis Cluster in order to experiment a bit with the system. - -USAGE ---- - -To create a cluster, follow these steps: - -1. Edit create-cluster and change the start / end port, depending on the -number of instances you want to create. -2. Use "./create-cluster start" in order to run the instances. -3. Use "./create-cluster create" in order to execute redis-cli --cluster create, so that -an actual Redis cluster will be created. (If you're accessing your setup via a local container, ensure that the CLUSTER_HOST value is changed to your local IP) -4. Now you are ready to play with the cluster. AOF files and logs for each instances are created in the current directory. - -In order to stop a cluster: - -1. Use "./create-cluster stop" to stop all the instances. After you stopped the instances you can use "./create-cluster start" to restart them if you change your mind. -2. Use "./create-cluster clean" to remove all the AOF / log files to restart with a clean environment. - -Use the command "./create-cluster help" to get the full list of features. diff --git a/examples/redis-unstable/utils/create-cluster/create-cluster b/examples/redis-unstable/utils/create-cluster/create-cluster deleted file mode 100755 index dd20083..0000000 --- a/examples/redis-unstable/utils/create-cluster/create-cluster +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -# Settings -BIN_PATH="$SCRIPT_DIR/../../src/" -CLUSTER_HOST=127.0.0.1 -PORT=30000 -TIMEOUT=2000 -NODES=6 -REPLICAS=1 -PROTECTED_MODE=yes -ADDITIONAL_OPTIONS="" - -# You may want to put the above config parameters into config.sh in order to -# override the defaults without modifying this script. - -if [ -a config.sh ] -then - source "config.sh" -fi - -# Computed vars -ENDPORT=$((PORT+NODES)) - -if [ "$1" == "start" ] -then - while [ $((PORT < ENDPORT)) != "0" ]; do - PORT=$((PORT+1)) - echo "Starting $PORT" - $BIN_PATH/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --appenddirname appendonlydir-${PORT} --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} - done - exit 0 -fi - -if [ "$1" == "create" ] -then - HOSTS="" - while [ $((PORT < ENDPORT)) != "0" ]; do - PORT=$((PORT+1)) - HOSTS="$HOSTS $CLUSTER_HOST:$PORT" - done - OPT_ARG="" - if [ "$2" == "-f" ]; then - OPT_ARG="--cluster-yes" - fi - $BIN_PATH/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS $OPT_ARG - exit 0 -fi - -if [ "$1" == "stop" ] -then - while [ $((PORT < ENDPORT)) != "0" ]; do - PORT=$((PORT+1)) - echo "Stopping $PORT" - $BIN_PATH/redis-cli -p $PORT shutdown nosave - done - exit 0 -fi - -if [ "$1" == "restart" ] -then - OLD_PORT=$PORT - while [ $((PORT < ENDPORT)) != "0" ]; do - PORT=$((PORT+1)) - echo "Stopping $PORT" - $BIN_PATH/redis-cli -p $PORT shutdown nosave - done - PORT=$OLD_PORT - while [ $((PORT < ENDPORT)) != "0" ]; do - PORT=$((PORT+1)) - echo "Starting $PORT" - $BIN_PATH/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --appenddirname appendonlydir-${PORT} --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} - done - exit 0 -fi - -if [ "$1" == "watch" ] -then - PORT=$((PORT+1)) - while [ 1 ]; do - clear - date - $BIN_PATH/redis-cli -p $PORT cluster nodes | head -30 - sleep 1 - done - exit 0 -fi - -if [ "$1" == "tail" ] -then - INSTANCE=$2 - PORT=$((PORT+INSTANCE)) - tail -f ${PORT}.log - exit 0 -fi - -if [ "$1" == "tailall" ] -then - tail -f *.log - exit 0 -fi - -if [ "$1" == "call" ] -then - while [ $((PORT < ENDPORT)) != "0" ]; do - PORT=$((PORT+1)) - $BIN_PATH/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9 - done - exit 0 -fi - -if [ "$1" == "clean" ] -then - echo "Cleaning *.log" - rm -rf *.log - echo "Cleaning appendonlydir-*" - rm -rf appendonlydir-* - echo "Cleaning dump-*.rdb" - rm -rf dump-*.rdb - echo "Cleaning nodes-*.conf" - rm -rf nodes-*.conf - exit 0 -fi - -if [ "$1" == "clean-logs" ] -then - echo "Cleaning *.log" - rm -rf *.log - exit 0 -fi - -echo "Usage: $0 [start|create|stop|restart|watch|tail|tailall|clean|clean-logs|call]" -echo "start -- Launch Redis Cluster instances." -echo "create [-f] -- Create a cluster using redis-cli --cluster create." -echo "stop -- Stop Redis Cluster instances." -echo "restart -- Restart Redis Cluster instances." -echo "watch -- Show CLUSTER NODES output (first 30 lines) of first node." -echo "tail <id> -- Run tail -f of instance at base port + ID." -echo "tailall -- Run tail -f for all the log files at once." -echo "clean -- Remove all instances data, logs, configs." -echo "clean-logs -- Remove just instances logs." -echo "call <cmd> -- Call a command (up to 7 arguments) on all nodes." diff --git a/examples/redis-unstable/utils/gen-test-certs.sh b/examples/redis-unstable/utils/gen-test-certs.sh deleted file mode 100755 index 6bc9d86..0000000 --- a/examples/redis-unstable/utils/gen-test-certs.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Generate some test certificates which are used by the regression test suite: -# -# tests/tls/ca.{crt,key} Self signed CA certificate. -# tests/tls/redis.{crt,key} A certificate with no key usage/policy restrictions. -# tests/tls/client.{crt,key} A certificate restricted for SSL client usage. -# tests/tls/server.{crt,key} A certificate restricted for SSL server usage. -# tests/tls/redis.dh DH Params file. - -generate_cert() { - local name=$1 - local cn="$2" - local opts="$3" - - local keyfile=tests/tls/${name}.key - local certfile=tests/tls/${name}.crt - - [ -f $keyfile ] || openssl genrsa -out $keyfile 2048 - openssl req \ - -new -sha256 \ - -subj "/O=Redis Test/CN=$cn" \ - -key $keyfile | \ - openssl x509 \ - -req -sha256 \ - -CA tests/tls/ca.crt \ - -CAkey tests/tls/ca.key \ - -CAserial tests/tls/ca.txt \ - -CAcreateserial \ - -days 365 \ - $opts \ - -out $certfile -} - -mkdir -p tests/tls -[ -f tests/tls/ca.key ] || openssl genrsa -out tests/tls/ca.key 4096 -openssl req \ - -x509 -new -nodes -sha256 \ - -key tests/tls/ca.key \ - -days 3650 \ - -subj '/O=Redis Test/CN=Certificate Authority' \ - -out tests/tls/ca.crt - -cat > tests/tls/openssl.cnf <<_END_ -[ server_cert ] -keyUsage = digitalSignature, keyEncipherment -nsCertType = server - -[ client_cert ] -keyUsage = digitalSignature, keyEncipherment -nsCertType = client -_END_ - -generate_cert server "Server-only" "-extfile tests/tls/openssl.cnf -extensions server_cert" -generate_cert client "Client-only" "-extfile tests/tls/openssl.cnf -extensions client_cert" -generate_cert redis "Generic-cert" - -[ -f tests/tls/redis.dh ] || openssl dhparam -out tests/tls/redis.dh 2048 diff --git a/examples/redis-unstable/utils/generate-command-code.py b/examples/redis-unstable/utils/generate-command-code.py deleted file mode 100755 index 76c8c3b..0000000 --- a/examples/redis-unstable/utils/generate-command-code.py +++ /dev/null @@ -1,629 +0,0 @@ -#!/usr/bin/env python3 -import glob -import json -import os -import argparse - -ARG_TYPES = { - "string": "ARG_TYPE_STRING", - "integer": "ARG_TYPE_INTEGER", - "double": "ARG_TYPE_DOUBLE", - "key": "ARG_TYPE_KEY", - "pattern": "ARG_TYPE_PATTERN", - "unix-time": "ARG_TYPE_UNIX_TIME", - "pure-token": "ARG_TYPE_PURE_TOKEN", - "oneof": "ARG_TYPE_ONEOF", - "block": "ARG_TYPE_BLOCK", -} - -GROUPS = { - "generic": "COMMAND_GROUP_GENERIC", - "string": "COMMAND_GROUP_STRING", - "list": "COMMAND_GROUP_LIST", - "set": "COMMAND_GROUP_SET", - "sorted_set": "COMMAND_GROUP_SORTED_SET", - "hash": "COMMAND_GROUP_HASH", - "pubsub": "COMMAND_GROUP_PUBSUB", - "transactions": "COMMAND_GROUP_TRANSACTIONS", - "connection": "COMMAND_GROUP_CONNECTION", - "server": "COMMAND_GROUP_SERVER", - "scripting": "COMMAND_GROUP_SCRIPTING", - "hyperloglog": "COMMAND_GROUP_HYPERLOGLOG", - "cluster": "COMMAND_GROUP_CLUSTER", - "sentinel": "COMMAND_GROUP_SENTINEL", - "geo": "COMMAND_GROUP_GEO", - "stream": "COMMAND_GROUP_STREAM", - "bitmap": "COMMAND_GROUP_BITMAP", -} - - -def get_optional_desc_string(desc, field, force_uppercase=False): - v = desc.get(field, None) - if v and force_uppercase: - v = v.upper() - ret = "\"%s\"" % v if v else "NULL" - return ret.replace("\n", "\\n") - - -def check_command_args_key_specs(args, command_key_specs_index_set, command_arg_key_specs_index_set): - if not args: - return True - - for arg in args: - if arg.key_spec_index is not None: - assert isinstance(arg.key_spec_index, int) - - if arg.key_spec_index not in command_key_specs_index_set: - print("command: %s arg: %s key_spec_index error" % (command.fullname(), arg.name)) - return False - - command_arg_key_specs_index_set.add(arg.key_spec_index) - - if not check_command_args_key_specs(arg.subargs, command_key_specs_index_set, command_arg_key_specs_index_set): - return False - - return True - -def check_command_key_specs(command): - if not command.key_specs: - return True - - assert isinstance(command.key_specs, list) - - for cmd_key_spec in command.key_specs: - if "flags" not in cmd_key_spec: - print("command: %s key_specs missing flags" % command.fullname()) - return False - - if "NOT_KEY" in cmd_key_spec["flags"]: - # Like SUNSUBSCRIBE / SPUBLISH / SSUBSCRIBE - return True - - command_key_specs_index_set = set(range(len(command.key_specs))) - command_arg_key_specs_index_set = set() - - # Collect key_spec used for each arg, including arg.subarg - if not check_command_args_key_specs(command.args, command_key_specs_index_set, command_arg_key_specs_index_set): - return False - - # Check if we have key_specs not used - if command_key_specs_index_set != command_arg_key_specs_index_set: - print("command: %s may have unused key_spec" % command.fullname()) - return False - - return True - - -# Globals -subcommands = {} # container_name -> dict(subcommand_name -> Subcommand) - Only subcommands -commands = {} # command_name -> Command - Only commands - - -class KeySpec(object): - def __init__(self, spec): - self.spec = spec - - def struct_code(self): - def _flags_code(): - s = "" - for flag in self.spec.get("flags", []): - s += "CMD_KEY_%s|" % flag - return s[:-1] if s else 0 - - def _begin_search_code(): - if self.spec["begin_search"].get("index"): - return "KSPEC_BS_INDEX,.bs.index={%d}" % ( - self.spec["begin_search"]["index"]["pos"] - ) - elif self.spec["begin_search"].get("keyword"): - return "KSPEC_BS_KEYWORD,.bs.keyword={\"%s\",%d}" % ( - self.spec["begin_search"]["keyword"]["keyword"], - self.spec["begin_search"]["keyword"]["startfrom"], - ) - elif "unknown" in self.spec["begin_search"]: - return "KSPEC_BS_UNKNOWN,{{0}}" - else: - print("Invalid begin_search! value=%s" % self.spec["begin_search"]) - exit(1) - - def _find_keys_code(): - if self.spec["find_keys"].get("range"): - return "KSPEC_FK_RANGE,.fk.range={%d,%d,%d}" % ( - self.spec["find_keys"]["range"]["lastkey"], - self.spec["find_keys"]["range"]["step"], - self.spec["find_keys"]["range"]["limit"] - ) - elif self.spec["find_keys"].get("keynum"): - return "KSPEC_FK_KEYNUM,.fk.keynum={%d,%d,%d}" % ( - self.spec["find_keys"]["keynum"]["keynumidx"], - self.spec["find_keys"]["keynum"]["firstkey"], - self.spec["find_keys"]["keynum"]["step"] - ) - elif "unknown" in self.spec["find_keys"]: - return "KSPEC_FK_UNKNOWN,{{0}}" - else: - print("Invalid find_keys! value=%s" % self.spec["find_keys"]) - exit(1) - - return "%s,%s,%s,%s" % ( - get_optional_desc_string(self.spec, "notes"), - _flags_code(), - _begin_search_code(), - _find_keys_code() - ) - - -def verify_no_dup_names(container_fullname, args): - name_list = [arg.name for arg in args] - name_set = set(name_list) - if len(name_list) != len(name_set): - print("{}: Dup argument names: {}".format(container_fullname, name_list)) - exit(1) - - -class Argument(object): - def __init__(self, parent_name, desc): - self.parent_name = parent_name - self.desc = desc - self.name = self.desc["name"].lower() - if "_" in self.name: - print("{}: name ({}) should not contain underscores".format(self.fullname(), self.name)) - exit(1) - self.type = self.desc["type"] - self.key_spec_index = self.desc.get("key_spec_index", None) - self.subargs = [] - if self.type in ["oneof", "block"]: - self.display = None - for subdesc in self.desc["arguments"]: - self.subargs.append(Argument(self.fullname(), subdesc)) - if len(self.subargs) < 2: - print("{}: oneof or block arg contains less than two subargs".format(self.fullname())) - exit(1) - verify_no_dup_names(self.fullname(), self.subargs) - else: - self.display = self.desc.get("display") - - def fullname(self): - return ("%s %s" % (self.parent_name, self.name)).replace("-", "_") - - def struct_name(self): - return "%s_Arg" % (self.fullname().replace(" ", "_")) - - def subarg_table_name(self): - assert self.subargs - return "%s_Subargs" % (self.fullname().replace(" ", "_")) - - def struct_code(self): - """ - Output example: - MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=GETEX_expiration_Subargs - """ - - def _flags_code(): - s = "" - if self.desc.get("optional", False): - s += "CMD_ARG_OPTIONAL|" - if self.desc.get("multiple", False): - s += "CMD_ARG_MULTIPLE|" - if self.desc.get("multiple_token", False): - assert self.desc.get("multiple", False) # Sanity - s += "CMD_ARG_MULTIPLE_TOKEN|" - return s[:-1] if s else "CMD_ARG_NONE" - - s = "MAKE_ARG(\"%s\",%s,%d,%s,%s,%s,%s,%d,%s)" % ( - self.name, - ARG_TYPES[self.type], - self.desc.get("key_spec_index", -1), - get_optional_desc_string(self.desc, "token", force_uppercase=True), - get_optional_desc_string(self.desc, "summary"), - get_optional_desc_string(self.desc, "since"), - _flags_code(), - len(self.subargs), - get_optional_desc_string(self.desc, "deprecated_since"), - ) - if "display" in self.desc: - s += ",.display_text=\"%s\"" % self.desc["display"].lower() - if self.subargs: - s += ",.subargs=%s" % self.subarg_table_name() - - return s - - def write_internal_structs(self, f): - if self.subargs: - for subarg in self.subargs: - subarg.write_internal_structs(f) - - f.write("/* %s argument table */\n" % self.fullname()) - f.write("struct COMMAND_ARG %s[] = {\n" % self.subarg_table_name()) - for subarg in self.subargs: - f.write("{%s},\n" % subarg.struct_code()) - f.write("};\n\n") - - -def to_c_name(str): - return str.replace(":", "").replace(".", "_").replace("$", "_")\ - .replace("^", "_").replace("*", "_").replace("-", "_") \ - .replace("\\", "_").replace("+", "_") - - -class ReplySchema(object): - def __init__(self, name, desc): - self.name = to_c_name(name) - self.schema = {} - if desc.get("type") == "object": - if desc.get("properties") and desc.get("additionalProperties") is None: - print("%s: Any object that has properties should have the additionalProperties field" % self.name) - exit(1) - elif desc.get("type") == "array": - if desc.get("items") and isinstance(desc["items"], list) and any([desc.get(k) is None for k in ["minItems", "maxItems"]]): - print("%s: Any array that has items should have the minItems and maxItems fields" % self.name) - exit(1) - for k, v in desc.items(): - if isinstance(v, dict): - self.schema[k] = ReplySchema("%s_%s" % (self.name, k), v) - elif isinstance(v, list): - self.schema[k] = [] - for i, subdesc in enumerate(v): - self.schema[k].append(ReplySchema("%s_%s_%i" % (self.name, k,i), subdesc)) - else: - self.schema[k] = v - - def write(self, f): - def struct_code(name, k, v): - if isinstance(v, ReplySchema): - t = "JSON_TYPE_OBJECT" - vstr = ".value.object=&%s" % name - elif isinstance(v, list): - t = "JSON_TYPE_ARRAY" - vstr = ".value.array={.objects=%s,.length=%d}" % (name, len(v)) - elif isinstance(v, bool): - t = "JSON_TYPE_BOOLEAN" - vstr = ".value.boolean=%d" % int(v) - elif isinstance(v, str): - t = "JSON_TYPE_STRING" - vstr = ".value.string=\"%s\"" % v - elif isinstance(v, int): - t = "JSON_TYPE_INTEGER" - vstr = ".value.integer=%d" % v - - return "%s,%s,%s" % (t, json.dumps(k), vstr) - - for k, v in self.schema.items(): - if isinstance(v, ReplySchema): - v.write(f) - elif isinstance(v, list): - for i, schema in enumerate(v): - schema.write(f) - name = to_c_name("%s_%s" % (self.name, k)) - f.write("/* %s array reply schema */\n" % name) - f.write("struct jsonObject *%s[] = {\n" % name) - for i, schema in enumerate(v): - f.write("&%s,\n" % schema.name) - f.write("};\n\n") - - f.write("/* %s reply schema */\n" % self.name) - f.write("struct jsonObjectElement %s_elements[] = {\n" % self.name) - for k, v in self.schema.items(): - name = to_c_name("%s_%s" % (self.name, k)) - f.write("{%s},\n" % struct_code(name, k, v)) - f.write("};\n\n") - f.write("struct jsonObject %s = {%s_elements,.length=%d};\n\n" % (self.name, self.name, len(self.schema))) - - -class Command(object): - def __init__(self, name, desc): - self.name = name.upper() - self.desc = desc - self.group = self.desc["group"] - self.key_specs = self.desc.get("key_specs", []) - self.subcommands = [] - self.args = [] - for arg_desc in self.desc.get("arguments", []): - self.args.append(Argument(self.fullname(), arg_desc)) - verify_no_dup_names(self.fullname(), self.args) - self.reply_schema = None - if "reply_schema" in self.desc: - self.reply_schema = ReplySchema(self.reply_schema_name(), self.desc["reply_schema"]) - - def fullname(self): - return self.name.replace("-", "_").replace(":", "") - - def return_types_table_name(self): - return "%s_ReturnInfo" % self.fullname().replace(" ", "_") - - def subcommand_table_name(self): - assert self.subcommands - return "%s_Subcommands" % self.name - - def history_table_name(self): - return "%s_History" % (self.fullname().replace(" ", "_")) - - def tips_table_name(self): - return "%s_Tips" % (self.fullname().replace(" ", "_")) - - def arg_table_name(self): - return "%s_Args" % (self.fullname().replace(" ", "_")) - - def key_specs_table_name(self): - return "%s_Keyspecs" % (self.fullname().replace(" ", "_")) - - def reply_schema_name(self): - return "%s_ReplySchema" % (self.fullname().replace(" ", "_")) - - def struct_name(self): - return "%s_Command" % (self.fullname().replace(" ", "_")) - - def history_code(self): - if not self.desc.get("history"): - return "" - s = "" - for tupl in self.desc["history"]: - s += "{\"%s\",\"%s\"},\n" % (tupl[0], tupl[1]) - return s - - def num_history(self): - if not self.desc.get("history"): - return 0 - return len(self.desc["history"]) - - def tips_code(self): - if not self.desc.get("command_tips"): - return "" - s = "" - for hint in self.desc["command_tips"]: - s += "\"%s\",\n" % hint.lower() - return s - - def num_tips(self): - if not self.desc.get("command_tips"): - return 0 - return len(self.desc["command_tips"]) - - def key_specs_code(self): - s = "" - for spec in self.key_specs: - s += "{%s}," % KeySpec(spec).struct_code() - return s[:-1] - - - def struct_code(self): - """ - Output example: - MAKE_CMD("set","Set the string value of a key","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SET_History,4,SET_Tips,0,setCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_STRING,SET_Keyspecs,1,setGetKeys,5),.args=SET_Args - """ - - def _flags_code(): - s = "" - for flag in self.desc.get("command_flags", []): - s += "CMD_%s|" % flag - return s[:-1] if s else 0 - - def _acl_categories_code(): - s = "" - for cat in self.desc.get("acl_categories", []): - s += "ACL_CATEGORY_%s|" % cat - return s[:-1] if s else 0 - - def _doc_flags_code(): - s = "" - for flag in self.desc.get("doc_flags", []): - s += "CMD_DOC_%s|" % flag - return s[:-1] if s else "CMD_DOC_NONE" - - s = "MAKE_CMD(\"%s\",%s,%s,%s,%s,%s,%s,%s,%s,%s,%d,%s,%d,%s,%d,%s,%s,%s,%d,%s,%d)," % ( - self.name.lower(), - get_optional_desc_string(self.desc, "summary"), - get_optional_desc_string(self.desc, "complexity"), - get_optional_desc_string(self.desc, "since"), - _doc_flags_code(), - get_optional_desc_string(self.desc, "replaced_by"), - get_optional_desc_string(self.desc, "deprecated_since"), - "\"%s\"" % self.group, - GROUPS[self.group], - self.history_table_name(), - self.num_history(), - self.tips_table_name(), - self.num_tips(), - self.desc.get("function", "NULL"), - self.desc["arity"], - _flags_code(), - _acl_categories_code(), - self.key_specs_table_name(), - len(self.key_specs), - self.desc.get("get_keys_function", "NULL"), - len(self.args), - ) - - if self.subcommands: - s += ".subcommands=%s," % self.subcommand_table_name() - - if self.args: - s += ".args=%s," % self.arg_table_name() - - if self.reply_schema and args.with_reply_schema: - s += ".reply_schema=&%s," % self.reply_schema_name() - - return s[:-1] - - def write_internal_structs(self, f): - if self.subcommands: - subcommand_list = sorted(self.subcommands, key=lambda cmd: cmd.name) - for subcommand in subcommand_list: - subcommand.write_internal_structs(f) - - f.write("/* %s command table */\n" % self.fullname()) - f.write("struct COMMAND_STRUCT %s[] = {\n" % self.subcommand_table_name()) - for subcommand in subcommand_list: - f.write("{%s},\n" % subcommand.struct_code()) - f.write("{0}\n") - f.write("};\n\n") - - f.write("/********** %s ********************/\n\n" % self.fullname()) - - f.write("#ifndef SKIP_CMD_HISTORY_TABLE\n") - f.write("/* %s history */\n" % self.fullname()) - code = self.history_code() - if code: - f.write("commandHistory %s[] = {\n" % self.history_table_name()) - f.write("%s" % code) - f.write("};\n") - else: - f.write("#define %s NULL\n" % self.history_table_name()) - f.write("#endif\n\n") - - f.write("#ifndef SKIP_CMD_TIPS_TABLE\n") - f.write("/* %s tips */\n" % self.fullname()) - code = self.tips_code() - if code: - f.write("const char *%s[] = {\n" % self.tips_table_name()) - f.write("%s" % code) - f.write("};\n") - else: - f.write("#define %s NULL\n" % self.tips_table_name()) - f.write("#endif\n\n") - - f.write("#ifndef SKIP_CMD_KEY_SPECS_TABLE\n") - f.write("/* %s key specs */\n" % self.fullname()) - code = self.key_specs_code() - if code: - f.write("keySpec %s[%d] = {\n" % (self.key_specs_table_name(), len(self.key_specs))) - f.write("%s\n" % code) - f.write("};\n") - else: - f.write("#define %s NULL\n" % self.key_specs_table_name()) - f.write("#endif\n\n") - - if self.args: - for arg in self.args: - arg.write_internal_structs(f) - - f.write("/* %s argument table */\n" % self.fullname()) - f.write("struct COMMAND_ARG %s[] = {\n" % self.arg_table_name()) - for arg in self.args: - f.write("{%s},\n" % arg.struct_code()) - f.write("};\n\n") - - if self.reply_schema and args.with_reply_schema: - self.reply_schema.write(f) - - -class Subcommand(Command): - def __init__(self, name, desc): - self.container_name = desc["container"].upper() - super(Subcommand, self).__init__(name, desc) - - def fullname(self): - return "%s %s" % (self.container_name, self.name.replace("-", "_").replace(":", "")) - - -def create_command(name, desc): - flags = desc.get("command_flags") - if flags and "EXPERIMENTAL" in flags: - print("Command %s is experimental, skipping..." % name) - return - - if desc.get("container"): - cmd = Subcommand(name.upper(), desc) - subcommands.setdefault(desc["container"].upper(), {})[name] = cmd - else: - cmd = Command(name.upper(), desc) - commands[name.upper()] = cmd - - -# MAIN - -# Figure out where the sources are -srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src") - -parser = argparse.ArgumentParser() -parser.add_argument('--with-reply-schema', action='store_true') -args = parser.parse_args() - -# Create all command objects -print("Processing json files...") -for filename in glob.glob('%s/commands/*.json' % srcdir): - with open(filename, "r") as f: - try: - d = json.load(f) - for name, desc in d.items(): - create_command(name, desc) - except json.decoder.JSONDecodeError as err: - print("Error processing %s: %s" % (filename, err)) - exit(1) - -# Link subcommands to containers -print("Linking container command to subcommands...") -for command in commands.values(): - assert command.group - if command.name not in subcommands: - continue - for subcommand in subcommands[command.name].values(): - assert not subcommand.group or subcommand.group == command.group - subcommand.group = command.group - command.subcommands.append(subcommand) - -check_command_error_counter = 0 # An error counter is used to count errors in command checking. - -print("Checking all commands...") -for command in commands.values(): - if not check_command_key_specs(command): - check_command_error_counter += 1 - -if check_command_error_counter != 0: - print("Error: There are errors in the commands check, please check the above logs.") - exit(1) - -commands_filename = "commands_with_reply_schema" if args.with_reply_schema else "commands" -print("Generating %s.def..." % commands_filename) -with open("%s/%s.def" % (srcdir, commands_filename), "w") as f: - f.write("/* Automatically generated by %s, do not edit. */\n\n" % os.path.basename(__file__)) - f.write( -""" -/* We have fabulous commands from - * the fantastic - * Redis Command Table! */ - -/* Must match redisCommandGroup */ -const char *COMMAND_GROUP_STR[] = { - "generic", - "string", - "list", - "set", - "sorted-set", - "hash", - "pubsub", - "transactions", - "connection", - "server", - "scripting", - "hyperloglog", - "cluster", - "sentinel", - "geo", - "stream", - "bitmap", - "module" -}; - -const char *commandGroupStr(int index) { - return COMMAND_GROUP_STR[index]; -} -""" - ) - - command_list = sorted(commands.values(), key=lambda cmd: (cmd.group, cmd.name)) - for command in command_list: - command.write_internal_structs(f) - - f.write("/* Main command table */\n") - f.write("struct COMMAND_STRUCT redisCommandTable[] = {\n") - curr_group = None - for command in command_list: - if curr_group != command.group: - curr_group = command.group - f.write("/* %s */\n" % curr_group) - f.write("{%s},\n" % command.struct_code()) - f.write("{0}\n") - f.write("};\n") - -print("All done, exiting.") diff --git a/examples/redis-unstable/utils/generate-commands-json.py b/examples/redis-unstable/utils/generate-commands-json.py deleted file mode 100755 index ea18a79..0000000 --- a/examples/redis-unstable/utils/generate-commands-json.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import os -import sys -import subprocess -from collections import OrderedDict - - -def convert_flags_to_boolean_dict(flags): - """Return a dict with a key set to `True` per element in the flags list.""" - return {f: True for f in flags} - - -def set_if_not_none_or_empty(dst, key, value): - """Set 'key' in 'dst' if 'value' is not `None` or an empty list.""" - if value is not None and (type(value) is not list or len(value)): - dst[key] = value - - -def convert_argument(arg): - """Transform an argument.""" - arg.update(convert_flags_to_boolean_dict(arg.pop('flags', []))) - set_if_not_none_or_empty(arg, 'arguments', - [convert_argument(x) for x in arg.pop('arguments', [])]) - return arg - - -def convert_keyspec(spec): - """Transform a key spec.""" - spec.update(convert_flags_to_boolean_dict(spec.pop('flags', []))) - return spec - - -def convert_entry_to_objects_array(cmd, docs): - """Transform the JSON output of `COMMAND` to a friendlier format. - - cmd is the output of `COMMAND` as follows: - 1. Name (lower case, e.g. "lolwut") - 2. Arity - 3. Flags - 4-6. First/last/step key specification (deprecated as of Redis v7.0) - 7. ACL categories - 8. hints (as of Redis 7.0) - 9. key-specs (as of Redis 7.0) - 10. subcommands (as of Redis 7.0) - - docs is the output of `COMMAND DOCS`, which holds a map of additional metadata - - This returns a list with a dict for the command and per each of its - subcommands. Each dict contains one key, the command's full name, with a - value of a dict that's set with the command's properties and meta - information.""" - assert len(cmd) >= 9 - obj = {} - rep = [obj] - name = cmd[0].upper() - arity = cmd[1] - command_flags = cmd[2] - acl_categories = cmd[6] - hints = cmd[7] - keyspecs = cmd[8] - subcommands = cmd[9] if len(cmd) > 9 else [] - key = name.replace('|', ' ') - - subcommand_docs = docs.pop('subcommands', []) - rep.extend([convert_entry_to_objects_array(x, subcommand_docs[x[0]])[0] for x in subcommands]) - - # The command's value is ordered so the interesting stuff that we care about - # is at the start. Optional `None` and empty list values are filtered out. - value = OrderedDict() - group = docs.pop('group') - if group == 'module': - set_if_not_none_or_empty(value, 'summary', docs.pop('summary', None)) - set_if_not_none_or_empty(value, 'since', docs.pop('since', None)) - else: - # "summary" and "since" are required for all non-module commands - value['summary'] = docs.pop('summary') - value['since'] = docs.pop('since') - value['group'] = group - set_if_not_none_or_empty(value, 'complexity', docs.pop('complexity', None)) - set_if_not_none_or_empty(value, 'deprecated_since', docs.pop('deprecated_since', None)) - set_if_not_none_or_empty(value, 'replaced_by', docs.pop('replaced_by', None)) - set_if_not_none_or_empty(value, 'history', docs.pop('history', [])) - set_if_not_none_or_empty(value, 'acl_categories', acl_categories) - value['arity'] = arity - set_if_not_none_or_empty(value, 'key_specs', - [convert_keyspec(x) for x in keyspecs]) - set_if_not_none_or_empty(value, 'arguments', - [convert_argument(x) for x in docs.pop('arguments', [])]) - set_if_not_none_or_empty(value, 'command_flags', command_flags) - set_if_not_none_or_empty(value, 'doc_flags', docs.pop('doc_flags', [])) - set_if_not_none_or_empty(value, 'hints', hints) - - # All remaining docs key-value tuples, if any, are appended to the command - # to be future-proof. - while len(docs) > 0: - (k, v) = docs.popitem() - value[k] = v - - obj[key] = value - return rep - - -# Figure out where the sources are -srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src") - -# MAIN -if __name__ == '__main__': - opts = { - 'description': 'Transform the output from `redis-cli --json` using COMMAND and COMMAND DOCS to a single commands.json format.', - 'epilog': f'Usage example: {sys.argv[0]} --cli src/redis-cli --port 6379 > commands.json' - } - parser = argparse.ArgumentParser(**opts) - parser.add_argument('--host', type=str, default='localhost') - parser.add_argument('--port', type=int, default=6379) - parser.add_argument('--cli', type=str, default='%s/redis-cli' % srcdir) - args = parser.parse_args() - - payload = OrderedDict() - cmds = [] - - p = subprocess.Popen([args.cli, '-h', args.host, '-p', str(args.port), '--json', 'command'], stdout=subprocess.PIPE) - stdout, stderr = p.communicate() - commands = json.loads(stdout) - - p = subprocess.Popen([args.cli, '-h', args.host, '-p', str(args.port), '--json', 'command', 'docs'], - stdout=subprocess.PIPE) - stdout, stderr = p.communicate() - docs = json.loads(stdout) - - for entry in commands: - cmd = convert_entry_to_objects_array(entry, docs[entry[0]]) - cmds.extend(cmd) - - # The final output is a dict of all commands, ordered by name. - cmds.sort(key=lambda x: list(x.keys())[0]) - for cmd in cmds: - name = list(cmd.keys())[0] - payload[name] = cmd[name] - - # Print the final JSON output. If the output is piped and the pipe is closed (e.g., by 'less' or 'head'), - # catch BrokenPipeError to prevent a traceback and exit gracefully. - try: - print(json.dumps(payload, indent=4)) - except BrokenPipeError: - sys.stderr.close() - sys.exit(0) diff --git a/examples/redis-unstable/utils/generate-fmtargs.py b/examples/redis-unstable/utils/generate-fmtargs.py deleted file mode 100755 index 1eced02..0000000 --- a/examples/redis-unstable/utils/generate-fmtargs.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 - -# Outputs the generated part of src/fmtargs.h -MAX_ARGS = 160 - -import os -print("/* Everything below this line is automatically generated by") -print(" * %s. Do not manually edit. */\n" % os.path.basename(__file__)) - -print('#define ARG_N(' + ', '.join(['_' + str(i) for i in range(1, MAX_ARGS + 1, 1)]) + ', N, ...) N') - -print('\n#define RSEQ_N() ' + ', '.join([str(i) for i in range(MAX_ARGS, -1, -1)])) - -print('\n#define COMPACT_FMT_2(fmt, value) fmt') -for i in range(4, MAX_ARGS + 1, 2): - print('#define COMPACT_FMT_{}(fmt, value, ...) fmt COMPACT_FMT_{}(__VA_ARGS__)'.format(i, i - 2)) - -print('\n#define COMPACT_VALUES_2(fmt, value) value') -for i in range(4, MAX_ARGS + 1, 2): - print('#define COMPACT_VALUES_{}(fmt, value, ...) value, COMPACT_VALUES_{}(__VA_ARGS__)'.format(i, i - 2)) - -print("\n#endif") diff --git a/examples/redis-unstable/utils/generate-module-api-doc.rb b/examples/redis-unstable/utils/generate-module-api-doc.rb deleted file mode 100755 index cd45c93..0000000 --- a/examples/redis-unstable/utils/generate-module-api-doc.rb +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env ruby -# coding: utf-8 -# gendoc.rb -- Converts the top-comments inside module.c to modules API -# reference documentation in markdown format. - -# Convert the C comment to markdown -def markdown(s) - s = s.gsub(/\*\/$/,"") - s = s.gsub(/^ ?\* ?/,"") - s = s.gsub(/^\/\*\*? ?/,"") - s.chop! while s[-1] == "\n" || s[-1] == " " - lines = s.split("\n") - newlines = [] - # Fix some markdown - lines.each{|l| - # Rewrite RM_Xyz() to RedisModule_Xyz(). - l = l.gsub(/(?<![A-Z_])RM_(?=[A-Z])/, 'RedisModule_') - # Fix more markdown, except in code blocks indented by 4 spaces, which we - # don't want to mess with. - if not l.start_with?(' ') - # Add backquotes around RedisModule functions and type where missing. - l = l.gsub(/(?<!`)RedisModule[A-z]+(?:\*?\(\))?/){|x| "`#{x}`"} - # Add backquotes around c functions like malloc() where missing. - l = l.gsub(/(?<![`A-z.])[a-z_]+\(\)/, '`\0`') - # Add backquotes around macro and var names containing underscores. - l = l.gsub(/(?<![`A-z\*])[A-Za-z]+_[A-Za-z0-9_]+/){|x| "`#{x}`"} - # Link URLs preceded by space or newline (not already linked) - l = l.gsub(/(^| )(https?:\/\/[A-Za-z0-9_\/\.\-]+[A-Za-z0-9\/])/, - '\1[\2](\2)') - # Replace double-dash with unicode ndash - l = l.gsub(/ -- /, ' – ') - end - # Link function names to their definition within the page - l = l.gsub(/`(RedisModule_[A-z0-9]+)[()]*`/) {|x| - $index[$1] ? "[#{x}](\##{$1})" : x - } - newlines << l - } - return newlines.join("\n") -end - -# Linebreak a prototype longer than 80 characters on the commas, but only -# between balanced parentheses so that we don't linebreak args which are -# function pointers, and then aligning each arg under each other. -def linebreak_proto(proto, indent) - if proto.bytesize <= 80 - return proto - end - parts = proto.split(/,\s*/); - if parts.length == 1 - return proto; - end - align_pos = proto.index("(") + 1; - align = " " * align_pos - result = parts.shift; - bracket_balance = 0; - parts.each{|part| - if bracket_balance == 0 - result += ",\n" + indent + align - else - result += ", " - end - result += part - bracket_balance += part.count("(") - part.count(")") - } - return result; -end - -# Given the source code array and the index at which an exported symbol was -# detected, extracts and outputs the documentation. -def docufy(src,i) - m = /RM_[A-z0-9]+/.match(src[i]) - name = m[0] - name = name.sub("RM_","RedisModule_") - proto = src[i].sub("{","").strip+";\n" - proto = proto.sub("RM_","RedisModule_") - proto = linebreak_proto(proto, " "); - # Add a link target with the function name. (We don't trust the exact id of - # the generated one, which depends on the Markdown implementation.) - puts "<span id=\"#{name}\"></span>\n\n" - puts "### `#{name}`\n\n" - puts " #{proto}\n" - puts "**Available since:** #{$since[name] or "unreleased"}\n\n" - comment = "" - while true - i = i-1 - comment = src[i]+comment - break if src[i] =~ /\/\*/ - end - comment = markdown(comment) - puts comment+"\n\n" -end - -# Print a comment from line until */ is found, as markdown. -def section_doc(src, i) - name = get_section_heading(src, i) - comment = "<span id=\"#{section_name_to_id(name)}\"></span>\n\n" - while true - # append line, except if it's a horizontal divider - comment = comment + src[i] if src[i] !~ /^[\/ ]?\*{1,2} ?-{50,}/ - break if src[i] =~ /\*\// - i = i+1 - end - comment = markdown(comment) - puts comment+"\n\n" -end - -# generates an id suitable for links within the page -def section_name_to_id(name) - return "section-" + - name.strip.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/^-+|-+$/, '') -end - -# Returns the name of the first section heading in the comment block for which -# is_section_doc(src, i) is true -def get_section_heading(src, i) - if src[i] =~ /^\/\*\*? \#+ *(.*)/ - heading = $1 - elsif src[i+1] =~ /^ ?\* \#+ *(.*)/ - heading = $1 - end - return heading.gsub(' -- ', ' – ') -end - -# Returns true if the line is the start of a generic documentation section. Such -# section must start with the # symbol, i.e. a markdown heading, on the first or -# the second line. -def is_section_doc(src, i) - return src[i] =~ /^\/\*\*? \#/ || - (src[i] =~ /^\/\*/ && src[i+1] =~ /^ ?\* \#/) -end - -def is_func_line(src, i) - line = src[i] - return line =~ /RM_/ && - line[0] != ' ' && line[0] != '#' && line[0] != '/' && - src[i-1] =~ /\*\// -end - -puts "<!-- This file is generated from module.c using\n" -puts " redis/redis:utils/generate-module-api-doc.rb -->\n\n" -src = File.open(File.dirname(__FILE__) ++ "/../src/module.c").to_a - -# Build function index -$index = {} -src.each_with_index do |line,i| - if is_func_line(src, i) - line =~ /RM_([A-z0-9]+)/ - name = "RedisModule_#{$1}" - $index[name] = true - end -end - -# Populate the 'since' map (name => version) if we're in a git repo. -$since = {} -git_dir = File.dirname(__FILE__) ++ "/../.git" -if File.directory?(git_dir) && `which git` != "" - `git --git-dir="#{git_dir}" tag --sort=v:refname`.each_line do |version| - next if version !~ /^(\d+)\.\d+\.\d+?$/ || $1.to_i < 4 - version.chomp! - `git --git-dir="#{git_dir}" cat-file blob "#{version}:src/module.c"`.each_line do |line| - if line =~ /^\w.*[ \*]RM_([A-z0-9]+)/ - name = "RedisModule_#{$1}" - if ! $since[name] - $since[name] = version - end - end - end - end -end - -# Print TOC -puts "## Sections\n\n" -src.each_with_index do |_line,i| - if is_section_doc(src, i) - name = get_section_heading(src, i) - puts "* [#{name}](\##{section_name_to_id(name)})\n" - end -end -puts "* [Function index](#section-function-index)\n\n" - -# Docufy: Print function prototype and markdown docs -src.each_with_index do |_line,i| - if is_func_line(src, i) - docufy(src, i) - elsif is_section_doc(src, i) - section_doc(src, i) - end -end - -# Print function index -puts "<span id=\"section-function-index\"></span>\n\n" -puts "## Function index\n\n" -$index.keys.sort.each{|x| puts "* [`#{x}`](\##{x})\n"} -puts "\n" diff --git a/examples/redis-unstable/utils/graphs/commits-over-time/README.md b/examples/redis-unstable/utils/graphs/commits-over-time/README.md deleted file mode 100644 index b28019e..0000000 --- a/examples/redis-unstable/utils/graphs/commits-over-time/README.md +++ /dev/null @@ -1,16 +0,0 @@ -This Tcl script is what I used in order to generate the graph you -can find at http://antirez.com/news/98. It's really quick & dirty, more -a trow away program than anything else, but probably could be reused or -modified in the future in order to visualize other similar data or an -updated version of the same data. - -The usage is trivial: - - ./genhtml.tcl > output.html - -The generated HTML is quite broken but good enough to grab a screenshot -from the browser. Feel free to improve it if you got time / interest. - -Note that the code filtering the tags, and the hardcoded branch name, does -not make the script, as it is, able to analyze a different repository. -However the changes needed are trivial. diff --git a/examples/redis-unstable/utils/graphs/commits-over-time/genhtml.tcl b/examples/redis-unstable/utils/graphs/commits-over-time/genhtml.tcl deleted file mode 100755 index c4b4e09..0000000 --- a/examples/redis-unstable/utils/graphs/commits-over-time/genhtml.tcl +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env tclsh - -# Load commits history as "sha1 unixtime". -set commits [exec git log unstable {--pretty="%H %at"}] -set raw_tags [exec git tag] - -# Load all the tags that are about stable releases. -foreach tag $raw_tags { - if {[string match v*-stable $tag]} { - set tag [string range $tag 1 end-7] - puts $tag - } - if {[regexp {^[0-9]+.[0-9]+.[0-9]+$} $tag]} { - lappend tags $tag - } -} - -# For each tag, create a list of "name unixtime" -foreach tag $tags { - set taginfo [exec git log $tag -n 1 "--pretty=\"$tag %at\""] - set taginfo [string trim $taginfo {"}] - lappend labels $taginfo -} - -# For each commit, check the amount of code changed and create an array -# mapping the commit to the number of lines affected. -foreach c $commits { - set stat [exec git show --oneline --numstat [lindex $c 0]] - set linenum 0 - set affected 0 - foreach line [split $stat "\n"] { - incr linenum - if {$linenum == 1 || [string match *deps/* $line]} continue - if {[catch {llength $line} numfields]} continue - if {$numfields == 0} continue - catch { - incr affected [lindex $line 0] - incr affected [lindex $line 1] - } - } - set commit_to_affected([lindex $c 0]) $affected -} - -set base_time [lindex [lindex $commits end] 1] -puts [clock format $base_time] - -# Generate a graph made of HTML DIVs. -puts {<html> -<style> -.box { - position:absolute; - width:10px; - height:5px; - border:1px black solid; - background-color:#44aa33; - opacity: 0.04; -} -.label { - position:absolute; - background-color:#dddddd; - font-family:helvetica; - font-size:12px; - padding:2px; - color:#666; - border:1px #aaa solid; - border-radius: 5px; -} -#outer { - position:relative; - width:1500; - height:500; - border:1px #aaa solid; -} -</style> -<div id="outer"> -} -foreach c $commits { - set sha [lindex $c 0] - set t [expr {([lindex $c 1]-$base_time)/(3600*24*2)}] - set affected [expr $commit_to_affected($sha)] - set left $t - set height [expr {log($affected)*20}] - puts "<div class=\"box\" style=\"left:$left; bottom:0; height:$height\"></div>" -} - -set bottom -30 -foreach l $labels { - set name [lindex $l 0] - set t [expr {([lindex $l 1]-$base_time)/(3600*24*2)}] - set left $t - if {$left < 0} continue - incr bottom -20 - if {$bottom == -210} {set bottom -30} - puts "<div class=\"label\" style=\"left:$left; bottom:$bottom\">$name</div>" -} -puts {</div></html>} diff --git a/examples/redis-unstable/utils/hyperloglog/.gitignore b/examples/redis-unstable/utils/hyperloglog/.gitignore deleted file mode 100644 index 2211df6..0000000 --- a/examples/redis-unstable/utils/hyperloglog/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.txt diff --git a/examples/redis-unstable/utils/hyperloglog/hll-err.rb b/examples/redis-unstable/utils/hyperloglog/hll-err.rb deleted file mode 100644 index 1278397..0000000 --- a/examples/redis-unstable/utils/hyperloglog/hll-err.rb +++ /dev/null @@ -1,30 +0,0 @@ -# hll-err.rb - Copyright (C) 2014-Present Redis Ltd. -# -# Licensed under your choice of (a) the Redis Source Available License 2.0 -# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the -# GNU Affero General Public License v3 (AGPLv3). -# -# Check error of HyperLogLog Redis implementation for different set sizes. - -require 'rubygems' -require 'redis' -require 'digest/sha1' - -r = Redis.new -r.del('hll') -i = 0 -while true do - 100.times { - elements = [] - 1000.times { - ele = Digest::SHA1.hexdigest(i.to_s) - elements << ele - i += 1 - } - r.pfadd('hll',elements) - } - approx = r.pfcount('hll') - abs_err = (approx-i).abs - rel_err = 100.to_f*abs_err/i - puts "#{i} vs #{approx}: #{rel_err}%" -end diff --git a/examples/redis-unstable/utils/hyperloglog/hll-gnuplot-graph.rb b/examples/redis-unstable/utils/hyperloglog/hll-gnuplot-graph.rb deleted file mode 100644 index 1e48e6c..0000000 --- a/examples/redis-unstable/utils/hyperloglog/hll-gnuplot-graph.rb +++ /dev/null @@ -1,91 +0,0 @@ -# hll-err.rb - Copyright (C) 2014-Present Redis Ltd. -# -# Licensed under your choice of (a) the Redis Source Available License 2.0 -# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the -# GNU Affero General Public License v3 (AGPLv3). -# -# This program is suited to output average and maximum errors of -# the Redis HyperLogLog implementation in a format suitable to print -# graphs using gnuplot. - -require 'rubygems' -require 'redis' -require 'digest/sha1' - -# Generate an array of [cardinality,relative_error] pairs -# in the 0 - max range, with the specified step. -# -# 'r' is the Redis object used to perform the queries. -# 'seed' must be different every time you want a test performed -# with a different set. The function guarantees that if 'seed' is the -# same, exactly the same dataset is used, and when it is different, -# a totally unrelated different data set is used (without any common -# element in practice). -def run_experiment(r,seed,max,step) - r.del('hll') - i = 0 - samples = [] - step = 1000 if step > 1000 - while i < max do - elements = [] - step.times { - ele = Digest::SHA1.hexdigest(i.to_s+seed.to_s) - elements << ele - i += 1 - } - r.pfadd('hll',elements) - approx = r.pfcount('hll') - err = approx-i - rel_err = 100.to_f*err/i - samples << [i,rel_err] - end - samples -end - -def filter_samples(numsets,max,step,filter) - r = Redis.new - dataset = {} - (0...numsets).each{|i| - dataset[i] = run_experiment(r,i,max,step) - STDERR.puts "Set #{i}" - } - dataset[0].each_with_index{|ele,index| - if filter == :max - card=ele[0] - err=ele[1].abs - (1...numsets).each{|i| - err = dataset[i][index][1] if err < dataset[i][index][1] - } - puts "#{card} #{err}" - elsif filter == :avg - card=ele[0] - err = 0 - (0...numsets).each{|i| - err += dataset[i][index][1] - } - err /= numsets - puts "#{card} #{err}" - elsif filter == :absavg - card=ele[0] - err = 0 - (0...numsets).each{|i| - err += dataset[i][index][1].abs - } - err /= numsets - puts "#{card} #{err}" - elsif filter == :all - (0...numsets).each{|i| - card,err = dataset[i][index] - puts "#{card} #{err}" - } - else - raise "Unknown filter #{filter}" - end - } -end - -if ARGV.length != 4 - puts "Usage: hll-gnuplot-graph <samples> <max> <step> (max|avg|absavg|all)" - exit 1 -end -filter_samples(ARGV[0].to_i,ARGV[1].to_i,ARGV[2].to_i,ARGV[3].to_sym) diff --git a/examples/redis-unstable/utils/install_server.sh b/examples/redis-unstable/utils/install_server.sh deleted file mode 100755 index efda7da..0000000 --- a/examples/redis-unstable/utils/install_server.sh +++ /dev/null @@ -1,291 +0,0 @@ -#!/bin/sh - -# Copyright 2011 Dvir Volk <dvirsk at gmail dot com>. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -# EVENT SHALL Dvir Volk OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -################################################################################ -# -# Service installer for redis server, runs interactively by default. -# -# To run this script non-interactively (for automation/provisioning purposes), -# feed the variables into the script. Any missing variables will be prompted! -# Tip: Environment variables also support command substitution (see REDIS_EXECUTABLE) -# -# Example: -# -# sudo REDIS_PORT=1234 \ -# REDIS_CONFIG_FILE=/etc/redis/1234.conf \ -# REDIS_LOG_FILE=/var/log/redis_1234.log \ -# REDIS_DATA_DIR=/var/lib/redis/1234 \ -# REDIS_EXECUTABLE=`command -v redis-server` ./utils/install_server.sh -# -# This generates a redis config file and an /etc/init.d script, and installs them. -# -# /!\ This script should be run as root -# -# NOTE: This script will not work on Mac OSX. -# It supports Debian and Ubuntu Linux. -# -################################################################################ - -die () { - echo "ERROR: $1. Aborting!" - exit 1 -} - - -#Absolute path to this script -SCRIPT=$(readlink -f $0) -#Absolute path this script is in -SCRIPTPATH=$(dirname $SCRIPT) - -#Initial defaults -_REDIS_PORT=6379 -_MANUAL_EXECUTION=false - -echo "Welcome to the redis service installer" -echo "This script will help you easily set up a running redis server" -echo - -#check for root user -if [ "$(id -u)" -ne 0 ] ; then - echo "You must run this script as root. Sorry!" - exit 1 -fi - -#bail if this system is managed by systemd -_pid_1_exe="$(readlink -f /proc/1/exe)" -if [ "${_pid_1_exe##*/}" = systemd ] -then - echo "This systems seems to use systemd." - echo "Please take a look at the provided example service unit files in this directory, and adapt and install them. Sorry!" - exit 1 -fi -unset _pid_1_exe - -if ! echo $REDIS_PORT | egrep -q '^[0-9]+$' ; then - _MANUAL_EXECUTION=true - #Read the redis port - read -p "Please select the redis port for this instance: [$_REDIS_PORT] " REDIS_PORT - if ! echo $REDIS_PORT | egrep -q '^[0-9]+$' ; then - echo "Selecting default: $_REDIS_PORT" - REDIS_PORT=$_REDIS_PORT - fi -fi - -if [ -z "$REDIS_CONFIG_FILE" ] ; then - _MANUAL_EXECUTION=true - #read the redis config file - _REDIS_CONFIG_FILE="/etc/redis/$REDIS_PORT.conf" - read -p "Please select the redis config file name [$_REDIS_CONFIG_FILE] " REDIS_CONFIG_FILE - if [ -z "$REDIS_CONFIG_FILE" ] ; then - REDIS_CONFIG_FILE=$_REDIS_CONFIG_FILE - echo "Selected default - $REDIS_CONFIG_FILE" - fi -fi - -if [ -z "$REDIS_LOG_FILE" ] ; then - _MANUAL_EXECUTION=true - #read the redis log file path - _REDIS_LOG_FILE="/var/log/redis_$REDIS_PORT.log" - read -p "Please select the redis log file name [$_REDIS_LOG_FILE] " REDIS_LOG_FILE - if [ -z "$REDIS_LOG_FILE" ] ; then - REDIS_LOG_FILE=$_REDIS_LOG_FILE - echo "Selected default - $REDIS_LOG_FILE" - fi -fi - -if [ -z "$REDIS_DATA_DIR" ] ; then - _MANUAL_EXECUTION=true - #get the redis data directory - _REDIS_DATA_DIR="/var/lib/redis/$REDIS_PORT" - read -p "Please select the data directory for this instance [$_REDIS_DATA_DIR] " REDIS_DATA_DIR - if [ -z "$REDIS_DATA_DIR" ] ; then - REDIS_DATA_DIR=$_REDIS_DATA_DIR - echo "Selected default - $REDIS_DATA_DIR" - fi -fi - -if [ ! -x "$REDIS_EXECUTABLE" ] ; then - _MANUAL_EXECUTION=true - #get the redis executable path - _REDIS_EXECUTABLE=`command -v redis-server` - read -p "Please select the redis executable path [$_REDIS_EXECUTABLE] " REDIS_EXECUTABLE - if [ ! -x "$REDIS_EXECUTABLE" ] ; then - REDIS_EXECUTABLE=$_REDIS_EXECUTABLE - - if [ ! -x "$REDIS_EXECUTABLE" ] ; then - echo "Mmmmm... it seems like you don't have a redis executable. Did you run make install yet?" - exit 1 - fi - fi -fi - -#check the default for redis cli -CLI_EXEC=`command -v redis-cli` -if [ -z "$CLI_EXEC" ] ; then - CLI_EXEC=`dirname $REDIS_EXECUTABLE`"/redis-cli" -fi - -echo "Selected config:" - -echo "Port : $REDIS_PORT" -echo "Config file : $REDIS_CONFIG_FILE" -echo "Log file : $REDIS_LOG_FILE" -echo "Data dir : $REDIS_DATA_DIR" -echo "Executable : $REDIS_EXECUTABLE" -echo "Cli Executable : $CLI_EXEC" - -if $_MANUAL_EXECUTION == true ; then - read -p "Is this ok? Then press ENTER to go on or Ctrl-C to abort." _UNUSED_ -fi - -mkdir -p `dirname "$REDIS_CONFIG_FILE"` || die "Could not create redis config directory" -mkdir -p `dirname "$REDIS_LOG_FILE"` || die "Could not create redis log dir" -mkdir -p "$REDIS_DATA_DIR" || die "Could not create redis data directory" - -#render the templates -TMP_FILE="/tmp/${REDIS_PORT}.conf" -DEFAULT_CONFIG="${SCRIPTPATH}/../redis.conf" -INIT_TPL_FILE="${SCRIPTPATH}/redis_init_script.tpl" -INIT_SCRIPT_DEST="/etc/init.d/redis_${REDIS_PORT}" -PIDFILE="/var/run/redis_${REDIS_PORT}.pid" - -if [ ! -f "$DEFAULT_CONFIG" ]; then - echo "Mmmmm... the default config is missing. Did you switch to the utils directory?" - exit 1 -fi - -#Generate config file from the default config file as template -#changing only the stuff we're controlling from this script -echo "## Generated by install_server.sh ##" > $TMP_FILE - -read -r SED_EXPR <<-EOF -s#^port .\+#port ${REDIS_PORT}#; \ -s#^logfile .\+#logfile ${REDIS_LOG_FILE}#; \ -s#^dir .\+#dir ${REDIS_DATA_DIR}#; \ -s#^pidfile .\+#pidfile ${PIDFILE}#; \ -s#^daemonize no#daemonize yes#; -EOF -sed "$SED_EXPR" $DEFAULT_CONFIG >> $TMP_FILE - -#cat $TPL_FILE | while read line; do eval "echo \"$line\"" >> $TMP_FILE; done -cp $TMP_FILE $REDIS_CONFIG_FILE || die "Could not write redis config file $REDIS_CONFIG_FILE" - -#Generate sample script from template file -rm -f $TMP_FILE - -#we hard code the configs here to avoid issues with templates containing env vars -#kinda lame but works! -REDIS_INIT_HEADER=\ -"#!/bin/sh\n -#Configurations injected by install_server below....\n\n -EXEC=$REDIS_EXECUTABLE\n -CLIEXEC=$CLI_EXEC\n -PIDFILE=\"$PIDFILE\"\n -CONF=\"$REDIS_CONFIG_FILE\"\n\n -REDISPORT=\"$REDIS_PORT\"\n\n -###############\n\n" - -REDIS_CHKCONFIG_INFO=\ -"# REDHAT chkconfig header\n\n -# chkconfig: - 58 74\n -# description: redis_${REDIS_PORT} is the redis daemon.\n -### BEGIN INIT INFO\n -# Provides: redis_6379\n -# Required-Start: \$network \$local_fs \$remote_fs\n -# Required-Stop: \$network \$local_fs \$remote_fs\n -# Default-Start: 2 3 4 5\n -# Default-Stop: 0 1 6\n -# Should-Start: \$syslog \$named\n -# Should-Stop: \$syslog \$named\n -# Short-Description: start and stop redis_${REDIS_PORT}\n -# Description: Redis daemon\n -### END INIT INFO\n\n" - -if command -v chkconfig >/dev/null; then - #if we're a box with chkconfig on it we want to include info for chkconfig - echo "$REDIS_INIT_HEADER" "$REDIS_CHKCONFIG_INFO" > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE" -else - #combine the header and the template (which is actually a static footer) - echo "$REDIS_INIT_HEADER" > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE" -fi - -### -# Generate sample script from template file -# - No need to check which system we are on. The init info are comments and -# do not interfere with update_rc.d systems. Additionally: -# Ubuntu/debian by default does not come with chkconfig, but does issue a -# warning if init info is not available. - -cat > ${TMP_FILE} <<EOT -#!/bin/sh -#Configurations injected by install_server below.... - -EXEC=$REDIS_EXECUTABLE -CLIEXEC=$CLI_EXEC -PIDFILE=$PIDFILE -CONF="$REDIS_CONFIG_FILE" -REDISPORT="$REDIS_PORT" -############### -# SysV Init Information -# chkconfig: - 58 74 -# description: redis_${REDIS_PORT} is the redis daemon. -### BEGIN INIT INFO -# Provides: redis_${REDIS_PORT} -# Required-Start: \$network \$local_fs \$remote_fs -# Required-Stop: \$network \$local_fs \$remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Should-Start: \$syslog \$named -# Should-Stop: \$syslog \$named -# Short-Description: start and stop redis_${REDIS_PORT} -# Description: Redis daemon -### END INIT INFO - -EOT -cat ${INIT_TPL_FILE} >> ${TMP_FILE} - -#copy to /etc/init.d -cp $TMP_FILE $INIT_SCRIPT_DEST && \ - chmod +x $INIT_SCRIPT_DEST || die "Could not copy redis init script to $INIT_SCRIPT_DEST" -echo "Copied $TMP_FILE => $INIT_SCRIPT_DEST" - -#Install the service -echo "Installing service..." -if command -v chkconfig >/dev/null 2>&1; then - # we're chkconfig, so lets add to chkconfig and put in runlevel 345 - chkconfig --add redis_${REDIS_PORT} && echo "Successfully added to chkconfig!" - chkconfig --level 345 redis_${REDIS_PORT} on && echo "Successfully added to runlevels 345!" -elif command -v update-rc.d >/dev/null 2>&1; then - #if we're not a chkconfig box assume we're able to use update-rc.d - update-rc.d redis_${REDIS_PORT} defaults && echo "Success!" -else - echo "No supported init tool found." -fi - -/etc/init.d/redis_$REDIS_PORT start || die "Failed starting service..." - -#tada -echo "Installation successful!" -exit 0 diff --git a/examples/redis-unstable/utils/lru/README b/examples/redis-unstable/utils/lru/README deleted file mode 100644 index f043b29..0000000 --- a/examples/redis-unstable/utils/lru/README +++ /dev/null @@ -1,19 +0,0 @@ -The test-lru.rb program can be used in order to check the behavior of the -Redis approximated LRU algorithm against the theoretical output of true -LRU algorithm. - -In order to use the program you need to recompile Redis setting the define -REDIS_LRU_CLOCK_RESOLUTION to 1, by editing the file server.h. -This allows to execute the program in a fast way since the 1 ms resolution -is enough for all the objects to have a different enough time stamp during -the test. - -The program is executed like this: - - ruby test-lru.rb /tmp/lru.html - -You can optionally specify a number of times to run, so that the program -will output averages of different runs, by adding an additional argument. -For instance in order to run the test 10 times use: - - ruby test-lru.rb /tmp/lru.html 10 diff --git a/examples/redis-unstable/utils/lru/lfu-simulation.c b/examples/redis-unstable/utils/lru/lfu-simulation.c deleted file mode 100644 index 60105e5..0000000 --- a/examples/redis-unstable/utils/lru/lfu-simulation.c +++ /dev/null @@ -1,158 +0,0 @@ -#include <stdio.h> -#include <time.h> -#include <stdint.h> -#include <stdlib.h> - -int decr_every = 1; -int keyspace_size = 1000000; -time_t switch_after = 30; /* Switch access pattern after N seconds. */ - -struct entry { - /* Field that the LFU Redis implementation will have (we have - * 24 bits of total space in the object->lru field). */ - uint8_t counter; /* Logarithmic counter. */ - uint16_t decrtime; /* (Reduced precision) time of last decrement. */ - - /* Fields only useful for visualization. */ - uint64_t hits; /* Number of real accesses. */ - time_t ctime; /* Key creation time. */ -}; - -#define to_16bit_minutes(x) ((x/60) & 65535) -#define LFU_INIT_VAL 5 - -/* Compute the difference in minutes between two 16 bit minutes times - * obtained with to_16bit_minutes(). Since they can wrap around if - * we detect the overflow we account for it as if the counter wrapped - * a single time. */ -uint16_t minutes_diff(uint16_t now, uint16_t prev) { - if (now >= prev) return now-prev; - return 65535-prev+now; -} - -/* Increment a counter logarithmically: the greatest is its value, the - * less likely is that the counter is really incremented. - * The maximum value of the counter is saturated at 255. */ -uint8_t log_incr(uint8_t counter) { - if (counter == 255) return counter; - double r = (double)rand()/RAND_MAX; - double baseval = counter-LFU_INIT_VAL; - if (baseval < 0) baseval = 0; - double limit = 1.0/(baseval*10+1); - if (r < limit) counter++; - return counter; -} - -/* Simulate an access to an entry. */ -void access_entry(struct entry *e) { - e->counter = log_incr(e->counter); - e->hits++; -} - -/* Return the entry LFU value and as a side effect decrement the - * entry value if the decrement time was reached. */ -uint8_t scan_entry(struct entry *e) { - if (minutes_diff(to_16bit_minutes(time(NULL)),e->decrtime) - >= decr_every) - { - if (e->counter) { - if (e->counter > LFU_INIT_VAL*2) { - e->counter /= 2; - } else { - e->counter--; - } - } - e->decrtime = to_16bit_minutes(time(NULL)); - } - return e->counter; -} - -/* Print the entry info. */ -void show_entry(long pos, struct entry *e) { - char *tag = "normal "; - - if (pos >= 10 && pos <= 14) tag = "new no access"; - if (pos >= 15 && pos <= 19) tag = "new accessed "; - if (pos >= keyspace_size -5) tag= "old no access"; - - printf("%ld] <%s> frequency:%d decrtime:%d [%lu hits | age:%ld sec]\n", - pos, tag, e->counter, e->decrtime, (unsigned long)e->hits, - time(NULL) - e->ctime); -} - -int main(void) { - time_t start = time(NULL); - time_t new_entry_time = start; - time_t display_time = start; - struct entry *entries = malloc(sizeof(*entries)*keyspace_size); - long j; - - /* Initialize. */ - for (j = 0; j < keyspace_size; j++) { - entries[j].counter = LFU_INIT_VAL; - entries[j].decrtime = to_16bit_minutes(start); - entries[j].hits = 0; - entries[j].ctime = time(NULL); - } - - while(1) { - time_t now = time(NULL); - long idx; - - /* Scan N random entries (simulates the eviction under maxmemory). */ - for (j = 0; j < 3; j++) { - scan_entry(entries+(rand()%keyspace_size)); - } - - /* Access a random entry: use a power-law access pattern up to - * 'switch_after' seconds. Then revert to flat access pattern. */ - if (now-start < switch_after) { - /* Power law. */ - idx = 1; - while((rand() % 21) != 0 && idx < keyspace_size) idx *= 2; - if (idx > keyspace_size) idx = keyspace_size; - idx = rand() % idx; - } else { - /* Flat. */ - idx = rand() % keyspace_size; - } - - /* Never access entries between position 10 and 14, so that - * we simulate what happens to new entries that are never - * accessed VS new entries which are accessed in positions - * 15-19. - * - * Also never access last 5 entry, so that we have keys which - * are never recreated (old), and never accessed. */ - if ((idx < 10 || idx > 14) && (idx < keyspace_size-5)) - access_entry(entries+idx); - - /* Simulate the addition of new entries at positions between - * 10 and 19, a random one every 10 seconds. */ - if (new_entry_time <= now) { - idx = 10+(rand()%10); - entries[idx].counter = LFU_INIT_VAL; - entries[idx].decrtime = to_16bit_minutes(time(NULL)); - entries[idx].hits = 0; - entries[idx].ctime = time(NULL); - new_entry_time = now+10; - } - - /* Show the first 20 entries and the last 20 entries. */ - if (display_time != now) { - printf("=============================\n"); - printf("Current minutes time: %d\n", (int)to_16bit_minutes(now)); - printf("Access method: %s\n", - (now-start < switch_after) ? "power-law" : "flat"); - - for (j = 0; j < 20; j++) - show_entry(j,entries+j); - - for (j = keyspace_size-20; j < keyspace_size; j++) - show_entry(j,entries+j); - display_time = now; - } - } - return 0; -} - diff --git a/examples/redis-unstable/utils/lru/test-lru.rb b/examples/redis-unstable/utils/lru/test-lru.rb deleted file mode 100644 index d511e20..0000000 --- a/examples/redis-unstable/utils/lru/test-lru.rb +++ /dev/null @@ -1,223 +0,0 @@ -require 'rubygems' -require 'redis' - -$runs = []; # Remember the error rate of each run for average purposes. -$o = {}; # Options set parsing arguments - -def testit(filename) - r = Redis.new - r.config("SET","maxmemory","2000000") - if $o[:ttl] - r.config("SET","maxmemory-policy","volatile-ttl") - else - r.config("SET","maxmemory-policy","allkeys-lru") - end - r.config("SET","maxmemory-samples",5) - r.config("RESETSTAT") - r.flushall - - html = "" - html << <<EOF - <html> - <body> - <style> - .box { - width:5px; - height:5px; - float:left; - margin: 1px; - } - - .old { - border: 1px black solid; - } - - .new { - border: 1px green solid; - } - - .otherdb { - border: 1px red solid; - } - - .ex { - background-color: #666; - } - </style> - <pre> -EOF - - # Fill the DB up to the first eviction. - oldsize = r.dbsize - id = 0 - while true - id += 1 - begin - r.set(id,"foo") - rescue - break - end - newsize = r.dbsize - break if newsize == oldsize # A key was evicted? Stop. - oldsize = newsize - end - - inserted = r.dbsize - first_set_max_id = id - html << "#{r.dbsize} keys inserted.\n" - - # Access keys sequentially, so that in theory the first part will be expired - # and the latter part will not, according to perfect LRU. - - if $o[:ttl] - STDERR.puts "Set increasing expire value" - (1..first_set_max_id).each{|id| - r.expire(id,1000+id) - STDERR.print(".") if (id % 150) == 0 - } - else - STDERR.puts "Access keys sequentially" - (1..first_set_max_id).each{|id| - r.get(id) - sleep 0.001 - STDERR.print(".") if (id % 150) == 0 - } - end - STDERR.puts - - # Insert more 50% keys. We expect that the new keys will rarely be expired - # since their last access time is recent compared to the others. - # - # Note that we insert the first 100 keys of the new set into DB1 instead - # of DB0, so that we can try how cross-DB eviction works. - half = inserted/2 - html << "Insert enough keys to evict half the keys we inserted.\n" - add = 0 - - otherdb_start_idx = id+1 - otherdb_end_idx = id+100 - while true - add += 1 - id += 1 - if id >= otherdb_start_idx && id <= otherdb_end_idx - r.select(1) - r.set(id,"foo") - r.select(0) - else - r.set(id,"foo") - end - break if r.info['evicted_keys'].to_i >= half - end - - html << "#{add} additional keys added.\n" - html << "#{r.dbsize} keys in DB.\n" - - # Check if evicted keys respect LRU - # We consider errors from 1 to N progressively more serious as they violate - # more the access pattern. - - errors = 0 - e = 1 - error_per_key = 100000.0/first_set_max_id - half_set_size = first_set_max_id/2 - maxerr = 0 - (1..(first_set_max_id/2)).each{|id| - if id >= otherdb_start_idx && id <= otherdb_end_idx - r.select(1) - exists = r.exists(id) - r.select(0) - else - exists = r.exists(id) - end - if id < first_set_max_id/2 - thiserr = error_per_key * ((half_set_size-id).to_f/half_set_size) - maxerr += thiserr - errors += thiserr if exists - elsif id >= first_set_max_id/2 - thiserr = error_per_key * ((id-half_set_size).to_f/half_set_size) - maxerr += thiserr - errors += thiserr if !exists - end - } - errors = errors*100/maxerr - - STDERR.puts "Test finished with #{errors}% error! Generating HTML on stdout." - - html << "#{errors}% error!\n" - html << "</pre>" - $runs << errors - - # Generate the graphical representation - (1..id).each{|id| - # Mark first set and added items in a different way. - c = "box" - if id >= otherdb_start_idx && id <= otherdb_end_idx - c << " otherdb" - elsif id <= first_set_max_id - c << " old" - else - c << " new" - end - - # Add class if exists - if id >= otherdb_start_idx && id <= otherdb_end_idx - r.select(1) - exists = r.exists(id) - r.select(0) - else - exists = r.exists(id) - end - - c << " ex" if exists - html << "<div title=\"#{id}\" class=\"#{c}\"></div>" - } - - # Close HTML page - - html << <<EOF - </body> - </html> -EOF - - f = File.open(filename,"w") - f.write(html) - f.close -end - -def print_avg - avg = ($runs.reduce {|a,b| a+b}) / $runs.length - puts "#{$runs.length} runs, AVG is #{avg}" -end - -if ARGV.length < 1 - STDERR.puts "Usage: ruby test-lru.rb <html-output-filename> [--runs <count>] [--ttl]" - STDERR.puts "Options:" - STDERR.puts " --runs <count> Execute the test <count> times." - STDERR.puts " --ttl Set keys with increasing TTL values" - STDERR.puts " (starting from 1000 seconds) in order to" - STDERR.puts " test the volatile-lru policy." - exit 1 -end - -filename = ARGV[0] -$o[:numruns] = 1 - -# Options parsing -i = 1 -while i < ARGV.length - if ARGV[i] == '--runs' - $o[:numruns] = ARGV[i+1].to_i - i+= 1 - elsif ARGV[i] == '--ttl' - $o[:ttl] = true - else - STDERR.puts "Unknown option #{ARGV[i]}" - exit 1 - end - i+= 1 -end - -$o[:numruns].times { - testit(filename) - print_avg if $o[:numruns] != 1 -} diff --git a/examples/redis-unstable/utils/redis-copy.rb b/examples/redis-unstable/utils/redis-copy.rb deleted file mode 100644 index 816b1e4..0000000 --- a/examples/redis-unstable/utils/redis-copy.rb +++ /dev/null @@ -1,38 +0,0 @@ -# redis-copy.rb - Copyright (C) 2009-Present Redis Ltd. All rights reserved. -# -# Licensed under your choice of (a) the Redis Source Available License 2.0 -# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the -# GNU Affero General Public License v3 (AGPLv3). -# -# Copy the whole dataset from one Redis instance to another one -# -# WARNING: this utility is deprecated and serves as a legacy adapter -# for the more-robust redis-copy gem. - -require 'shellwords' - -def redisCopy(opts={}) - src = "#{opts[:srchost]}:#{opts[:srcport]}" - dst = "#{opts[:dsthost]}:#{opts[:dstport]}" - `redis-copy #{src.shellescape} #{dst.shellescape}` -rescue Errno::ENOENT - $stderr.puts 'This utility requires the redis-copy executable', - 'from the redis-copy gem on https://rubygems.org', - 'To install it, run `gem install redis-copy`.' - exit 1 -end - -$stderr.puts "This utility is deprecated. Use the redis-copy gem instead." -if ARGV.length != 4 - puts "Usage: redis-copy.rb <srchost> <srcport> <dsthost> <dstport>" - exit 1 -end -puts "WARNING: it's up to you to FLUSHDB the destination host before to continue, press any key when ready." -STDIN.gets -srchost = ARGV[0] -srcport = ARGV[1] -dsthost = ARGV[2] -dstport = ARGV[3] -puts "Copying #{srchost}:#{srcport} into #{dsthost}:#{dstport}" -redisCopy(:srchost => srchost, :srcport => srcport.to_i, - :dsthost => dsthost, :dstport => dstport.to_i) diff --git a/examples/redis-unstable/utils/redis-sha1.rb b/examples/redis-unstable/utils/redis-sha1.rb deleted file mode 100644 index dc87c77..0000000 --- a/examples/redis-unstable/utils/redis-sha1.rb +++ /dev/null @@ -1,55 +0,0 @@ -# redis-sha1.rb - Copyright (C) 2009-Present Redis Ltd. All rights reserved. -# -# Licensed under your choice of (a) the Redis Source Available License 2.0 -# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the -# GNU Affero General Public License v3 (AGPLv3). -# -# Performs the SHA1 sum of the whole dataset. -# This is useful to spot bugs in persistence related code and to make sure -# Slaves and Masters are in SYNC. -# -# If you hack this code make sure to sort keys and set elements as this are -# unsorted elements. Otherwise the sum may differ with equal dataset. - -require 'rubygems' -require 'redis' -require 'digest/sha1' - -def redisSha1(opts={}) - sha1="" - r = Redis.new(opts) - r.keys('*').sort.each{|k| - vtype = r.type?(k) - if vtype == "string" - len = 1 - sha1 = Digest::SHA1.hexdigest(sha1+k) - sha1 = Digest::SHA1.hexdigest(sha1+r.get(k)) - elsif vtype == "list" - len = r.llen(k) - if len != 0 - sha1 = Digest::SHA1.hexdigest(sha1+k) - sha1 = Digest::SHA1.hexdigest(sha1+r.list_range(k,0,-1).join("\x01")) - end - elsif vtype == "set" - len = r.scard(k) - if len != 0 - sha1 = Digest::SHA1.hexdigest(sha1+k) - sha1 = Digest::SHA1.hexdigest(sha1+r.set_members(k).to_a.sort.join("\x02")) - end - elsif vtype == "zset" - len = r.zcard(k) - if len != 0 - sha1 = Digest::SHA1.hexdigest(sha1+k) - sha1 = Digest::SHA1.hexdigest(sha1+r.zrange(k,0,-1).join("\x01")) - end - end - # puts "#{k} => #{sha1}" if len != 0 - } - sha1 -end - -host = ARGV[0] || "127.0.0.1" -port = ARGV[1] || "6379" -db = ARGV[2] || "0" -puts "Performing SHA1 of Redis server #{host} #{port} DB: #{db}" -p "Dataset SHA1: #{redisSha1(:host => host, :port => port.to_i, :db => db)}" diff --git a/examples/redis-unstable/utils/redis_init_script b/examples/redis-unstable/utils/redis_init_script deleted file mode 100755 index 006db87..0000000 --- a/examples/redis-unstable/utils/redis_init_script +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -# -# Simple Redis init.d script conceived to work on Linux systems -# as it does use of the /proc filesystem. - -### BEGIN INIT INFO -# Provides: redis_6379 -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Redis data structure server -# Description: Redis data structure server. See https://redis.io -### END INIT INFO - -REDISPORT=6379 -EXEC=/usr/local/bin/redis-server -CLIEXEC=/usr/local/bin/redis-cli - -PIDFILE=/var/run/redis_${REDISPORT}.pid -CONF="/etc/redis/${REDISPORT}.conf" - -case "$1" in - start) - if [ -f $PIDFILE ] - then - echo "$PIDFILE exists, process is already running or crashed" - else - echo "Starting Redis server..." - $EXEC $CONF - fi - ;; - stop) - if [ ! -f $PIDFILE ] - then - echo "$PIDFILE does not exist, process is not running" - else - PID=$(cat $PIDFILE) - echo "Stopping ..." - $CLIEXEC -p $REDISPORT shutdown - while [ -x /proc/${PID} ] - do - echo "Waiting for Redis to shutdown ..." - sleep 1 - done - echo "Redis stopped" - fi - ;; - *) - echo "Please use start or stop as first argument" - ;; -esac diff --git a/examples/redis-unstable/utils/redis_init_script.tpl b/examples/redis-unstable/utils/redis_init_script.tpl deleted file mode 100755 index 2e5b613..0000000 --- a/examples/redis-unstable/utils/redis_init_script.tpl +++ /dev/null @@ -1,44 +0,0 @@ - -case "$1" in - start) - if [ -f $PIDFILE ] - then - echo "$PIDFILE exists, process is already running or crashed" - else - echo "Starting Redis server..." - $EXEC $CONF - fi - ;; - stop) - if [ ! -f $PIDFILE ] - then - echo "$PIDFILE does not exist, process is not running" - else - PID=$(cat $PIDFILE) - echo "Stopping ..." - $CLIEXEC -p $REDISPORT shutdown - while [ -x /proc/${PID} ] - do - echo "Waiting for Redis to shutdown ..." - sleep 1 - done - echo "Redis stopped" - fi - ;; - status) - PID=$(cat $PIDFILE) - if [ ! -x /proc/${PID} ] - then - echo 'Redis is not running' - else - echo "Redis is running ($PID)" - fi - ;; - restart) - $0 stop - $0 start - ;; - *) - echo "Please use start, stop, restart or status as first argument" - ;; -esac diff --git a/examples/redis-unstable/utils/releasetools/01_create_tarball.sh b/examples/redis-unstable/utils/releasetools/01_create_tarball.sh deleted file mode 100755 index 366a61e..0000000 --- a/examples/redis-unstable/utils/releasetools/01_create_tarball.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -if [ $# != "1" ] -then - echo "Usage: ./utils/releasetools/01_create_tarball.sh <version_tag>" - exit 1 -fi - -TAG=$1 -TARNAME="redis-${TAG}.tar" -echo "Generating /tmp/${TARNAME}" -git archive $TAG --prefix redis-${TAG}/ > /tmp/$TARNAME || exit 1 -echo "Gizipping the archive" -rm -f /tmp/$TARNAME.gz -gzip -9 /tmp/$TARNAME diff --git a/examples/redis-unstable/utils/releasetools/02_upload_tarball.sh b/examples/redis-unstable/utils/releasetools/02_upload_tarball.sh deleted file mode 100755 index ef1e777..0000000 --- a/examples/redis-unstable/utils/releasetools/02_upload_tarball.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -if [ $# != "1" ] -then - echo "Usage: ./utils/releasetools/02_upload_tarball.sh <version_tag>" - exit 1 -fi - -echo "Uploading..." -scp /tmp/redis-${1}.tar.gz ubuntu@host.redis.io:/var/www/download/releases/ -echo "Updating web site... " -echo "Please check the github action tests for the release." -echo "Press any key if it is a stable release, or Ctrl+C to abort" -read x -ssh ubuntu@host.redis.io "cd /var/www/download; - rm -rf redis-${1}.tar.gz; - wget http://download.redis.io/releases/redis-${1}.tar.gz; - tar xvzf redis-${1}.tar.gz; - rm -rf redis-stable; - mv redis-${1} redis-stable; - tar cvzf redis-stable.tar.gz redis-stable; - rm -rf redis-${1}.tar.gz; - shasum -a 256 redis-stable.tar.gz > redis-stable.tar.gz.SHA256SUM; - " diff --git a/examples/redis-unstable/utils/releasetools/03_test_release.sh b/examples/redis-unstable/utils/releasetools/03_test_release.sh deleted file mode 100755 index 493d0b7..0000000 --- a/examples/redis-unstable/utils/releasetools/03_test_release.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -set -e -if [ $# != "1" ] -then - echo "Usage: ./utils/releasetools/03_test_release.sh <version_tag>" - exit 1 -fi - -TAG=$1 -TARNAME="redis-${TAG}.tar.gz" -DOWNLOADURL="http://download.redis.io/releases/${TARNAME}" - -echo "Doing sanity test on the actual tarball" - -cd /tmp -rm -rf test_release_tmp_dir -mkdir test_release_tmp_dir -cd test_release_tmp_dir -rm -f $TARNAME -rm -rf redis-${TAG} -wget $DOWNLOADURL -tar xvzf $TARNAME -cd redis-${TAG} -make -./runtest -./runtest-sentinel -./runtest-cluster -./runtest-moduleapi diff --git a/examples/redis-unstable/utils/releasetools/04_release_hash.sh b/examples/redis-unstable/utils/releasetools/04_release_hash.sh deleted file mode 100755 index d932928..0000000 --- a/examples/redis-unstable/utils/releasetools/04_release_hash.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -if [ $# != "1" ] -then - echo "Usage: ./utils/releasetools/04_release_hash.sh <version_tag>" - exit 1 -fi - -SHA=$(curl -s http://download.redis.io/releases/redis-${1}.tar.gz | shasum -a 256 | cut -f 1 -d' ') -ENTRY="hash redis-${1}.tar.gz sha256 $SHA http://download.redis.io/releases/redis-${1}.tar.gz" -echo $ENTRY >> ../redis-hashes/README -echo "Press any key to commit, Ctrl-C to abort)." -read yes -(cd ../redis-hashes; git commit -a -m "${1} hash."; git push) diff --git a/examples/redis-unstable/utils/releasetools/changelog.tcl b/examples/redis-unstable/utils/releasetools/changelog.tcl deleted file mode 100755 index 2288794..0000000 --- a/examples/redis-unstable/utils/releasetools/changelog.tcl +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env tclsh - -if {[llength $::argv] != 2 && [llength $::argv] != 3} { - puts "Usage: $::argv0 <branch> <version> \[<num-commits>\]" - exit 1 -} - -set branch [lindex $::argv 0] -set ver [lindex $::argv 1] -if {[llength $::argv] == 3} { - set count [lindex ::$argv 2] -} else { - set count 100 -} - -set template { -================================================================================ -Redis %ver% Released %date% -================================================================================ - -Upgrade urgency <URGENCY>: <DESCRIPTION> -} - -set template [string trim $template] -append template "\n\n" -set date [clock format [clock seconds]] -set template [string map [list %ver% $ver %date% $date] $template] - -append template [exec git log $branch~$count..$branch "--format=format:%an in commit %h:%n %s" --shortstat] - -#Older, more verbose version. -# -#append template [exec git log $branch~30..$branch "--format=format:+-------------------------------------------------------------------------------%n| %s%n| By %an, %ai%n+--------------------------------------------------------------------------------%nhttps://github.com/redis/redis/commit/%H%n%n%b" --stat] - -puts $template diff --git a/examples/redis-unstable/utils/reply_schema_linter.js b/examples/redis-unstable/utils/reply_schema_linter.js deleted file mode 100644 index e2358d4..0000000 --- a/examples/redis-unstable/utils/reply_schema_linter.js +++ /dev/null @@ -1,31 +0,0 @@ -function validate_schema(command_schema) { - var error_status = false - const Ajv = require("ajv/dist/2019") - const ajv = new Ajv({strict: true, strictTuples: false}) - let json = require('../src/commands/'+ command_schema); - for (var item in json) { - const schema = json[item].reply_schema - if (schema == undefined) - continue; - try { - ajv.compile(schema) - } catch (error) { - console.error(command_schema + " : " + error.toString()) - error_status = true - } - } - return error_status -} - -const schema_directory_path = './src/commands' -const path = require('path') -var fs = require('fs'); -var files = fs.readdirSync(schema_directory_path); -jsonFiles = files.filter(el => path.extname(el) === '.json') -var error_status = false -jsonFiles.forEach(function(file){ - if (validate_schema(file)) - error_status = true -}) -if (error_status) - process.exit(1) diff --git a/examples/redis-unstable/utils/req-res-log-validator.py b/examples/redis-unstable/utils/req-res-log-validator.py deleted file mode 100755 index 0f00935..0000000 --- a/examples/redis-unstable/utils/req-res-log-validator.py +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/env python3 -import os -import glob -import json -import sys - -import jsonschema -import subprocess -import redis -import time -import argparse -import multiprocessing -import collections -import io -import traceback -from datetime import timedelta -from functools import partial -try: - from jsonschema import Draft201909Validator as schema_validator -except ImportError: - from jsonschema import Draft7Validator as schema_validator - -""" -The purpose of this file is to validate the reply_schema values of COMMAND DOCS. -Basically, this is what it does: -1. Goes over req-res files, generated by redis-servers, spawned by the testsuite (see logreqres.c) -2. For each request-response pair, it validates the response against the request's reply_schema (obtained from COMMAND DOCS) - -This script spins up a redis-server and a redis-cli in order to obtain COMMAND DOCS. - -In order to use this file you must run the redis testsuite with the following flags: -./runtest --dont-clean --force-resp3 --log-req-res - -And then: -./utils/req-res-log-validator.py - -The script will fail only if: -1. One or more of the replies doesn't comply with its schema. -2. One or more of the commands in COMMANDS DOCS doesn't have the reply_schema field (with --fail-missing-reply-schemas) -3. The testsuite didn't execute all of the commands (with --fail-commands-not-all-hit) - -Future validations: -1. Fail the script if one or more of the branches of the reply schema (e.g. oneOf, anyOf) was not hit. -""" - -IGNORED_COMMANDS = { - # Commands that don't work in a req-res manner (see logreqres.c) - "debug", # because of DEBUG SEGFAULT - "sync", - "psync", - "monitor", - "subscribe", - "unsubscribe", - "ssubscribe", - "sunsubscribe", - "psubscribe", - "punsubscribe", - # Commands to which we decided not write a reply schema - "pfdebug", - "lolwut", - # TODO: for vector-sets module - "VADD", - "VCARD", - "VDIM", - "VEMB", - "VGETATTR", - "VINFO", - "VLINKS", - "VRANDMEMBER", - "VREM", - "VSETATTR", - "VSIM", - "VISMEMBER", - "VRANGE", -} - -class Request(object): - """ - This class represents a Redis request (AKA command, argv) - """ - def __init__(self, f, docs, line_counter): - """ - Read lines from `f` (generated by logreqres.c) and populates the argv array - """ - self.command = None - self.schema = None - self.argv = [] - - while True: - line = f.readline() - line_counter[0] += 1 - if not line: - break - length = int(line) - arg = str(f.read(length)) - f.read(2) # read \r\n - line_counter[0] += 1 - if arg == "__argv_end__": - break - self.argv.append(arg) - - if not self.argv: - return - - self.command = self.argv[0].lower() - doc = docs.get(self.command, {}) - if not doc and len(self.argv) > 1: - self.command = f"{self.argv[0].lower()}|{self.argv[1].lower()}" - doc = docs.get(self.command, {}) - - if not doc: - self.command = None - return - - self.schema = doc.get("reply_schema") - - def __str__(self): - return json.dumps(self.argv) - - -class Response(object): - """ - This class represents a Redis response in RESP3 - """ - def __init__(self, f, line_counter): - """ - Read lines from `f` (generated by logreqres.c) and build the JSON representing the response in RESP3 - """ - self.error = False - self.queued = False - self.json = None - - line = f.readline()[:-2] - line_counter[0] += 1 - if line[0] == '+': - self.json = line[1:] - if self.json == "QUEUED": - self.queued = True - elif line[0] == '-': - self.json = line[1:] - self.error = True - elif line[0] == '$': - self.json = str(f.read(int(line[1:]))) - f.read(2) # read \r\n - line_counter[0] += 1 - elif line[0] == ':': - self.json = int(line[1:]) - elif line[0] == ',': - self.json = float(line[1:]) - elif line[0] == '_': - self.json = None - elif line[0] == '#': - self.json = line[1] == 't' - elif line[0] == '!': - self.json = str(f.read(int(line[1:]))) - f.read(2) # read \r\n - line_counter[0] += 1 - self.error = True - elif line[0] == '=': - self.json = str(f.read(int(line[1:])))[4:] # skip "txt:" or "mkd:" - f.read(2) # read \r\n - line_counter[0] += 1 + self.json.count("\r\n") - elif line[0] == '(': - self.json = line[1:] # big-number is actually a string - elif line[0] in ['*', '~', '>']: # unfortunately JSON doesn't tell the difference between a list and a set - self.json = [] - count = int(line[1:]) - for i in range(count): - ele = Response(f, line_counter) - self.json.append(ele.json) - elif line[0] in ['%', '|']: - self.json = {} - count = int(line[1:]) - for i in range(count): - field = Response(f, line_counter) - # Redis allows fields to be non-strings but JSON doesn't. - # Luckily, for any kind of response we can validate, the fields are - # always strings (example: XINFO STREAM) - # The reason we can't always convert to string is because of DEBUG PROTOCOL MAP - # which anyway doesn't have a schema - if isinstance(field.json, str): - field = field.json - value = Response(f, line_counter) - self.json[field] = value.json - if line[0] == '|': - # We don't care about the attributes, read the real response - real_res = Response(f, line_counter) - self.__dict__.update(real_res.__dict__) - - - def __str__(self): - return json.dumps(self.json) - - -def process_file(docs, path): - """ - This function processes a single file generated by logreqres.c - """ - line_counter = [0] # A list with one integer: to force python to pass it by reference - command_counter = dict() - - print(f"Processing {path} ...") - - # Convert file to StringIO in order to minimize IO operations - with open(path, "r", newline="\r\n", encoding="latin-1") as f: - content = f.read() - - with io.StringIO(content) as fakefile: - while True: - try: - req = Request(fakefile, docs, line_counter) - if not req.argv: - # EOF - break - res = Response(fakefile, line_counter) - except json.decoder.JSONDecodeError as err: - print(f"JSON decoder error while processing {path}:{line_counter[0]}: {err}") - print(traceback.format_exc()) - raise - except Exception as err: - print(f"General error while processing {path}:{line_counter[0]}: {err}") - print(traceback.format_exc()) - raise - - if not req.command: - # Unknown command - continue - - command_counter[req.command] = command_counter.get(req.command, 0) + 1 - - if res.error or res.queued: - continue - - if req.command in IGNORED_COMMANDS: - continue - - try: - jsonschema.validate(instance=res.json, schema=req.schema, cls=schema_validator) - except (jsonschema.ValidationError, jsonschema.exceptions.SchemaError) as err: - print(f"JSON schema validation error on {path}: {err}") - print(f"argv: {req.argv}") - try: - print(f"Response: {res}") - except UnicodeDecodeError as err: - print("Response: (unprintable)") - print(f"Schema: {json.dumps(req.schema, indent=2)}") - print(traceback.format_exc()) - raise - - return command_counter - - -def fetch_schemas(cli, port, args, docs): - redis_proc = subprocess.Popen(args, stdout=subprocess.PIPE) - - while True: - try: - print('Connecting to Redis...') - r = redis.Redis(port=port) - r.ping() - break - except Exception as e: - time.sleep(0.1) - - print('Connected') - - cli_proc = subprocess.Popen([cli, '-p', str(port), '--json', 'command', 'docs'], stdout=subprocess.PIPE) - stdout, stderr = cli_proc.communicate() - docs_response = json.loads(stdout) - - for name, doc in docs_response.items(): - if "subcommands" in doc: - for subname, subdoc in doc["subcommands"].items(): - docs[subname] = subdoc - else: - docs[name] = doc - - redis_proc.terminate() - redis_proc.wait() - - -if __name__ == '__main__': - # Figure out where the sources are - srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src") - testdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../tests") - - parser = argparse.ArgumentParser() - parser.add_argument('--server', type=str, default='%s/redis-server' % srcdir) - parser.add_argument('--port', type=int, default=6534) - parser.add_argument('--cli', type=str, default='%s/redis-cli' % srcdir) - parser.add_argument('--module', type=str, action='append', default=[]) - parser.add_argument('--verbose', action='store_true') - parser.add_argument('--fail-commands-not-all-hit', action='store_true') - parser.add_argument('--fail-missing-reply-schemas', action='store_true') - args = parser.parse_args() - - docs = dict() - - # Fetch schemas from a Redis instance - print('Starting Redis server') - redis_args = [args.server, '--port', str(args.port)] - for module in args.module: - redis_args += ['--loadmodule', 'tests/modules/%s.so' % module] - - fetch_schemas(args.cli, args.port, redis_args, docs) - - # Fetch schemas from a sentinel - print('Starting Redis sentinel') - - # Sentinel needs a config file to start - config_file = "tmpsentinel.conf" - open(config_file, 'a').close() - - sentinel_args = [args.server, config_file, '--port', str(args.port), "--sentinel"] - fetch_schemas(args.cli, args.port, sentinel_args, docs) - os.unlink(config_file) - - missing_schema = [k for k, v in docs.items() - if "reply_schema" not in v and k not in IGNORED_COMMANDS] - if missing_schema: - print("WARNING! The following commands are missing a reply_schema:") - for k in sorted(missing_schema): - print(f" {k}") - if args.fail_missing_reply_schemas: - print("ERROR! at least one command does not have a reply_schema") - sys.exit(1) - - start = time.time() - - # Obtain all the files to processes - paths = [] - for path in glob.glob('%s/tmp/*/*.reqres' % testdir): - paths.append(path) - - for path in glob.glob('%s/cluster/tmp/*/*.reqres' % testdir): - paths.append(path) - - for path in glob.glob('%s/sentinel/tmp/*/*.reqres' % testdir): - paths.append(path) - - counter = collections.Counter() - # Spin several processes to handle the files in parallel - with multiprocessing.Pool(multiprocessing.cpu_count()) as pool: - func = partial(process_file, docs) - # pool.map blocks until all the files have been processed - for result in pool.map(func, paths): - counter.update(result) - command_counter = dict(counter) - - elapsed = time.time() - start - print(f"Done. ({timedelta(seconds=elapsed)})") - print("Hits per command:") - for k, v in sorted(command_counter.items()): - print(f" {k}: {v}") - not_hit = set(set(docs.keys()) - set(command_counter.keys()) - set(IGNORED_COMMANDS)) - if not_hit: - if args.verbose: - print("WARNING! The following commands were not hit at all:") - for k in sorted(not_hit): - print(f" {k}") - if args.fail_commands_not_all_hit: - print("ERROR! at least one command was not hit by the tests") - sys.exit(1) - diff --git a/examples/redis-unstable/utils/req-res-validator/requirements.txt b/examples/redis-unstable/utils/req-res-validator/requirements.txt deleted file mode 100644 index 0e3024b..0000000 --- a/examples/redis-unstable/utils/req-res-validator/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -jsonschema==4.17.3 -redis==4.5.1
\ No newline at end of file diff --git a/examples/redis-unstable/utils/speed-regression.tcl b/examples/redis-unstable/utils/speed-regression.tcl deleted file mode 100755 index 62b5ec8..0000000 --- a/examples/redis-unstable/utils/speed-regression.tcl +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env tclsh8.5 -# Copyright (C) 2011-Present Redis Ltd. All rights reserved. -# -# Licensed under your choice of (a) the Redis Source Available License 2.0 -# (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the -# GNU Affero General Public License v3 (AGPLv3). - -source ../tests/support/redis.tcl -set ::port 12123 -set ::tests {PING,SET,GET,INCR,LPUSH,LPOP,SADD,SPOP,LRANGE_100,LRANGE_600,MSET} -set ::datasize 16 -set ::requests 100000 - -proc run-tests branches { - set runs {} - set branch_id 0 - foreach b $branches { - cd ../src - puts "Benchmarking $b" - exec -ignorestderr git checkout $b 2> /dev/null - exec -ignorestderr make clean 2> /dev/null - puts " compiling..." - exec -ignorestderr make 2> /dev/null - - if {$branch_id == 0} { - puts " copy redis-benchmark from unstable to /tmp..." - exec -ignorestderr cp ./redis-benchmark /tmp - incr branch_id - continue - } - - # Start the Redis server - puts " starting the server... [exec ./redis-server -v]" - set pids [exec echo "port $::port\nloglevel warning\n" | ./redis-server - > /dev/null 2> /dev/null &] - puts " pids: $pids" - after 1000 - puts " running the benchmark" - - set r [redis 127.0.0.1 $::port] - set i [$r info] - puts " redis INFO shows version: [lindex [split $i] 0]" - $r close - - set output [exec /tmp/redis-benchmark -n $::requests -t $::tests -d $::datasize --csv -p $::port] - lappend runs $b $output - puts " killing server..." - catch {exec kill -9 [lindex $pids 0]} - catch {exec kill -9 [lindex $pids 1]} - incr branch_id - } - return $runs -} - -proc get-result-with-name {output name} { - foreach line [split $output "\n"] { - lassign [split $line ","] key value - set key [string tolower [string range $key 1 end-1]] - set value [string range $value 1 end-1] - if {$key eq [string tolower $name]} { - return $value - } - } - return "n/a" -} - -proc get-test-names output { - set names {} - foreach line [split $output "\n"] { - lassign [split $line ","] key value - set key [string tolower [string range $key 1 end-1]] - lappend names $key - } - return $names -} - -proc combine-results {results} { - set tests [get-test-names [lindex $results 1]] - foreach test $tests { - puts $test - foreach {branch output} $results { - puts [format "%-20s %s" \ - $branch [get-result-with-name $output $test]] - } - puts {} - } -} - -proc main {} { - # Note: the first branch is only used in order to get the redis-benchmark - # executable. Tests are performed starting from the second branch. - set branches { - slowset 2.2.0 2.4.0 unstable slowset - } - set results [run-tests $branches] - puts "\n" - puts "# Test results: datasize=$::datasize requests=$::requests" - puts [combine-results $results] -} - -# Force the user to run the script from the 'utils' directory. -if {![file exists speed-regression.tcl]} { - puts "Please make sure to run speed-regression.tcl while inside /utils." - puts "Example: cd utils; ./speed-regression.tcl" - exit 1 -} - -# Make sure there is not already a server running on port 12123 -set is_not_running [catch {set r [redis 127.0.0.1 $::port]}] -if {!$is_not_running} { - puts "Sorry, you have a running server on port $::port" - exit 1 -} - -# parse arguments -for {set j 0} {$j < [llength $argv]} {incr j} { - set opt [lindex $argv $j] - set arg [lindex $argv [expr $j+1]] - if {$opt eq {--tests}} { - set ::tests $arg - incr j - } elseif {$opt eq {--datasize}} { - set ::datasize $arg - incr j - } elseif {$opt eq {--requests}} { - set ::requests $arg - incr j - } else { - puts "Wrong argument: $opt" - exit 1 - } -} - -main diff --git a/examples/redis-unstable/utils/srandmember/README.md b/examples/redis-unstable/utils/srandmember/README.md deleted file mode 100644 index d3da1e8..0000000 --- a/examples/redis-unstable/utils/srandmember/README.md +++ /dev/null @@ -1,14 +0,0 @@ -The utilities in this directory plot the distribution of SRANDMEMBER to -evaluate how fair it is. - -See http://theshfl.com/redis_sets for more information on the topic that lead -to such investigation fix. - -showdist.rb -- shows the distribution of the frequency elements are returned. - The x axis is the number of times elements were returned, and - the y axis is how many elements were returned with such - frequency. - -showfreq.rb -- shows the frequency each element was returned. - The x axis is the element number. - The y axis is the times it was returned. diff --git a/examples/redis-unstable/utils/srandmember/showdist.rb b/examples/redis-unstable/utils/srandmember/showdist.rb deleted file mode 100644 index 2435857..0000000 --- a/examples/redis-unstable/utils/srandmember/showdist.rb +++ /dev/null @@ -1,33 +0,0 @@ -require 'redis' - -r = Redis.new -r.select(9) -r.del("myset"); -r.sadd("myset",(0..999).to_a) -freq = {} -100.times { - res = r.pipelined { - 1000.times { - r.srandmember("myset") - } - } - res.each{|ele| - freq[ele] = 0 if freq[ele] == nil - freq[ele] += 1 - } -} - -# Convert into frequency distribution -dist = {} -freq.each{|item,count| - dist[count] = 0 if dist[count] == nil - dist[count] += 1 -} - -min = dist.keys.min -max = dist.keys.max -(min..max).each{|x| - count = dist[x] - count = 0 if count == nil - puts "#{x} -> #{"*"*count}" -} diff --git a/examples/redis-unstable/utils/srandmember/showfreq.rb b/examples/redis-unstable/utils/srandmember/showfreq.rb deleted file mode 100644 index 625519c..0000000 --- a/examples/redis-unstable/utils/srandmember/showfreq.rb +++ /dev/null @@ -1,23 +0,0 @@ -require 'redis' - -r = Redis.new -r.select(9) -r.del("myset"); -r.sadd("myset",(0..999).to_a) -freq = {} -500.times { - res = r.pipelined { - 1000.times { - r.srandmember("myset") - } - } - res.each{|ele| - freq[ele] = 0 if freq[ele] == nil - freq[ele] += 1 - } -} - -# Print the frequency each element was yield to process it with gnuplot -freq.each{|item,count| - puts "#{item} #{count}" -} diff --git a/examples/redis-unstable/utils/systemd-redis_multiple_servers@.service b/examples/redis-unstable/utils/systemd-redis_multiple_servers@.service deleted file mode 100644 index 108ccfc..0000000 --- a/examples/redis-unstable/utils/systemd-redis_multiple_servers@.service +++ /dev/null @@ -1,37 +0,0 @@ -# example systemd template service unit file for multiple redis-servers -# -# You can use this file as a blueprint for your actual template service unit -# file, if you intend to run multiple independent redis-server instances in -# parallel using systemd's "template unit files" feature. If you do, you will -# want to choose a better basename for your service unit by renaming this file -# when copying it. -# -# Please take a look at the provided "systemd-redis_server.service" example -# service unit file, too, if you choose to use this approach at managing -# multiple redis-server instances via systemd. - -[Unit] -Description=Redis data structure server - instance %i -Documentation=https://redis.io/documentation -# This template unit assumes your redis-server configuration file(s) -# to live at /etc/redis/redis_server_<INSTANCE_NAME>.conf -AssertPathExists=/etc/redis/redis_server_%i.conf -#Before=your_application.service another_example_application.service -#AssertPathExists=/var/lib/redis - -[Service] -ExecStart=/usr/local/bin/redis-server /etc/redis/redis_server_%i.conf -LimitNOFILE=10032 -NoNewPrivileges=yes -#OOMScoreAdjust=-900 -#PrivateTmp=yes -Type=notify -TimeoutStartSec=infinity -TimeoutStopSec=infinity -UMask=0077 -#User=redis -#Group=redis -#WorkingDirectory=/var/lib/redis - -[Install] -WantedBy=multi-user.target diff --git a/examples/redis-unstable/utils/systemd-redis_server.service b/examples/redis-unstable/utils/systemd-redis_server.service deleted file mode 100644 index cf15864..0000000 --- a/examples/redis-unstable/utils/systemd-redis_server.service +++ /dev/null @@ -1,43 +0,0 @@ -# example systemd service unit file for redis-server -# -# In order to use this as a template for providing a redis service in your -# environment, _at the very least_ make sure to adapt the redis configuration -# file you intend to use as needed (make sure to set "supervised systemd"), and -# to set sane TimeoutStartSec and TimeoutStopSec property values in the unit's -# "[Service]" section to fit your needs. -# -# Some properties, such as User= and Group=, are highly desirable for virtually -# all deployments of redis, but cannot be provided in a manner that fits all -# expectable environments. Some of these properties have been commented out in -# this example service unit file, but you are highly encouraged to set them to -# fit your needs. -# -# Please refer to systemd.unit(5), systemd.service(5), and systemd.exec(5) for -# more information. - -[Unit] -Description=Redis data structure server -Documentation=https://redis.io/documentation -#Before=your_application.service another_example_application.service -#AssertPathExists=/var/lib/redis -Wants=network-online.target -After=network-online.target - -[Service] -ExecStart=/usr/local/bin/redis-server --supervised systemd --daemonize no -## Alternatively, have redis-server load a configuration file: -#ExecStart=/usr/local/bin/redis-server /path/to/your/redis.conf -LimitNOFILE=10032 -NoNewPrivileges=yes -#OOMScoreAdjust=-900 -#PrivateTmp=yes -Type=notify -TimeoutStartSec=infinity -TimeoutStopSec=infinity -UMask=0077 -#User=redis -#Group=redis -#WorkingDirectory=/var/lib/redis - -[Install] -WantedBy=multi-user.target diff --git a/examples/redis-unstable/utils/tracking_collisions.c b/examples/redis-unstable/utils/tracking_collisions.c deleted file mode 100644 index 4273092..0000000 --- a/examples/redis-unstable/utils/tracking_collisions.c +++ /dev/null @@ -1,79 +0,0 @@ -/* This is a small program used in order to understand the collision rate - * of CRC64 (ISO version) VS other stronger hashing functions in the context - * of hashing keys for the Redis "tracking" feature (client side caching - * assisted by the server). - * - * The program attempts to hash keys with common names in the form of - * - * prefix:<counter> - * - * And counts the resulting collisions generated in the 24 bits of output - * needed for the tracking feature invalidation table (16 millions + entries) - * - * Compile with: - * - * cc -O2 ./tracking_collisions.c ../src/crc64.c ../src/sha1.c - * ./a.out - * - * -------------------------------------------------------------------------- - * - * Copyright (C) 2019-Present Redis Ltd. All rights reserved. - * - * Licensed under your choice of (a) the Redis Source Available License 2.0 - * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the - * GNU Affero General Public License v3 (AGPLv3). - */ - -#include <stdlib.h> -#include <stdint.h> -#include <string.h> -#include <stdio.h> -#include "../src/crc64.h" -#include "../src/sha1.h" - -#define TABLE_SIZE (1<<24) -int Table[TABLE_SIZE]; - -uint64_t crc64Hash(char *key, size_t len) { - return crc64(0,(unsigned char*)key,len); -} - -uint64_t sha1Hash(char *key, size_t len) { - SHA1_CTX ctx; - unsigned char hash[20]; - - SHA1Init(&ctx); - SHA1Update(&ctx,(unsigned char*)key,len); - SHA1Final(hash,&ctx); - uint64_t hash64; - memcpy(&hash64,hash,sizeof(hash64)); - return hash64; -} - -/* Test the hashing function provided as callback and return the - * number of collisions found. */ -unsigned long testHashingFunction(uint64_t (*hash)(char *, size_t)) { - unsigned long collisions = 0; - memset(Table,0,sizeof(Table)); - char *prefixes[] = {"object", "message", "user", NULL}; - for (int i = 0; prefixes[i] != NULL; i++) { - for (int j = 0; j < TABLE_SIZE/2; j++) { - char keyname[128]; - size_t keylen = snprintf(keyname,sizeof(keyname),"%s:%d", - prefixes[i],j); - uint64_t bucket = hash(keyname,keylen) % TABLE_SIZE; - if (Table[bucket]) { - collisions++; - } else { - Table[bucket] = 1; - } - } - } - return collisions; -} - -int main(void) { - printf("SHA1 : %lu\n", testHashingFunction(sha1Hash)); - printf("CRC64: %lu\n", testHashingFunction(crc64Hash)); - return 0; -} diff --git a/examples/redis-unstable/utils/whatisdoing.sh b/examples/redis-unstable/utils/whatisdoing.sh deleted file mode 100755 index 09c0b08..0000000 --- a/examples/redis-unstable/utils/whatisdoing.sh +++ /dev/null @@ -1,24 +0,0 @@ -# This script is from http://poormansprofiler.org/ -# -# NOTE: Instead of using this script, you should use the Redis -# Software Watchdog, which provides a similar functionality but in -# a more reliable / easy to use way. -# -# Check https://redis.io/docs/latest/operate/oss_and_stack/management/optimization/latency/ for more information. - -#!/bin/bash -nsamples=1 -sleeptime=0 -pid=$(ps auxww | grep '[r]edis-server' | awk '{print $2}') - -for x in $(seq 1 $nsamples) - do - gdb -ex "set pagination 0" -ex "thread apply all bt" -batch -p $pid - sleep $sleeptime - done | \ -awk ' - BEGIN { s = ""; } - /Thread/ { print s; s = ""; } - /^\#/ { if (s != "" ) { s = s "," $4} else { s = $4 } } - END { print s }' | \ -sort | uniq -c | sort -r -n -k 1,1 |