aboutsummaryrefslogtreecommitdiff
path: root/examples/redis-unstable/utils/req-res-log-validator.py
diff options
context:
space:
mode:
Diffstat (limited to 'examples/redis-unstable/utils/req-res-log-validator.py')
-rwxr-xr-xexamples/redis-unstable/utils/req-res-log-validator.py364
1 files changed, 0 insertions, 364 deletions
diff --git a/examples/redis-unstable/utils/req-res-log-validator.py b/examples/redis-unstable/utils/req-res-log-validator.py
deleted file mode 100755
index 0f00935..0000000
--- a/examples/redis-unstable/utils/req-res-log-validator.py
+++ /dev/null
@@ -1,364 +0,0 @@
1#!/usr/bin/env python3
2import os
3import glob
4import json
5import sys
6
7import jsonschema
8import subprocess
9import redis
10import time
11import argparse
12import multiprocessing
13import collections
14import io
15import traceback
16from datetime import timedelta
17from functools import partial
18try:
19 from jsonschema import Draft201909Validator as schema_validator
20except ImportError:
21 from jsonschema import Draft7Validator as schema_validator
22
23"""
24The purpose of this file is to validate the reply_schema values of COMMAND DOCS.
25Basically, this is what it does:
261. Goes over req-res files, generated by redis-servers, spawned by the testsuite (see logreqres.c)
272. For each request-response pair, it validates the response against the request's reply_schema (obtained from COMMAND DOCS)
28
29This script spins up a redis-server and a redis-cli in order to obtain COMMAND DOCS.
30
31In order to use this file you must run the redis testsuite with the following flags:
32./runtest --dont-clean --force-resp3 --log-req-res
33
34And then:
35./utils/req-res-log-validator.py
36
37The script will fail only if:
381. One or more of the replies doesn't comply with its schema.
392. One or more of the commands in COMMANDS DOCS doesn't have the reply_schema field (with --fail-missing-reply-schemas)
403. The testsuite didn't execute all of the commands (with --fail-commands-not-all-hit)
41
42Future validations:
431. Fail the script if one or more of the branches of the reply schema (e.g. oneOf, anyOf) was not hit.
44"""
45
46IGNORED_COMMANDS = {
47 # Commands that don't work in a req-res manner (see logreqres.c)
48 "debug", # because of DEBUG SEGFAULT
49 "sync",
50 "psync",
51 "monitor",
52 "subscribe",
53 "unsubscribe",
54 "ssubscribe",
55 "sunsubscribe",
56 "psubscribe",
57 "punsubscribe",
58 # Commands to which we decided not write a reply schema
59 "pfdebug",
60 "lolwut",
61 # TODO: for vector-sets module
62 "VADD",
63 "VCARD",
64 "VDIM",
65 "VEMB",
66 "VGETATTR",
67 "VINFO",
68 "VLINKS",
69 "VRANDMEMBER",
70 "VREM",
71 "VSETATTR",
72 "VSIM",
73 "VISMEMBER",
74 "VRANGE",
75}
76
77class Request(object):
78 """
79 This class represents a Redis request (AKA command, argv)
80 """
81 def __init__(self, f, docs, line_counter):
82 """
83 Read lines from `f` (generated by logreqres.c) and populates the argv array
84 """
85 self.command = None
86 self.schema = None
87 self.argv = []
88
89 while True:
90 line = f.readline()
91 line_counter[0] += 1
92 if not line:
93 break
94 length = int(line)
95 arg = str(f.read(length))
96 f.read(2) # read \r\n
97 line_counter[0] += 1
98 if arg == "__argv_end__":
99 break
100 self.argv.append(arg)
101
102 if not self.argv:
103 return
104
105 self.command = self.argv[0].lower()
106 doc = docs.get(self.command, {})
107 if not doc and len(self.argv) > 1:
108 self.command = f"{self.argv[0].lower()}|{self.argv[1].lower()}"
109 doc = docs.get(self.command, {})
110
111 if not doc:
112 self.command = None
113 return
114
115 self.schema = doc.get("reply_schema")
116
117 def __str__(self):
118 return json.dumps(self.argv)
119
120
121class Response(object):
122 """
123 This class represents a Redis response in RESP3
124 """
125 def __init__(self, f, line_counter):
126 """
127 Read lines from `f` (generated by logreqres.c) and build the JSON representing the response in RESP3
128 """
129 self.error = False
130 self.queued = False
131 self.json = None
132
133 line = f.readline()[:-2]
134 line_counter[0] += 1
135 if line[0] == '+':
136 self.json = line[1:]
137 if self.json == "QUEUED":
138 self.queued = True
139 elif line[0] == '-':
140 self.json = line[1:]
141 self.error = True
142 elif line[0] == '$':
143 self.json = str(f.read(int(line[1:])))
144 f.read(2) # read \r\n
145 line_counter[0] += 1
146 elif line[0] == ':':
147 self.json = int(line[1:])
148 elif line[0] == ',':
149 self.json = float(line[1:])
150 elif line[0] == '_':
151 self.json = None
152 elif line[0] == '#':
153 self.json = line[1] == 't'
154 elif line[0] == '!':
155 self.json = str(f.read(int(line[1:])))
156 f.read(2) # read \r\n
157 line_counter[0] += 1
158 self.error = True
159 elif line[0] == '=':
160 self.json = str(f.read(int(line[1:])))[4:] # skip "txt:" or "mkd:"
161 f.read(2) # read \r\n
162 line_counter[0] += 1 + self.json.count("\r\n")
163 elif line[0] == '(':
164 self.json = line[1:] # big-number is actually a string
165 elif line[0] in ['*', '~', '>']: # unfortunately JSON doesn't tell the difference between a list and a set
166 self.json = []
167 count = int(line[1:])
168 for i in range(count):
169 ele = Response(f, line_counter)
170 self.json.append(ele.json)
171 elif line[0] in ['%', '|']:
172 self.json = {}
173 count = int(line[1:])
174 for i in range(count):
175 field = Response(f, line_counter)
176 # Redis allows fields to be non-strings but JSON doesn't.
177 # Luckily, for any kind of response we can validate, the fields are
178 # always strings (example: XINFO STREAM)
179 # The reason we can't always convert to string is because of DEBUG PROTOCOL MAP
180 # which anyway doesn't have a schema
181 if isinstance(field.json, str):
182 field = field.json
183 value = Response(f, line_counter)
184 self.json[field] = value.json
185 if line[0] == '|':
186 # We don't care about the attributes, read the real response
187 real_res = Response(f, line_counter)
188 self.__dict__.update(real_res.__dict__)
189
190
191 def __str__(self):
192 return json.dumps(self.json)
193
194
195def process_file(docs, path):
196 """
197 This function processes a single file generated by logreqres.c
198 """
199 line_counter = [0] # A list with one integer: to force python to pass it by reference
200 command_counter = dict()
201
202 print(f"Processing {path} ...")
203
204 # Convert file to StringIO in order to minimize IO operations
205 with open(path, "r", newline="\r\n", encoding="latin-1") as f:
206 content = f.read()
207
208 with io.StringIO(content) as fakefile:
209 while True:
210 try:
211 req = Request(fakefile, docs, line_counter)
212 if not req.argv:
213 # EOF
214 break
215 res = Response(fakefile, line_counter)
216 except json.decoder.JSONDecodeError as err:
217 print(f"JSON decoder error while processing {path}:{line_counter[0]}: {err}")
218 print(traceback.format_exc())
219 raise
220 except Exception as err:
221 print(f"General error while processing {path}:{line_counter[0]}: {err}")
222 print(traceback.format_exc())
223 raise
224
225 if not req.command:
226 # Unknown command
227 continue
228
229 command_counter[req.command] = command_counter.get(req.command, 0) + 1
230
231 if res.error or res.queued:
232 continue
233
234 if req.command in IGNORED_COMMANDS:
235 continue
236
237 try:
238 jsonschema.validate(instance=res.json, schema=req.schema, cls=schema_validator)
239 except (jsonschema.ValidationError, jsonschema.exceptions.SchemaError) as err:
240 print(f"JSON schema validation error on {path}: {err}")
241 print(f"argv: {req.argv}")
242 try:
243 print(f"Response: {res}")
244 except UnicodeDecodeError as err:
245 print("Response: (unprintable)")
246 print(f"Schema: {json.dumps(req.schema, indent=2)}")
247 print(traceback.format_exc())
248 raise
249
250 return command_counter
251
252
253def fetch_schemas(cli, port, args, docs):
254 redis_proc = subprocess.Popen(args, stdout=subprocess.PIPE)
255
256 while True:
257 try:
258 print('Connecting to Redis...')
259 r = redis.Redis(port=port)
260 r.ping()
261 break
262 except Exception as e:
263 time.sleep(0.1)
264
265 print('Connected')
266
267 cli_proc = subprocess.Popen([cli, '-p', str(port), '--json', 'command', 'docs'], stdout=subprocess.PIPE)
268 stdout, stderr = cli_proc.communicate()
269 docs_response = json.loads(stdout)
270
271 for name, doc in docs_response.items():
272 if "subcommands" in doc:
273 for subname, subdoc in doc["subcommands"].items():
274 docs[subname] = subdoc
275 else:
276 docs[name] = doc
277
278 redis_proc.terminate()
279 redis_proc.wait()
280
281
282if __name__ == '__main__':
283 # Figure out where the sources are
284 srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src")
285 testdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../tests")
286
287 parser = argparse.ArgumentParser()
288 parser.add_argument('--server', type=str, default='%s/redis-server' % srcdir)
289 parser.add_argument('--port', type=int, default=6534)
290 parser.add_argument('--cli', type=str, default='%s/redis-cli' % srcdir)
291 parser.add_argument('--module', type=str, action='append', default=[])
292 parser.add_argument('--verbose', action='store_true')
293 parser.add_argument('--fail-commands-not-all-hit', action='store_true')
294 parser.add_argument('--fail-missing-reply-schemas', action='store_true')
295 args = parser.parse_args()
296
297 docs = dict()
298
299 # Fetch schemas from a Redis instance
300 print('Starting Redis server')
301 redis_args = [args.server, '--port', str(args.port)]
302 for module in args.module:
303 redis_args += ['--loadmodule', 'tests/modules/%s.so' % module]
304
305 fetch_schemas(args.cli, args.port, redis_args, docs)
306
307 # Fetch schemas from a sentinel
308 print('Starting Redis sentinel')
309
310 # Sentinel needs a config file to start
311 config_file = "tmpsentinel.conf"
312 open(config_file, 'a').close()
313
314 sentinel_args = [args.server, config_file, '--port', str(args.port), "--sentinel"]
315 fetch_schemas(args.cli, args.port, sentinel_args, docs)
316 os.unlink(config_file)
317
318 missing_schema = [k for k, v in docs.items()
319 if "reply_schema" not in v and k not in IGNORED_COMMANDS]
320 if missing_schema:
321 print("WARNING! The following commands are missing a reply_schema:")
322 for k in sorted(missing_schema):
323 print(f" {k}")
324 if args.fail_missing_reply_schemas:
325 print("ERROR! at least one command does not have a reply_schema")
326 sys.exit(1)
327
328 start = time.time()
329
330 # Obtain all the files to processes
331 paths = []
332 for path in glob.glob('%s/tmp/*/*.reqres' % testdir):
333 paths.append(path)
334
335 for path in glob.glob('%s/cluster/tmp/*/*.reqres' % testdir):
336 paths.append(path)
337
338 for path in glob.glob('%s/sentinel/tmp/*/*.reqres' % testdir):
339 paths.append(path)
340
341 counter = collections.Counter()
342 # Spin several processes to handle the files in parallel
343 with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:
344 func = partial(process_file, docs)
345 # pool.map blocks until all the files have been processed
346 for result in pool.map(func, paths):
347 counter.update(result)
348 command_counter = dict(counter)
349
350 elapsed = time.time() - start
351 print(f"Done. ({timedelta(seconds=elapsed)})")
352 print("Hits per command:")
353 for k, v in sorted(command_counter.items()):
354 print(f" {k}: {v}")
355 not_hit = set(set(docs.keys()) - set(command_counter.keys()) - set(IGNORED_COMMANDS))
356 if not_hit:
357 if args.verbose:
358 print("WARNING! The following commands were not hit at all:")
359 for k in sorted(not_hit):
360 print(f" {k}")
361 if args.fail_commands_not_all_hit:
362 print("ERROR! at least one command was not hit by the tests")
363 sys.exit(1)
364
generated by cgit v1.2.3 (git 2.46.0) at 2026-04-30 12:20:27 +0000