luna - curl/tests/http/testenv/curl.py

   1#!/usr/bin/env python3
   2# -*- coding: utf-8 -*-
   3#***************************************************************************
   4#                                  _   _ ____  _
   5#  Project                     ___| | | |  _ \| |
   6#                             / __| | | | |_) | |
   7#                            | (__| |_| |  _ <| |___
   8#                             \___|\___/|_| \_\_____|
   9#
  10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
  11#
  12# This software is licensed as described in the file COPYING, which
  13# you should have received as part of this distribution. The terms
  14# are also available at https://curl.se/docs/copyright.html.
  15#
  16# You may opt to use, copy, modify, merge, publish, distribute and/or sell
  17# copies of the Software, and permit persons to whom the Software is
  18# furnished to do so, under the terms of the COPYING file.
  19#
  20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  21# KIND, either express or implied.
  22#
  23# SPDX-License-Identifier: curl
  24#
  25###########################################################################
  26#
  27import json
  28import logging
  29import os
  30import re
  31import shutil
  32import subprocess
  33import sys
  34import time
  35from datetime import datetime, timedelta, timezone
  36from functools import cmp_to_key
  37from statistics import fmean, mean
  38from threading import Thread
  39from typing import Any, Dict, List, Optional, Union
  40from urllib.parse import urlparse
  41
  42import psutil
  43
  44from .env import Env
  45
  46log = logging.getLogger(__name__)
  47
  48
  49class RunProfile:
  50
  51    STAT_KEYS = ['cpu', 'rss', 'vsz']
  52
  53    @classmethod
  54    def AverageStats(cls, profiles: List['RunProfile']):
  55        avg = {}
  56        stats = [p.stats for p in profiles]
  57        for key in cls.STAT_KEYS:
  58            vals = [s[key] for s in stats]
  59            avg[key] = mean(vals) if len(vals) else 0.0
  60        return avg
  61
  62    def __init__(self, pid: int, started_at: datetime, run_dir):
  63        self._pid = pid
  64        self._started_at = started_at
  65        self._duration = timedelta(seconds=0)
  66        self._run_dir = run_dir
  67        self._samples = []
  68        self._psu = None
  69        self._stats = None
  70
  71    @property
  72    def duration(self) -> timedelta:
  73        return self._duration
  74
  75    @property
  76    def stats(self) -> Optional[Dict[str,Any]]:
  77        return self._stats
  78
  79    def sample(self):
  80        elapsed = datetime.now() - self._started_at
  81        try:
  82            if self._psu is None:
  83                self._psu = psutil.Process(pid=self._pid)
  84            mem = self._psu.memory_info()
  85            self._samples.append({
  86                'time': elapsed,
  87                'cpu': self._psu.cpu_percent(),
  88                'vsz': mem.vms,
  89                'rss': mem.rss,
  90            })
  91        except psutil.NoSuchProcess:
  92            pass
  93
  94    def finish(self):
  95        self._duration = datetime.now() - self._started_at
  96        if len(self._samples) > 0:
  97            weights = [s['time'].total_seconds() for s in self._samples]
  98            self._stats = {}
  99            for key in self.STAT_KEYS:
 100                self._stats[key] = fmean([s[key] for s in self._samples], weights)
 101        else:
 102            self._stats = None
 103        self._psu = None
 104
 105    def __repr__(self):
 106        return f'RunProfile[pid={self._pid}, '\
 107               f'duration={self.duration.total_seconds():.3f}s, '\
 108               f'stats={self.stats}]'
 109
 110
 111class PerfProfile:
 112
 113    def __init__(self, pid: int, run_dir):
 114        self._pid = pid
 115        self._run_dir = run_dir
 116        self._proc = None
 117        self._rc = 0
 118        self._file = os.path.join(self._run_dir, 'curl.perf_stacks')
 119
 120    def start(self):
 121        if os.path.exists(self._file):
 122            os.remove(self._file)
 123        args = [
 124            'sudo', 'perf', 'record', '-F', '99', '-p', f'{self._pid}',
 125            '-g', '--', 'sleep', '60'
 126        ]
 127        self._proc = subprocess.Popen(args, text=True, cwd=self._run_dir, shell=False)
 128        assert self._proc
 129
 130    def finish(self):
 131        if self._proc:
 132            self._proc.terminate()
 133            self._rc = self._proc.returncode
 134        with open(self._file, 'w') as cout:
 135            p = subprocess.run([
 136                'sudo', 'perf', 'script'
 137            ], stdout=cout, cwd=self._run_dir, shell=False)
 138            rc = p.returncode
 139            if rc != 0:
 140                raise Exception(f'perf returned error {rc}')
 141
 142    @property
 143    def file(self):
 144        return self._file
 145
 146
 147class DTraceProfile:
 148
 149    def __init__(self, pid: int, run_dir):
 150        self._pid = pid
 151        self._run_dir = run_dir
 152        self._proc = None
 153        self._rc = 0
 154        self._file = os.path.join(self._run_dir, 'curl.dtrace_stacks')
 155
 156    def start(self):
 157        if os.path.exists(self._file):
 158            os.remove(self._file)
 159        args = [
 160            'sudo', 'dtrace',
 161            '-x', 'ustackframes=100',
 162            '-n', f'profile-97 /pid == {self._pid}/ {{ @[ustack()] = count(); }} tick-60s {{ exit(0); }}',
 163            '-o', f'{self._file}'
 164        ]
 165        if sys.platform.startswith('darwin'):
 166            # macOS seems to like this for producing symbols in user stacks
 167            args.extend(['-p', f'{self._pid}'])
 168        self._proc = subprocess.Popen(args, text=True, cwd=self._run_dir, shell=False)
 169        assert self._proc
 170
 171    def finish(self):
 172        if self._proc:
 173            self._proc.terminate()
 174            self._rc = self._proc.returncode
 175
 176    @property
 177    def file(self):
 178        return self._file
 179
 180
 181class RunTcpDump:
 182
 183    def __init__(self, env, run_dir):
 184        self._env = env
 185        self._run_dir = run_dir
 186        self._proc = None
 187        self._stdoutfile = os.path.join(self._run_dir, 'tcpdump.out')
 188        self._stderrfile = os.path.join(self._run_dir, 'tcpdump.err')
 189
 190    def get_rsts(self, ports: List[int]|None = None) -> Optional[List[str]]:
 191        if self._proc:
 192            raise Exception('tcpdump still running')
 193        lines = []
 194        for line in open(self._stdoutfile):
 195            m = re.match(r'.* IP 127\.0\.0\.1\.(\d+) [<>] 127\.0\.0\.1\.(\d+):.*', line)
 196            if m:
 197                sport = int(m.group(1))
 198                dport = int(m.group(2))
 199                if ports is None or sport in ports or dport in ports:
 200                    lines.append(line)
 201        return lines
 202
 203    @property
 204    def stats(self) -> Optional[List[str]]:
 205        return self.get_rsts()
 206
 207    @property
 208    def stderr(self) -> List[str]:
 209        if self._proc:
 210            raise Exception('tcpdump still running')
 211        return open(self._stderrfile).readlines()
 212
 213    def sample(self):
 214        # not sure how to make that detection reliable for all platforms
 215        local_if = 'lo0' if sys.platform.startswith('darwin') else 'lo'
 216        try:
 217            tcpdump = self._env.tcpdump()
 218            if tcpdump is None:
 219                raise Exception('tcpdump not available')
 220            # look with tcpdump for TCP RST packets which indicate
 221            # we did not shut down connections cleanly
 222            args = []
 223            # at least on Linux, we need root permissions to run tcpdump
 224            if sys.platform.startswith('linux'):
 225                args.append('sudo')
 226            args.extend([
 227                tcpdump, '-i', local_if, '-n', 'tcp[tcpflags] & (tcp-rst)!=0'
 228            ])
 229            with open(self._stdoutfile, 'w') as cout, open(self._stderrfile, 'w') as cerr:
 230                self._proc = subprocess.Popen(args, stdout=cout, stderr=cerr,
 231                                              text=True, cwd=self._run_dir,
 232                                              shell=False)
 233                assert self._proc
 234                assert self._proc.returncode is None
 235                while self._proc:
 236                    try:
 237                        self._proc.wait(timeout=1)
 238                    except subprocess.TimeoutExpired:
 239                        pass
 240        except Exception:
 241            log.exception('Tcpdump')
 242
 243    def start(self):
 244        def do_sample():
 245            self.sample()
 246        t = Thread(target=do_sample)
 247        t.start()
 248
 249    def finish(self):
 250        if self._proc:
 251            time.sleep(1)
 252            self._proc.terminate()
 253            self._proc = None
 254
 255
 256class ExecResult:
 257
 258    def __init__(self, args: List[str], exit_code: int,
 259                 stdout: List[str], stderr: List[str],
 260                 duration: Optional[timedelta] = None,
 261                 with_stats: bool = False,
 262                 exception: Optional[str] = None,
 263                 profile: Optional[RunProfile] = None,
 264                 tcpdump: Optional[RunTcpDump] = None):
 265        self._args = args
 266        self._exit_code = exit_code
 267        self._exception = exception
 268        self._stdout = stdout
 269        self._stderr = stderr
 270        self._profile = profile
 271        self._tcpdump = tcpdump
 272        self._duration = duration if duration is not None else timedelta()
 273        self._response = None
 274        self._responses = []
 275        self._results = {}
 276        self._assets = []
 277        self._stats = []
 278        self._json_out = None
 279        self._with_stats = with_stats
 280        if with_stats:
 281            self._parse_stats()
 282        else:
 283            # noinspection PyBroadException
 284            try:
 285                out = ''.join(self._stdout)
 286                self._json_out = json.loads(out)
 287            except:  # noqa: E722
 288                pass
 289
 290    def __repr__(self):
 291        return f"ExecResult[code={self.exit_code}, exception={self._exception}, "\
 292               f"args={self._args}, stdout={self._stdout}, stderr={self._stderr}]"
 293
 294    def _parse_stats(self):
 295        self._stats = []
 296        for line in self._stdout:
 297            try:
 298                self._stats.append(json.loads(line))
 299            # TODO: specify specific exceptions here
 300            except:  # noqa: E722
 301                log.exception(f'not a JSON stat: {line}')
 302                break
 303
 304    @property
 305    def exit_code(self) -> int:
 306        return self._exit_code
 307
 308    @property
 309    def args(self) -> List[str]:
 310        return self._args
 311
 312    @property
 313    def outraw(self) -> bytes:
 314        return ''.join(self._stdout).encode()
 315
 316    @property
 317    def stdout(self) -> str:
 318        return ''.join(self._stdout)
 319
 320    @property
 321    def json(self) -> Optional[Dict]:
 322        """Output as JSON dictionary or None if not parseable."""
 323        return self._json_out
 324
 325    @property
 326    def stderr(self) -> str:
 327        return ''.join(self._stderr)
 328
 329    @property
 330    def trace_lines(self) -> List[str]:
 331        return self._stderr
 332
 333    @property
 334    def duration(self) -> timedelta:
 335        return self._duration
 336
 337    @property
 338    def profile(self) -> Optional[RunProfile]:
 339        return self._profile
 340
 341    @property
 342    def tcpdump(self) -> Optional[RunTcpDump]:
 343        return self._tcpdump
 344
 345    @property
 346    def response(self) -> Optional[Dict]:
 347        return self._response
 348
 349    @property
 350    def responses(self) -> List[Dict]:
 351        return self._responses
 352
 353    @property
 354    def results(self) -> Dict:
 355        return self._results
 356
 357    @property
 358    def assets(self) -> List:
 359        return self._assets
 360
 361    @property
 362    def with_stats(self) -> bool:
 363        return self._with_stats
 364
 365    @property
 366    def stats(self) -> List:
 367        return self._stats
 368
 369    @property
 370    def total_connects(self) -> Optional[int]:
 371        if len(self.stats):
 372            n = 0
 373            for stat in self.stats:
 374                n += stat['num_connects']
 375            return n
 376        return None
 377
 378    def add_response(self, resp: Dict):
 379        self._response = resp
 380        self._responses.append(resp)
 381
 382    def add_results(self, results: Dict):
 383        self._results.update(results)
 384        if 'response' in results:
 385            self.add_response(results['response'])
 386
 387    def add_assets(self, assets: List):
 388        self._assets.extend(assets)
 389
 390    def check_exit_code(self, code: Union[int, bool]):
 391        if code is True:
 392            assert self.exit_code == 0, f'expected exit code {code}, '\
 393                                        f'got {self.exit_code}\n{self.dump_logs()}'
 394        elif code is False:
 395            assert self.exit_code != 0, f'expected exit code {code}, '\
 396                                                f'got {self.exit_code}\n{self.dump_logs()}'
 397        else:
 398            assert self.exit_code == code, f'expected exit code {code}, '\
 399                                           f'got {self.exit_code}\n{self.dump_logs()}'
 400
 401    def check_response(self, http_status: Optional[int] = 200,
 402                       count: Optional[int] = 1,
 403                       protocol: Optional[str] = None,
 404                       exitcode: Optional[int] = 0,
 405                       connect_count: Optional[int] = None):
 406        if exitcode:
 407            self.check_exit_code(exitcode)
 408            if self.with_stats and isinstance(exitcode, int):
 409                for idx, x in enumerate(self.stats):
 410                    if 'exitcode' in x:
 411                        assert int(x['exitcode']) == exitcode, \
 412                            f'response #{idx} exitcode: expected {exitcode}, '\
 413                            f'got {x["exitcode"]}\n{self.dump_logs()}'
 414
 415        if self.with_stats:
 416            assert len(self.stats) == count, \
 417                f'response count: expected {count}, ' \
 418                f'got {len(self.stats)}\n{self.dump_logs()}'
 419        else:
 420            assert len(self.responses) == count, \
 421                f'response count: expected {count}, ' \
 422                f'got {len(self.responses)}\n{self.dump_logs()}'
 423        if http_status is not None:
 424            if self.with_stats:
 425                for idx, x in enumerate(self.stats):
 426                    assert 'http_code' in x, \
 427                        f'response #{idx} reports no http_code\n{self.dump_stat(x)}'
 428                    assert x['http_code'] == http_status, \
 429                        f'response #{idx} http_code: expected {http_status}, '\
 430                        f'got {x["http_code"]}\n{self.dump_stat(x)}'
 431            else:
 432                for idx, x in enumerate(self.responses):
 433                    assert x['status'] == http_status, \
 434                        f'response #{idx} status: expected {http_status},'\
 435                        f'got {x["status"]}\n{self.dump_stat(x)}'
 436        if protocol is not None:
 437            if self.with_stats:
 438                http_version = None
 439                if protocol == 'HTTP/1.1':
 440                    http_version = '1.1'
 441                elif protocol == 'HTTP/2':
 442                    http_version = '2'
 443                elif protocol == 'HTTP/3':
 444                    http_version = '3'
 445                if http_version is not None:
 446                    for idx, x in enumerate(self.stats):
 447                        assert x['http_version'] == http_version, \
 448                            f'response #{idx} protocol: expected http/{http_version},' \
 449                            f'got version {x["http_version"]}\n{self.dump_stat(x)}'
 450            else:
 451                for idx, x in enumerate(self.responses):
 452                    assert x['protocol'] == protocol, \
 453                        f'response #{idx} protocol: expected {protocol},'\
 454                        f'got {x["protocol"]}\n{self.dump_logs()}'
 455        if connect_count is not None:
 456            assert self.total_connects == connect_count, \
 457                f'expected {connect_count}, but {self.total_connects} '\
 458                f'were made\n{self.dump_logs()}'
 459
 460    def check_stats(self, count: int, http_status: Optional[int] = None,
 461                    exitcode: Optional[Union[int, List[int]]] = None,
 462                    remote_port: Optional[int] = None,
 463                    remote_ip: Optional[str] = None):
 464        if exitcode is None:
 465            self.check_exit_code(0)
 466        elif isinstance(exitcode, int):
 467            exitcode = [exitcode]
 468        assert len(self.stats) == count, \
 469            f'stats count: expected {count}, got {len(self.stats)}\n{self.dump_logs()}'
 470        if http_status is not None:
 471            for idx, x in enumerate(self.stats):
 472                assert 'http_code' in x, \
 473                    f'status #{idx} reports no http_code\n{self.dump_stat(x)}'
 474                assert x['http_code'] == http_status, \
 475                    f'status #{idx} http_code: expected {http_status}, '\
 476                    f'got {x["http_code"]}\n{self.dump_stat(x)}'
 477        if exitcode is not None:
 478            for idx, x in enumerate(self.stats):
 479                if 'exitcode' in x:
 480                    assert x['exitcode'] in exitcode, \
 481                        f'status #{idx} exitcode: expected {exitcode}, '\
 482                        f'got {x["exitcode"]}\n{self.dump_stat(x)}'
 483        if remote_port is not None:
 484            for idx, x in enumerate(self.stats):
 485                assert 'remote_port' in x, f'remote_port missing\n{self.dump_stat(x)}'
 486                assert x['remote_port'] == remote_port, \
 487                        f'status #{idx} remote_port: expected {remote_port}, '\
 488                        f'got {x["remote_port"]}\n{self.dump_stat(x)}'
 489        if remote_ip is not None:
 490            for idx, x in enumerate(self.stats):
 491                assert 'remote_ip' in x, f'remote_ip missing\n{self.dump_stat(x)}'
 492                assert x['remote_ip'] == remote_ip, \
 493                        f'status #{idx} remote_ip: expected {remote_ip}, '\
 494                        f'got {x["remote_ip"]}\n{self.dump_stat(x)}'
 495
 496    def check_stat_positive(self, s, idx, key):
 497        assert key in s, f'stat #{idx} "{key}" missing: {s}'
 498        assert s[key] > 0, f'stat #{idx} "{key}" not positive: {s}'
 499
 500    def check_stat_positive_or_0(self, s, idx, key):
 501        assert key in s, f'stat #{idx} "{key}" missing: {s}'
 502        assert s[key] >= 0, f'stat #{idx} "{key}" not positive: {s}'
 503
 504    def check_stat_zero(self, s, key):
 505        assert key in s, f'stat "{key}" missing: {s}'
 506        assert s[key] == 0, f'stat "{key}" not zero: {s}'
 507
 508    def check_stats_timelines(self):
 509        for i in range(len(self._stats)):
 510            self.check_stats_timeline(i)
 511
 512    def check_stats_timeline(self, idx):
 513        # check timings reported on a transfer for consistency
 514        s = self._stats[idx]
 515
 516        url = s['url_effective']
 517
 518        self.check_stat_positive_or_0(s, idx, 'time_connect')
 519        # all stat keys which reporting timings
 520        all_keys = {
 521            'time_queue', 'time_namelookup',
 522            'time_connect', 'time_appconnect',
 523            'time_pretransfer', 'time_posttransfer',
 524            'time_starttransfer', 'time_total',
 525        }
 526        # stat keys where we expect a positive value
 527        ref_tl = []
 528        # time_queue has its own start timestamp. Other timers start *after*
 529        # queueing is done. queue duration might therefore be anywhere.
 530        somewhere_keys = ['time_queue']
 531        exact_match = True
 532        # redirects mess up the times, some are accumulative
 533        if s['time_redirect']:
 534            exact_match = False
 535        # connect events?
 536        if url.startswith('ftp'):
 537            # ftp is messy with connect events for its DATA connection
 538            exact_match = False
 539        elif s['num_connects'] > 0:
 540            ref_tl += ['time_namelookup', 'time_connect']
 541            if url.startswith('https:'):
 542                ref_tl += ['time_appconnect']
 543        # what kind of transfer was it?
 544        if s['size_upload'] == 0 and s['size_download'] > 0:
 545            # this is a download
 546            dl_tl = ['time_pretransfer']
 547            if s['size_request'] > 0:
 548                dl_tl = ['time_posttransfer'] + dl_tl
 549            ref_tl += dl_tl
 550            # the first byte of the response may arrive before we
 551            # track the other times when the client is slow (CI).
 552            somewhere_keys.extend(['time_starttransfer'])
 553        elif s['size_upload'] > 0 and s['size_download'] == 0:
 554            # this is an upload
 555            ul_tl = ['time_pretransfer', 'time_posttransfer']
 556            ref_tl += ul_tl
 557        else:
 558            # could be a 0-length upload or 0-length download, not sure
 559            exact_match = False
 560        # always there at the end
 561        ref_tl += ['time_total']
 562
 563        # assert all events in reference timeline are > 0
 564        for key in ref_tl:
 565            self.check_stat_positive(s, idx, key)
 566        if exact_match:
 567            # assert all events not in reference timeline are 0
 568            for key in [key for key in all_keys if key not in ref_tl and key not in somewhere_keys]:
 569                self.check_stat_zero(s, key)
 570
 571        # calculate the timeline that did happen
 572        def cmp_ts(t1, t2):
 573            n = s[t1] - s[t2]
 574            if not n:  # same timestamp, order to expected occurrence
 575                return ref_tl.index(t1) - ref_tl.index(t2)
 576            return n
 577
 578        seen_tl = sorted(ref_tl, key=cmp_to_key(cmp_ts))
 579        assert seen_tl == ref_tl, f'timeline {idx}: {[f"{ts}: {s[ts]}" for ts in seen_tl]}\n{self.dump_logs()}'
 580        for key in somewhere_keys:
 581            self.check_stat_positive(s, idx, key)
 582            assert s[key] <= s['time_total']
 583
 584    def dump_logs(self):
 585        lines = ['>>--stdout ----------------------------------------------\n']
 586        lines.extend(self._stdout)
 587        lines.append('>>--stderr ----------------------------------------------\n')
 588        lines.extend(self._stderr)
 589        lines.append('<<-------------------------------------------------------\n')
 590        return ''.join(lines)
 591
 592    def dump_stat(self, x):
 593        lines = [
 594            'json stat from curl:',
 595            json.JSONEncoder(indent=2).encode(x),
 596        ]
 597        if 'xfer_id' in x:
 598            xfer_id = x['xfer_id']
 599            lines.append(f'>>--xfer {xfer_id} trace:\n')
 600            lines.extend(self.xfer_trace_for(xfer_id))
 601        else:
 602            lines.append('>>--full trace-------------------------------------------\n')
 603            lines.extend(self._stderr)
 604            lines.append('<<-------------------------------------------------------\n')
 605        return ''.join(lines)
 606
 607    def xfer_trace_for(self, xfer_id) -> List[str]:
 608            pat = re.compile(f'^[^[]* \\[{xfer_id}-.*$')
 609            return [line for line in self._stderr if pat.match(line)]
 610
 611
 612class CurlClient:
 613
 614    ALPN_ARG = {
 615        'http/0.9': '--http0.9',
 616        'http/1.0': '--http1.0',
 617        'http/1.1': '--http1.1',
 618        'h2': '--http2',
 619        'h2c': '--http2',
 620        'h3': '--http3-only',
 621    }
 622
 623    def __init__(self, env: Env,
 624                 run_dir: Optional[str] = None,
 625                 timeout: Optional[float] = None,
 626                 silent: bool = False,
 627                 run_env: Optional[Dict[str, str]] = None,
 628                 server_addr: Optional[str] = None,
 629                 with_dtrace: bool = False,
 630                 with_perf: bool = False,
 631                 with_flame: bool = False,
 632                 force_resolv: bool = True,
 633                 socks_args: Optional[List[str]] = None):
 634        self.env = env
 635        self._timeout = timeout if timeout else env.test_timeout
 636        self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl
 637        self._run_dir = run_dir if run_dir else os.path.join(env.gen_dir, 'curl')
 638        self._stdoutfile = f'{self._run_dir}/curl.stdout'
 639        self._stderrfile = f'{self._run_dir}/curl.stderr'
 640        self._headerfile = f'{self._run_dir}/curl.headers'
 641        self._log_path = f'{self._run_dir}/curl.log'
 642        self._with_dtrace = with_dtrace
 643        self._with_perf = with_perf
 644        self._with_flame = with_flame
 645        self._fg_dir = None
 646        if self._with_flame:
 647            self._fg_dir = os.path.join(self.env.project_dir, '../FlameGraph')
 648            if 'FLAMEGRAPH' in os.environ:
 649                self._fg_dir = os.environ['FLAMEGRAPH']
 650            if not os.path.exists(self._fg_dir):
 651                raise Exception(f'FlameGraph checkout not found in {self._fg_dir}, set env variable FLAMEGRAPH')
 652            if sys.platform.startswith('linux'):
 653                self._with_perf = True
 654            elif sys.platform.startswith('darwin'):
 655                self._with_dtrace = True
 656            else:
 657                raise Exception(f'flame graphs unsupported on {sys.platform}')
 658        self._socks_args = socks_args
 659        self._silent = silent
 660        self._run_env = run_env
 661        self._server_addr = server_addr if server_addr else '127.0.0.1'
 662        self._force_resolv = force_resolv
 663        self._rmrf(self._run_dir)
 664        self._mkpath(self._run_dir)
 665
 666    @property
 667    def run_dir(self) -> str:
 668        return self._run_dir
 669
 670    def download_file(self, i: int) -> str:
 671        return os.path.join(self.run_dir, f'download_{i}.data')
 672
 673    def _rmf(self, path):
 674        if os.path.exists(path):
 675            os.remove(path)
 676
 677    def _rmrf(self, path):
 678        if os.path.exists(path):
 679            shutil.rmtree(path)
 680
 681    def _mkpath(self, path):
 682        if not os.path.exists(path):
 683            os.makedirs(path)
 684
 685    def get_proxy_args(self, proto: str = 'http/1.1',
 686                       proxys: bool = True, tunnel: bool = False,
 687                       use_ip: bool = False, use_ipv6: bool = False):
 688        proxy_name = '[::1]' if use_ipv6 else \
 689            self._server_addr if use_ip else self.env.proxy_domain
 690        if proxys:
 691            pport = self.env.pts_port(proto) if tunnel else self.env.proxys_port
 692            xargs = [
 693                '--proxy', f'https://{proxy_name}:{pport}/',
 694                '--proxy-cacert', self.env.ca.cert_file,
 695            ]
 696            if self._force_resolv and not use_ip and not use_ipv6:
 697                xargs.extend(['--resolve', f'{proxy_name}:{pport}:{self._server_addr}'])
 698            if proto == 'h2':
 699                xargs.append('--proxy-http2')
 700        else:
 701            xargs = [
 702                '--proxy', f'http://{proxy_name}:{self.env.proxy_port}/',
 703            ]
 704            if self._force_resolv and not use_ip and not use_ipv6:
 705                xargs.extend(['--resolve', f'{proxy_name}:{self.env.proxy_port}:{self._server_addr}'])
 706        if tunnel:
 707            xargs.append('--proxytunnel')
 708        return xargs
 709
 710    def http_get(self, url: str, extra_args: Optional[List[str]] = None,
 711                 alpn_proto: Optional[str] = None,
 712                 def_tracing: bool = True,
 713                 with_stats: bool = False,
 714                 with_profile: bool = False,
 715                 with_tcpdump: bool = False):
 716        return self._raw(url, options=extra_args,
 717                         with_stats=with_stats,
 718                         alpn_proto=alpn_proto,
 719                         def_tracing=def_tracing,
 720                         with_profile=with_profile,
 721                         with_tcpdump=with_tcpdump)
 722
 723    def http_download(self, urls: List[str],
 724                      alpn_proto: Optional[str] = None,
 725                      with_stats: bool = True,
 726                      with_headers: bool = False,
 727                      with_profile: bool = False,
 728                      with_tcpdump: bool = False,
 729                      no_save: bool = False,
 730                      limit_rate: Optional[str] = None,
 731                      extra_args: Optional[List[str]] = None,
 732                      url_options: Optional[Dict[str,List[str]]] = None):
 733        if extra_args is None:
 734            extra_args = []
 735        if no_save:
 736            extra_args.extend(['--out-null'])
 737        else:
 738            extra_args.extend(['-o', 'download_#1.data'])
 739        if limit_rate:
 740            extra_args.extend(['--limit-rate', limit_rate])
 741        # remove any existing ones
 742        for i in range(100):
 743            self._rmf(self.download_file(i))
 744        if with_stats:
 745            extra_args.extend([
 746                '-w', '%{json}\\n'
 747            ])
 748        return self._raw(urls, alpn_proto=alpn_proto, options=extra_args,
 749                         with_stats=with_stats,
 750                         url_options=url_options,
 751                         with_headers=with_headers,
 752                         with_profile=with_profile,
 753                         with_tcpdump=with_tcpdump)
 754
 755    def http_upload(self, urls: List[str], data: str,
 756                    alpn_proto: Optional[str] = None,
 757                    with_stats: bool = True,
 758                    with_headers: bool = False,
 759                    with_profile: bool = False,
 760                    with_tcpdump: bool = False,
 761                    extra_args: Optional[List[str]] = None):
 762        if extra_args is None:
 763            extra_args = []
 764        extra_args.extend([
 765            '--data-binary', data, '-o', 'download_#1.data',
 766        ])
 767        if with_stats:
 768            extra_args.extend([
 769                '-w', '%{json}\\n'
 770            ])
 771        return self._raw(urls, alpn_proto=alpn_proto, options=extra_args,
 772                         with_stats=with_stats,
 773                         with_headers=with_headers,
 774                         with_profile=with_profile,
 775                         with_tcpdump=with_tcpdump)
 776
 777    def http_delete(self, urls: List[str],
 778                    alpn_proto: Optional[str] = None,
 779                    with_stats: bool = True,
 780                    with_profile: bool = False,
 781                    extra_args: Optional[List[str]] = None):
 782        if extra_args is None:
 783            extra_args = []
 784        extra_args.extend([
 785            '-X', 'DELETE', '--out-null',
 786        ])
 787        if with_stats:
 788            extra_args.extend([
 789                '-w', '%{json}\\n'
 790            ])
 791        return self._raw(urls, alpn_proto=alpn_proto, options=extra_args,
 792                         with_stats=with_stats,
 793                         with_headers=False,
 794                         with_profile=with_profile)
 795
 796    def http_put(self, urls: List[str], data=None, fdata=None,
 797                 alpn_proto: Optional[str] = None,
 798                 with_stats: bool = True,
 799                 with_headers: bool = False,
 800                 with_profile: bool = False,
 801                 suppress_cl: bool = False,
 802                 extra_args: Optional[List[str]] = None):
 803        if extra_args is None:
 804            extra_args = []
 805        if fdata is not None:
 806            extra_args.extend(['-T', fdata])
 807        elif data is not None:
 808            extra_args.extend(['-T', '-'])
 809        extra_args.extend([
 810            '-o', 'download_#1.data',
 811        ])
 812        if with_stats:
 813            extra_args.extend([
 814                '-w', '%{json}\\n'
 815            ])
 816        if suppress_cl:
 817            extra_args.extend([
 818                '-H', 'Content-Length:',
 819                '-H', 'Transfer-Encoding: chunked',
 820            ])
 821        return self._raw(urls, intext=data,
 822                         alpn_proto=alpn_proto, options=extra_args,
 823                         with_stats=with_stats,
 824                         with_headers=with_headers,
 825                         with_profile=with_profile)
 826
 827    def http_form(self, urls: List[str], form: Dict[str, str],
 828                  alpn_proto: Optional[str] = None,
 829                  with_stats: bool = True,
 830                  with_headers: bool = False,
 831                  extra_args: Optional[List[str]] = None):
 832        if extra_args is None:
 833            extra_args = []
 834        for key, val in form.items():
 835            extra_args.extend(['-F', f'{key}={val}'])
 836        extra_args.extend([
 837            '-o', 'download_#1.data',
 838        ])
 839        if with_stats:
 840            extra_args.extend([
 841                '-w', '%{json}\\n'
 842            ])
 843        return self._raw(urls, alpn_proto=alpn_proto, options=extra_args,
 844                         with_stats=with_stats,
 845                         with_headers=with_headers)
 846
 847    def ftp_get(self, urls: List[str],
 848                      with_stats: bool = True,
 849                      with_profile: bool = False,
 850                      with_tcpdump: bool = False,
 851                      no_save: bool = False,
 852                      extra_args: Optional[List[str]] = None):
 853        if extra_args is None:
 854            extra_args = []
 855        if no_save:
 856            extra_args.extend([
 857                '--out-null',
 858            ])
 859        else:
 860            extra_args.extend([
 861                '-o', 'download_#1.data',
 862            ])
 863        # remove any existing ones
 864        for i in range(100):
 865            self._rmf(self.download_file(i))
 866        if with_stats:
 867            extra_args.extend([
 868                '-w', '%{json}\\n'
 869            ])
 870        return self._raw(urls, options=extra_args,
 871                         with_stats=with_stats,
 872                         with_headers=False,
 873                         with_profile=with_profile,
 874                         with_tcpdump=with_tcpdump)
 875
 876    def ftp_ssl_get(self, urls: List[str],
 877                      with_stats: bool = True,
 878                      with_profile: bool = False,
 879                      with_tcpdump: bool = False,
 880                      no_save: bool = False,
 881                      extra_args: Optional[List[str]] = None):
 882        if extra_args is None:
 883            extra_args = []
 884        extra_args.extend([
 885            '--ssl-reqd',
 886        ])
 887        return self.ftp_get(urls=urls, with_stats=with_stats,
 888                            with_profile=with_profile, no_save=no_save,
 889                            with_tcpdump=with_tcpdump,
 890                            extra_args=extra_args)
 891
 892    def ftp_upload(self, urls: List[str],
 893                   fupload: Optional[Any] = None,
 894                   updata: Optional[str] = None,
 895                   with_stats: bool = True,
 896                   with_profile: bool = False,
 897                   with_tcpdump: bool = False,
 898                   extra_args: Optional[List[str]] = None):
 899        if extra_args is None:
 900            extra_args = []
 901        if fupload is not None:
 902            extra_args.extend([
 903                '--upload-file', fupload
 904            ])
 905        elif updata is not None:
 906            extra_args.extend([
 907                '--upload-file', '-'
 908            ])
 909        else:
 910            raise Exception('need either file or data to upload')
 911        if with_stats:
 912            extra_args.extend([
 913                '-w', '%{json}\\n'
 914            ])
 915        return self._raw(urls, options=extra_args,
 916                         intext=updata,
 917                         with_stats=with_stats,
 918                         with_headers=False,
 919                         with_profile=with_profile,
 920                         with_tcpdump=with_tcpdump)
 921
 922    def ftp_ssl_upload(self, urls: List[str],
 923                       fupload: Optional[Any] = None,
 924                       updata: Optional[str] = None,
 925                       with_stats: bool = True,
 926                       with_profile: bool = False,
 927                       with_tcpdump: bool = False,
 928                       extra_args: Optional[List[str]] = None):
 929        if extra_args is None:
 930            extra_args = []
 931        extra_args.extend([
 932            '--ssl-reqd',
 933        ])
 934        return self.ftp_upload(urls=urls, fupload=fupload, updata=updata,
 935                               with_stats=with_stats, with_profile=with_profile,
 936                               with_tcpdump=with_tcpdump,
 937                               extra_args=extra_args)
 938
 939    def ssh_download(self, urls: List[str],
 940                     with_stats: bool = True,
 941                     with_profile: bool = False,
 942                     with_tcpdump: bool = False,
 943                     no_save: bool = False,
 944                     extra_args: Optional[List[str]] = None):
 945        if extra_args is None:
 946            extra_args = []
 947        if no_save:
 948            extra_args.extend([
 949                '--out-null',
 950            ])
 951        else:
 952            extra_args.extend([
 953                '-o', 'download_#1.data',
 954            ])
 955        # remove any existing ones
 956        for i in range(100):
 957            self._rmf(self.download_file(i))
 958        if with_stats:
 959            extra_args.extend([
 960                '-w', '%{json}\\n'
 961            ])
 962        return self._raw(urls, options=extra_args,
 963                         with_stats=with_stats,
 964                         with_headers=False,
 965                         with_profile=with_profile,
 966                         with_tcpdump=with_tcpdump)
 967
 968    def ssh_upload(self, urls: List[str],
 969                   fupload: Optional[Any] = None,
 970                   updata: Optional[str] = None,
 971                   with_stats: bool = True,
 972                   with_profile: bool = False,
 973                   with_tcpdump: bool = False,
 974                   extra_args: Optional[List[str]] = None):
 975        if extra_args is None:
 976            extra_args = []
 977        if fupload is not None:
 978            extra_args.extend([
 979                '--upload-file', fupload
 980            ])
 981        elif updata is not None:
 982            extra_args.extend([
 983                '--upload-file', '-'
 984            ])
 985        else:
 986            raise Exception('need either file or data to upload')
 987        if with_stats:
 988            extra_args.extend([
 989                '-w', '%{json}\\n'
 990            ])
 991        return self._raw(urls, options=extra_args,
 992                         intext=updata,
 993                         with_stats=with_stats,
 994                         with_headers=False,
 995                         with_profile=with_profile,
 996                         with_tcpdump=with_tcpdump)
 997
 998    def response_file(self, idx: int):
 999        return os.path.join(self._run_dir, f'download_{idx}.data')
1000
1001    def run_direct(self, args, with_stats: bool = False, with_profile: bool = False):
1002        my_args = [self._curl]
1003        if with_stats:
1004            my_args.extend([
1005                '-w', '%{json}\\n'
1006            ])
1007        my_args.extend([
1008            '-o', 'download.data',
1009        ])
1010        my_args.extend(args)
1011        return self._run(args=my_args, with_stats=with_stats, with_profile=with_profile)
1012
1013    def _run(self, args, intext='', with_stats: bool = False,
1014             with_profile: bool = True, with_tcpdump: bool = False):
1015        self._rmf(self._stdoutfile)
1016        self._rmf(self._stderrfile)
1017        self._rmf(self._headerfile)
1018        exception = None
1019        profile = None
1020        tcpdump = None
1021        perf = None
1022        dtrace = None
1023        if with_tcpdump:
1024            tcpdump = RunTcpDump(self.env, self._run_dir)
1025            tcpdump.start()
1026        started_at = datetime.now()
1027        try:
1028            with open(self._stdoutfile, 'w') as cout, open(self._stderrfile, 'w') as cerr:
1029                if with_profile:
1030                    end_at = started_at + timedelta(seconds=self._timeout) \
1031                        if self._timeout else None
1032                    log.info(f'starting: {args}')
1033                    p = subprocess.Popen(args, stderr=cerr, stdout=cout,
1034                                         cwd=self._run_dir, shell=False,
1035                                         env=self._run_env)
1036                    profile = RunProfile(p.pid, started_at, self._run_dir)
1037                    if intext is not None and False:
1038                        p.communicate(input=intext.encode(), timeout=1)
1039                    if self._with_perf:
1040                        perf = PerfProfile(p.pid, self._run_dir)
1041                        perf.start()
1042                    elif self._with_dtrace:
1043                        dtrace = DTraceProfile(p.pid, self._run_dir)
1044                        dtrace.start()
1045                    ptimeout = 0.0
1046                    while True:
1047                        try:
1048                            p.wait(timeout=ptimeout)
1049                            break
1050                        except subprocess.TimeoutExpired as e:
1051                            if end_at and datetime.now() >= end_at:
1052                                p.kill()
1053                                raise subprocess.TimeoutExpired(cmd=args, timeout=self._timeout) from e
1054                            profile.sample()
1055                            ptimeout = 0.01
1056                    exitcode = p.returncode
1057                    profile.finish()
1058                    log.info(f'done: exit={exitcode}, profile={profile}')
1059                else:
1060                    p = subprocess.run(args, stderr=cerr, stdout=cout,
1061                                       cwd=self._run_dir, shell=False,
1062                                       input=intext.encode() if intext else None,
1063                                       timeout=self._timeout,
1064                                       env=self._run_env)
1065                    exitcode = p.returncode
1066        except subprocess.TimeoutExpired:
1067            now = datetime.now()
1068            duration = now - started_at
1069            log.warning(f'Timeout at {now} after {duration.total_seconds()}s '
1070                        f'(configured {self._timeout}s): {args}')
1071            exitcode = -1
1072            exception = 'TimeoutExpired'
1073        ended_at = datetime.now()
1074        if tcpdump:
1075            tcpdump.finish()
1076        if perf:
1077            perf.finish()
1078        if dtrace:
1079            dtrace.finish()
1080        if self._with_flame:
1081            self._generate_flame(args, dtrace=dtrace, perf=perf)
1082        coutput = open(self._stdoutfile).readlines()
1083        cerrput = open(self._stderrfile).readlines()
1084        return ExecResult(args=args, exit_code=exitcode, exception=exception,
1085                          stdout=coutput, stderr=cerrput,
1086                          duration=ended_at - started_at,
1087                          with_stats=with_stats,
1088                          profile=profile, tcpdump=tcpdump)
1089
1090    def _raw(self, urls, intext='', timeout=None, options=None, insecure=False,
1091             alpn_proto: Optional[str] = None,
1092             url_options=None,
1093             with_stats=False,
1094             with_headers=True,
1095             def_tracing=True,
1096             with_profile=False,
1097             with_tcpdump=False):
1098        args = self._complete_args(
1099            urls=urls, timeout=timeout, options=options, insecure=insecure,
1100            alpn_proto=alpn_proto, with_headers=with_headers,
1101            def_tracing=def_tracing, url_options=url_options)
1102        r = self._run(args, intext=intext, with_stats=with_stats,
1103                      with_profile=with_profile, with_tcpdump=with_tcpdump)
1104        if r.exit_code == 0 and with_headers:
1105            self._parse_headerfile(self._headerfile, r=r)
1106        return r
1107
1108    def _complete_args(self, urls, timeout=None, options=None,
1109                       insecure=False, alpn_proto: Optional[str] = None,
1110                       url_options=None,
1111                       with_headers: bool = True,
1112                       def_tracing: bool = True):
1113        url_sep = []
1114        if not isinstance(urls, list):
1115            urls = [urls]
1116
1117        if options is not None and '--resolve' in options:
1118            force_resolve = False
1119        else:
1120            force_resolve = self._force_resolv
1121
1122        args = [self._curl, "-s", "--path-as-is"]
1123        if 'CURL_TEST_EVENT' in os.environ:
1124            args.append('--test-event')
1125
1126        if self._socks_args:
1127            args.extend(self._socks_args)
1128
1129        if with_headers:
1130            args.extend(["-D", self._headerfile])
1131        if def_tracing is not False and not self._silent:
1132            args.extend(['-v', '--trace-ids', '--trace-time'])
1133            if self.env.verbose > 1:
1134                args.extend(['--trace-config', 'http/2,http/3,h2-proxy,h1-proxy'])
1135
1136        active_options = options
1137        if options is not None and '--next' in options:
1138            active_options = options[options.index('--next') + 1:]
1139
1140        for url in urls:
1141            args.extend(url_sep)
1142            if url_options is not None:
1143                url_sep = ['--next']
1144
1145            u = urlparse(url)
1146            if url_options is not None and url in url_options:
1147                args.extend(url_options[url])
1148            if options:
1149                args.extend(options)
1150            if alpn_proto is not None:
1151                if alpn_proto not in self.ALPN_ARG:
1152                    raise Exception(f'unknown ALPN protocol: "{alpn_proto}"')
1153                args.append(self.ALPN_ARG[alpn_proto])
1154
1155            if u.scheme == 'http':
1156                pass
1157            elif insecure:
1158                args.append('--insecure')
1159            elif active_options and ("--cacert" in active_options or
1160                    "--capath" in active_options):
1161                pass
1162            elif u.hostname:
1163                args.extend(["--cacert", self.env.ca.cert_file])
1164
1165            if force_resolve and u.hostname and u.hostname != 'localhost' \
1166                    and not re.match(r'^(\d+|\[|:).*', u.hostname):
1167                port = u.port if u.port else 443
1168                args.extend([
1169                    '--resolve', f'{u.hostname}:{port}:{self._server_addr}',
1170                ])
1171            if timeout is not None and int(timeout) > 0:
1172                args.extend(["--connect-timeout", str(int(timeout))])
1173            args.append(url)
1174        return args
1175
1176    def _parse_headerfile(self, headerfile: str, r: Optional[ExecResult] = None) -> ExecResult:
1177        lines = open(headerfile).readlines()
1178        if r is None:
1179            r = ExecResult(args=[], exit_code=0, stdout=[], stderr=[])
1180
1181        response = None
1182
1183        def fin_response(resp):
1184            if resp:
1185                r.add_response(resp)
1186
1187        expected = ['status']
1188        for line in lines:
1189            line = line.strip()
1190            if re.match(r'^$', line):
1191                if 'trailer' in expected:
1192                    # end of trailers
1193                    fin_response(response)
1194                    response = None
1195                    expected = ['status']
1196                elif 'header' in expected:
1197                    # end of header, another status or trailers might follow
1198                    expected = ['status', 'trailer']
1199                else:
1200                    assert False, f"unexpected line: '{line}'"
1201                continue
1202            if 'status' in expected:
1203                # log.debug("reading 1st response line: %s", line)
1204                m = re.match(r'^(\S+) (\d+)( .*)?$', line)
1205                if m:
1206                    fin_response(response)
1207                    response = {
1208                        "protocol": m.group(1),
1209                        "status": int(m.group(2)),
1210                        "description": m.group(3),
1211                        "header": {},
1212                        "trailer": {},
1213                        "body": r.outraw
1214                    }
1215                    expected = ['header']
1216                    continue
1217            if 'trailer' in expected:
1218                m = re.match(r'^([^:]+):\s*(.*)$', line)
1219                if m:
1220                    response['trailer'][m.group(1).lower()] = m.group(2)
1221                    continue
1222            if 'header' in expected:
1223                m = re.match(r'^([^:]+):\s*(.*)$', line)
1224                if m:
1225                    response['header'][m.group(1).lower()] = m.group(2)
1226                    continue
1227            assert False, f"unexpected line: '{line}, expected: {expected}'"
1228
1229        fin_response(response)
1230        return r
1231
1232    def _perf_collapse(self, perf: PerfProfile, file_err):
1233        if not os.path.exists(perf.file):
1234            raise Exception(f'dtrace output file does not exist: {perf.file}')
1235        fg_collapse = os.path.join(self._fg_dir, 'stackcollapse-perf.pl')
1236        if not os.path.exists(fg_collapse):
1237            raise Exception(f'FlameGraph script not found: {fg_collapse}')
1238        stacks_collapsed = f'{perf.file}.collapsed'
1239        log.info(f'collapsing stacks into {stacks_collapsed}')
1240        with open(stacks_collapsed, 'w') as cout, open(file_err, 'w') as cerr:
1241            p = subprocess.run([
1242                fg_collapse, perf.file
1243            ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
1244            rc = p.returncode
1245            if rc != 0:
1246                raise Exception(f'{fg_collapse} returned error {rc}')
1247        return stacks_collapsed
1248
1249    def _dtrace_collapse(self, dtrace: DTraceProfile, file_err):
1250        if not os.path.exists(dtrace.file):
1251            raise Exception(f'dtrace output file does not exist: {dtrace.file}')
1252        fg_collapse = os.path.join(self._fg_dir, 'stackcollapse.pl')
1253        if not os.path.exists(fg_collapse):
1254            raise Exception(f'FlameGraph script not found: {fg_collapse}')
1255        stacks_collapsed = f'{dtrace.file}.collapsed'
1256        log.info(f'collapsing stacks into {stacks_collapsed}')
1257        with open(stacks_collapsed, 'w') as cout, open(file_err, 'a') as cerr:
1258            p = subprocess.run([
1259                fg_collapse, dtrace.file
1260            ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
1261            rc = p.returncode
1262            if rc != 0:
1263                raise Exception(f'{fg_collapse} returned error {rc}')
1264        return stacks_collapsed
1265
1266    def _generate_flame(self, curl_args: List[str],
1267                        dtrace: Optional[DTraceProfile] = None,
1268                        perf: Optional[PerfProfile] = None):
1269        fg_gen_flame = os.path.join(self._fg_dir, 'flamegraph.pl')
1270        file_svg = os.path.join(self._run_dir, 'curl.flamegraph.svg')
1271        if not os.path.exists(fg_gen_flame):
1272            raise Exception(f'FlameGraph script not found: {fg_gen_flame}')
1273
1274        log.info('waiting a sec for perf/dtrace to finish flushing')
1275        time.sleep(2)
1276        log.info('generating flame graph for this run')
1277        file_err = os.path.join(self._run_dir, 'curl.flamegraph.stderr')
1278        if perf:
1279            stacks_collapsed = self._perf_collapse(perf, file_err)
1280        elif dtrace:
1281            stacks_collapsed = self._dtrace_collapse(dtrace, file_err)
1282        else:
1283            raise Exception('no stacks measure given')
1284
1285        log.info(f'generating graph into {file_svg}')
1286        cmdline = ' '.join(curl_args)
1287        if len(cmdline) > 80:
1288            title = f'{cmdline[:80]}...'
1289            subtitle = f'...{cmdline[-80:]}'
1290        else:
1291            title = cmdline
1292            subtitle = ''
1293        with open(file_svg, 'w') as cout, open(file_err, 'a') as cerr:
1294            p = subprocess.run([
1295                fg_gen_flame, '--colors', 'green',
1296                '--title', title, '--subtitle', subtitle,
1297                stacks_collapsed
1298            ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
1299            rc = p.returncode
1300            if rc != 0:
1301                raise Exception(f'{fg_gen_flame} returned error {rc}')
1302
1303    def mk_altsvc_file(self, name, src_alpn, src_host, src_port,
1304                       dest_alpn, dest_host, dest_port):
1305        fpath = os.path.join(self.run_dir, f'{name}.altsvc')
1306        ts = datetime.now(timezone.utc) + timedelta(hours=1)
1307        ts = ts.strftime('%Y%m%d %H:%M:%S')
1308        with open(fpath, 'w') as fd:
1309            fd.write(f'{src_alpn} {src_host} {src_port} {dest_alpn} {dest_host} {dest_port} "{ts}" 1 0\n')
1310        return fpath