luna - curl/tests/runtests.pl

   1#!/usr/bin/env perl
   2#***************************************************************************
   3#                                  _   _ ____  _
   4#  Project                     ___| | | |  _ \| |
   5#                             / __| | | | |_) | |
   6#                            | (__| |_| |  _ <| |___
   7#                             \___|\___/|_| \_\_____|
   8#
   9# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
  10#
  11# This software is licensed as described in the file COPYING, which
  12# you should have received as part of this distribution. The terms
  13# are also available at https://curl.se/docs/copyright.html.
  14#
  15# You may opt to use, copy, modify, merge, publish, distribute and/or sell
  16# copies of the Software, and permit persons to whom the Software is
  17# furnished to do so, under the terms of the COPYING file.
  18#
  19# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  20# KIND, either express or implied.
  21#
  22# SPDX-License-Identifier: curl
  23#
  24###########################################################################
  25
  26# For documentation, see docs/runtests.md and README.md.
  27
  28# Experimental hooks are available to run tests remotely on machines that
  29# are able to run curl but are unable to run the test harness.
  30# The following sections need to be modified:
  31#
  32#  $HOSTIP, $HOST6IP - Set to the address of the host running the test suite
  33#  $CLIENTIP, $CLIENT6IP - Set to the address of the host running curl
  34#  runclient, runclientoutput - Modify to copy all the files in the log/
  35#    directory to the system running curl, run the given command remotely
  36#    and save the return code or returned stdout (respectively), then
  37#    copy all the files from the remote system's log/ directory back to
  38#    the host running the test suite.  This can be done a few ways, such
  39#    as using scp & ssh, rsync & telnet, or using a NFS shared directory
  40#    and ssh.
  41#
  42# 'make && make test' needs to be done on both machines before making the
  43# above changes and running runtests.pl manually.  In the shared NFS case,
  44# the contents of the tests/server/ directory must be from the host
  45# running the test suite, while the rest must be from the host running curl.
  46#
  47# Note that even with these changes a number of tests will still fail (mainly
  48# to do with cookies, those that set environment variables, or those that
  49# do more than touch the file system in a <precheck> or <postcheck>
  50# section). These can be added to the $TESTCASES line below,
  51# e.g. $TESTCASES="!8 !31 !63 !cookies..."
  52#
  53# Finally, to properly support -g and -n, checktestcmd needs to change
  54# to check the remote system's PATH, and the places in the code where
  55# the curl binary is read directly to determine its type also need to be
  56# fixed. As long as the -g option is never given, and the -n is always
  57# given, this will not be a problem.
  58
  59use strict;
  60use warnings;
  61use 5.006;
  62use POSIX qw(strftime);
  63
  64# These should be the only variables that might be needed to get edited:
  65
  66BEGIN {
  67    # Define srcdir to the location of the tests source directory. This is
  68    # usually set by the Makefile, but for out-of-tree builds with direct
  69    # invocation of runtests.pl, it may not be set.
  70    if(!defined $ENV{'srcdir'}) {
  71        use File::Basename;
  72        $ENV{'srcdir'} = dirname(__FILE__);
  73    }
  74    push(@INC, $ENV{'srcdir'});
  75    # run time statistics needs Time::HiRes
  76    eval {
  77        no warnings "all";
  78        require Time::HiRes;
  79        import  Time::HiRes qw( time );
  80    }
  81}
  82
  83use Digest::MD5 qw(md5);
  84use List::Util 'sum';
  85use I18N::Langinfo qw(langinfo CODESET);
  86use POSIX qw(setlocale LC_ALL);
  87
  88use serverhelp qw(
  89    server_exe
  90    );
  91use pathhelp qw(
  92    exe_ext
  93    sys_native_current_path
  94    shell_quote
  95    );
  96
  97use appveyor;
  98use azure;
  99use getpart;   # array functions
 100use servers;
 101use valgrind;  # valgrind report parser
 102use globalconfig;
 103use runner;
 104use testutil;
 105use memanalyzer;
 106
 107my %custom_skip_reasons;
 108
 109my $ACURL=$VCURL;  # what curl binary to use to talk to APIs (relevant for CI)
 110                   # ACURL is handy to set to the system one for reliability
 111my $CURLCONFIG="../curl-config"; # curl-config from current build
 112
 113# Normally, all test cases should be run, but at times it is handy to
 114# run a particular one:
 115my $TESTCASES="all";
 116
 117# To run specific test cases, set them like:
 118# $TESTCASES="1 2 3 7 8";
 119
 120#######################################################################
 121# No variables below this point should need to be modified
 122#
 123
 124my $libtool;
 125my $repeat = 0;
 126my $retry = 0;
 127
 128my $start;          # time at which testing started
 129my $args;           # command-line arguments
 130
 131my $uname_release = `uname -r`;
 132my $is_wsl = $uname_release =~ /Microsoft$/;
 133
 134my $http_ipv6;      # set if HTTP server has IPv6 support
 135my $http_unix;      # set if HTTP server has Unix sockets support
 136my $ftp_ipv6;       # set if FTP server has IPv6 support
 137
 138my $resolver;       # name of the resolver backend (for human presentation)
 139
 140my %skipped;    # skipped{reason}=counter, reasons for skip
 141my @teststat;   # teststat[testnum]=reason, reasons for skip
 142my %disabled_keywords;  # key words of tests to skip
 143my %ignored_keywords;   # key words of tests to ignore results
 144my %enabled_keywords;   # key words of tests to run
 145my %disabled;           # disabled test cases
 146my %ignored;            # ignored results of test cases
 147my %ignoretestcodes;    # if test results are to be ignored
 148
 149my $passedign;   # tests passed with results ignored
 150
 151my $timestats;   # time stamping and stats generation
 152my $fullstats;   # show time stats for every single test
 153my %timeprepini; # timestamp for each test preparation start
 154my %timesrvrini; # timestamp for each test required servers verification start
 155my %timesrvrend; # timestamp for each test required servers verification end
 156my %timetoolini; # timestamp for each test command run starting
 157my %timetoolend; # timestamp for each test command run stopping
 158my %timesrvrlog; # timestamp for each test server logs lock removal
 159my %timevrfyend; # timestamp for each test result verification end
 160my $globalabort; # flag signalling program abort
 161
 162# values for $singletest_state
 163use constant {
 164    ST_INIT => 0,
 165    ST_INITED => 2,
 166    ST_PREPROCESS => 3,
 167    ST_RUN => 4,
 168};
 169my %singletest_state;  # current state of singletest() by runner ID
 170my %singletest_logs;   # log messages while in singletest array ref by runner
 171my $singletest_bufferedrunner; # runner ID which is buffering logs
 172my %runnerids;         # runner IDs by number
 173my @runnersidle;       # runner IDs idle and ready to execute a test
 174my %countforrunner;    # test count by runner ID
 175my %runnersrunning;    # tests currently running by runner ID
 176
 177#######################################################################
 178# variables that command line options may set
 179#
 180my $short;
 181my $no_debuginfod;
 182my $keepoutfiles; # keep stdout and stderr files after tests
 183my $postmortem;   # display detailed info about failed tests
 184my $run_disabled; # run the specific tests even if listed in DISABLED
 185my $scrambleorder;
 186my $jobs = 0;
 187
 188# Azure Pipelines specific variables
 189my $AZURE_RUN_ID = 0;
 190my $AZURE_RESULT_ID = 0;
 191
 192#######################################################################
 193# logmsg is our general message logging subroutine.
 194#
 195sub logmsg {
 196    if($singletest_bufferedrunner) {
 197        # Logs are currently being buffered
 198        return singletest_logmsg(@_);
 199    }
 200    for(@_) {
 201        my $line = $_;
 202        if(!$line) {
 203            next;
 204        }
 205        if($is_wsl) {
 206            # use \r\n for WSL shell
 207            $line =~ s/\r?\n$/\r\n/g;
 208        }
 209        print $line;
 210    }
 211}
 212
 213#######################################################################
 214# enable logmsg buffering for the given runner ID
 215#
 216sub logmsg_bufferfortest {
 217    my ($runnerid)=@_;
 218    if($jobs) {
 219        # Only enable buffering in multiprocess mode
 220        $singletest_bufferedrunner = $runnerid;
 221    }
 222}
 223
 224#######################################################################
 225# Store a log message in a buffer for this test
 226# The messages can then be displayed all at once at the end of the test
 227# which prevents messages from different tests from being interleaved.
 228sub singletest_logmsg {
 229    if(!exists $singletest_logs{$singletest_bufferedrunner}) {
 230        # initialize to a reference to an empty anonymous array
 231        $singletest_logs{$singletest_bufferedrunner} = [];
 232    }
 233    my $logsref = $singletest_logs{$singletest_bufferedrunner};
 234    push @$logsref, @_;
 235}
 236
 237#######################################################################
 238# Stop buffering log messages, but do not touch them
 239sub singletest_unbufferlogs {
 240    undef $singletest_bufferedrunner;
 241}
 242
 243#######################################################################
 244# Clear the buffered log messages & stop buffering after returning them
 245sub singletest_dumplogs {
 246    if(!defined $singletest_bufferedrunner) {
 247        # probably not multiprocess mode and logs were not buffered
 248        return undef;
 249    }
 250    my $logsref = $singletest_logs{$singletest_bufferedrunner};
 251    my $msg = join("", @$logsref);
 252    delete $singletest_logs{$singletest_bufferedrunner};
 253    singletest_unbufferlogs();
 254    return $msg;
 255}
 256
 257sub catch_zap {
 258    my $signame = shift;
 259    print "runtests.pl received SIG$signame, exiting\r\n";
 260    $globalabort = 1;
 261}
 262$SIG{INT} = \&catch_zap;
 263$SIG{TERM} = \&catch_zap;
 264
 265sub catch_usr1 {
 266    print "runtests.pl internal state:\r\n";
 267    print scalar(%runnersrunning) . " busy test runner(s) of " . scalar(keys %runnerids) . "\r\n";
 268    foreach my $rid (sort(keys(%runnersrunning))) {
 269        my $runnernum = "unknown";
 270        foreach my $rnum (keys %runnerids) {
 271            if($runnerids{$rnum} == $rid) {
 272                $runnernum = $rnum;
 273                last;
 274            }
 275        }
 276        print "Runner $runnernum (id $rid) running test $runnersrunning{$rid} in state $singletest_state{$rid}\r\n";
 277    }
 278}
 279
 280eval {
 281    # some msys2 perl versions do not define SIGUSR1
 282    $SIG{USR1} = \&catch_usr1;
 283};
 284$SIG{PIPE} = 'IGNORE';  # these errors are captured in the read/write calls
 285
 286##########################################################################
 287# Clear all possible '*_proxy' environment variables for various protocols
 288# to prevent them to interfere with our testing!
 289
 290foreach my $protocol (('ftp', 'http', 'ftps', 'https', 'no', 'all')) {
 291    my $proxy = "${protocol}_proxy";
 292    # clear lowercase version
 293    delete $ENV{$proxy} if($ENV{$proxy});
 294    # clear uppercase version
 295    delete $ENV{uc($proxy)} if($ENV{uc($proxy)});
 296}
 297
 298# make sure we do not get affected by other variables that control our
 299# behavior
 300
 301delete $ENV{'SSL_CERT_DIR'} if($ENV{'SSL_CERT_DIR'});
 302delete $ENV{'SSL_CERT_PATH'} if($ENV{'SSL_CERT_PATH'});
 303delete $ENV{'CURL_CA_BUNDLE'} if($ENV{'CURL_CA_BUNDLE'});
 304
 305# provide defaults from our config file for ENV vars not explicitly
 306# set by the caller
 307if(open(my $fd, "<", "config")) {
 308    while(my $line = <$fd>) {
 309        next if($line =~ /^#/);
 310        chomp $line;
 311        my ($name, $val) = split(/\s*:\s*/, $line, 2);
 312        $ENV{$name} = $val if(!$ENV{$name});
 313    }
 314    close($fd);
 315}
 316
 317# Check if we have nghttpx available and if it talks http/3
 318my $nghttpx_h3 = 0;
 319if(!$ENV{"NGHTTPX"}) {
 320    $ENV{"NGHTTPX"} = checktestcmd("nghttpx");
 321}
 322if($ENV{"NGHTTPX"}) {
 323    my $cmd = "\"$ENV{'NGHTTPX'}\" -v 2>$dev_null";
 324    my $nghttpx_version=join(' ', `$cmd`);
 325    $nghttpx_h3 = $nghttpx_version =~ /nghttp3\//;
 326    chomp $nghttpx_h3;
 327}
 328
 329#######################################################################
 330# Get the list of tests that the tests/data/Makefile.am knows about!
 331#
 332my $disttests = "";
 333sub get_disttests {
 334    # If a non-default $TESTDIR is being used there may not be any
 335    # Makefile.am in which case there is nothing to do.
 336    open(my $dh, "<", "$TESTDIR/Makefile.am") or return;
 337    while(<$dh>) {
 338        chomp $_;
 339        if(($_ =~ /^#/) ||($_ !~ /test/)) {
 340            next;
 341        }
 342        $disttests .= $_;
 343    }
 344    close($dh);
 345}
 346
 347#######################################################################
 348# Remove all files in the specified directory
 349#
 350sub cleardir {
 351    my $dir = $_[0];
 352    my $done = 1;  # success
 353    my $file;
 354
 355    # Get all files
 356    opendir(my $dh, $dir) ||
 357        return 0; # cannot open dir
 358    while($file = readdir($dh)) {
 359        # Do not clear the $PIDDIR or $LOCKDIR since those need to live beyond
 360        # one test
 361        if(($file !~ /^(\.|\.\.)\z/) &&
 362            $file ne $PIDDIR && $file ne $LOCKDIR) {
 363            if(-d "$dir/$file") {
 364                if(!cleardir("$dir/$file")) {
 365                    $done = 0;
 366                }
 367                if(!rmdir("$dir/$file")) {
 368                    $done = 0;
 369                }
 370            }
 371            else {
 372                # Ignore stunnel since we cannot do anything about its locks
 373                if(!unlink("$dir/$file") && $file !~ /_stunnel\.log$/) {
 374                    $done = 0;
 375                }
 376            }
 377        }
 378    }
 379    closedir $dh;
 380    return $done;
 381}
 382
 383#######################################################################
 384# Given two array references, this function will store them in two temporary
 385# files, run 'diff' on them, store the result and return the diff output!
 386sub showdiff {
 387    my ($logdir, $firstref, $secondref)=@_;
 388
 389    my $file1="$logdir/check-generated";
 390    my $file2="$logdir/check-expected";
 391
 392    open(my $temp, ">", $file1) || die "Failure writing diff file";
 393    for(@$firstref) {
 394        my $l = $_;
 395        $l =~ s/\r/[CR]/g;
 396        $l =~ s/\n/[LF]/g;
 397        $l =~ s/([^\x20-\x7f])/sprintf "%%%02x", ord $1/eg;
 398        print $temp $l;
 399        print $temp "\n";
 400    }
 401    close($temp) || die "Failure writing diff file";
 402
 403    open($temp, ">", $file2) || die "Failure writing diff file";
 404    for(@$secondref) {
 405        my $l = $_;
 406        $l =~ s/\r/[CR]/g;
 407        $l =~ s/\n/[LF]/g;
 408        $l =~ s/([^\x20-\x7f])/sprintf "%%%02x", ord $1/eg;
 409        print $temp $l;
 410        print $temp "\n";
 411    }
 412    close($temp) || die "Failure writing diff file";
 413    my @out = `diff -u $file2 $file1 2>$dev_null`;
 414
 415    if(!$out[0]) {
 416        @out = `diff -c $file2 $file1 2>$dev_null`;
 417        if(!$out[0]) {
 418            logmsg "Failed to show diff. The diff tool may be missing.\n";
 419        }
 420    }
 421
 422    return @out;
 423}
 424
 425#######################################################################
 426# compare test results with the expected output, we might filter off
 427# some pattern that is allowed to differ, output test results
 428#
 429sub compare {
 430    my ($runnerid, $testnum, $testname, $subject, $firstref, $secondref)=@_;
 431
 432    my $result = compareparts($firstref, $secondref);
 433
 434    if($result) {
 435        # timestamp test result verification end
 436        $timevrfyend{$testnum} = Time::HiRes::time();
 437
 438        if(!$short) {
 439            logmsg "\n $testnum: $subject FAILED:\n";
 440            my $logdir = getrunnerlogdir($runnerid);
 441            logmsg showdiff($logdir, $firstref, $secondref);
 442        }
 443        elsif(!$automakestyle) {
 444            logmsg "FAILED\n";
 445        }
 446        else {
 447            # automakestyle
 448            logmsg "FAIL: $testnum - $testname - $subject\n";
 449        }
 450    }
 451    return $result;
 452}
 453
 454#######################################################################
 455# Numeric-sort words in a string
 456sub numsortwords {
 457    my ($string)=@_;
 458    return join(' ', sort { $a <=> $b } split(' ', $string));
 459}
 460
 461#######################################################################
 462# Parse and store the protocols in curl's Protocols: line
 463sub parseprotocols {
 464    my ($line)=@_;
 465
 466    @protocols = split(' ', lc($line));
 467
 468    # Generate a "proto-ipv6" version of each protocol to match the
 469    # IPv6 <server> name and a "proto-unix" to match the variant which
 470    # uses Unix domain sockets. This works even if support is not
 471    # compiled in because the <features> test will fail.
 472    push @protocols, map(("$_-ipv6", "$_-unix"), @protocols);
 473
 474    # 'http-proxy' is used in test cases to do CONNECT through
 475    push @protocols, 'http-proxy';
 476
 477    # 'https-mtls' is used for client certificate auth testing
 478    push @protocols, 'https-mtls';
 479
 480    # 'none' is used in test cases to mean no server
 481    push @protocols, 'none';
 482}
 483
 484#######################################################################
 485# Check if the operating environment supports UTF-8.
 486sub is_utf8_supported {
 487    my $result;
 488    my $old_LC_ALL;
 489    my $was_defined = defined $ENV{'LC_ALL'};
 490    if($was_defined) {
 491        $old_LC_ALL = $ENV{'LC_ALL'};
 492    }
 493    setlocale(LC_ALL, $ENV{'LC_ALL'} = "C.UTF-8");
 494    $result = lc(langinfo(CODESET())) eq "utf-8";
 495    if($was_defined) {
 496        $ENV{'LC_ALL'} = $old_LC_ALL;
 497    }
 498    else {
 499        delete $ENV{'LC_ALL'};
 500    }
 501    return $result;
 502}
 503
 504#######################################################################
 505# Check & display information about curl and the host the test suite runs on.
 506# Information to do with servers is displayed in displayserverfeatures, after
 507# the server initialization is performed.
 508sub checksystemfeatures {
 509    my $proto;
 510    my $feat;
 511    my $curl;
 512    my $libcurl;
 513    my $versretval;
 514    my $versnoexec;
 515    my @version=();
 516    my @disabled;
 517    my $dis = "";
 518
 519    my $curlverout="$LOGDIR/curlverout.log";
 520    my $curlvererr="$LOGDIR/curlvererr.log";
 521    my $versioncmd=exerunner() . shell_quote($CURL) . " --version 1>$curlverout 2>$curlvererr";
 522
 523    unlink($curlverout);
 524    unlink($curlvererr);
 525
 526    $versretval = runclient($versioncmd);
 527    $versnoexec = $!;
 528
 529    my $current_time = int(time());
 530    $ENV{'SOURCE_DATE_EPOCH'} = $current_time;
 531    $DATE = strftime "%Y-%m-%d", gmtime($current_time);
 532
 533    open(my $versout, "<", $curlverout);
 534    @version = <$versout>;
 535    close($versout);
 536
 537    open(my $disabledh, "-|", exerunner() . shell_quote($CURLINFO));
 538    while(<$disabledh>) {
 539        if($_ =~ /([^:]*): ([ONF]*)/) {
 540            my ($val, $toggle) = ($1, $2);
 541            push @disabled, $val if($toggle eq "OFF");
 542            $feature{$val} = 1 if($toggle eq "ON");
 543        }
 544    }
 545    close($disabledh);
 546
 547    if($disabled[0]) {
 548        s/[\r\n]//g for @disabled;
 549        $dis = join(", ", @disabled);
 550    }
 551
 552    $resolver="stock";
 553    for(@version) {
 554        chomp;
 555
 556        if($_ =~ /^curl ([^ ]*)/) {
 557            $curl = $_;
 558            $CURLVERSION = $1;
 559            $CURLVERNUM = $CURLVERSION;
 560            $CURLVERNUM =~ s/^([0-9.]+)(.*)/$1/; # leading dots and numbers
 561            $curl =~ s/^(.*)(libcurl.*)/$1/g || die "Failure determining curl binary version";
 562
 563            $libcurl = $2;
 564            if($curl =~ /win32|Windows|windows|mingw(32|64)/) {
 565                # This is a Windows MinGW build or native build, we need to use
 566                # Windows-style path.
 567                $pwd = sys_native_current_path();
 568                $feature{"win32"} = 1;
 569            }
 570            if($curl =~ /cygwin|msys/i) {
 571                $feature{"cygwin"} = 1;
 572            }
 573            if($libcurl =~ /\sschannel\b/i) {
 574                $feature{"Schannel"} = 1;
 575                $feature{"SSLpinning"} = 1;
 576            }
 577            elsif($libcurl =~ /\sopenssl\b/i) {
 578                $feature{"OpenSSL"} = 1;
 579                $feature{"SSLpinning"} = 1;
 580            }
 581            elsif($libcurl =~ /\sgnutls\b/i) {
 582                $feature{"GnuTLS"} = 1;
 583                $feature{"SSLpinning"} = 1;
 584            }
 585            elsif($libcurl =~ /\srustls-ffi\b/i) {
 586                $feature{"rustls"} = 1;
 587            }
 588            elsif($libcurl =~ /\swolfssl\b/i) {
 589                $feature{"wolfssl"} = 1;
 590                $feature{"SSLpinning"} = 1;
 591            }
 592            elsif($libcurl =~ /\s(AWS-LC|BoringSSL)\b/i) {
 593                # OpenSSL compatible API
 594                $feature{"OpenSSL"} = 1;
 595                $feature{"SSLpinning"} = 1;
 596            }
 597            elsif($libcurl =~ /\slibressl\b/i) {
 598                # OpenSSL compatible API
 599                $feature{"OpenSSL"} = 1;
 600                $feature{"SSLpinning"} = 1;
 601            }
 602            elsif($libcurl =~ /\squictls\b/i) {
 603                # OpenSSL compatible API
 604                $feature{"OpenSSL"} = 1;
 605                $feature{"SSLpinning"} = 1;
 606            }
 607            elsif($libcurl =~ /\smbedTLS\b/i) {
 608                $feature{"mbedtls"} = 1;
 609                $feature{"SSLpinning"} = 1;
 610            }
 611            if($libcurl =~ /ares/i) {
 612                $feature{"c-ares"} = 1;
 613                $resolver="c-ares";
 614            }
 615            if($libcurl =~ /nghttp2/i) {
 616                # nghttp2 supports h2c
 617                $feature{"h2c"} = 1;
 618            }
 619            if($libcurl =~ /AppleIDN/) {
 620                $feature{"AppleIDN"} = 1;
 621            }
 622            if($libcurl =~ /WinIDN/) {
 623                $feature{"WinIDN"} = 1;
 624            }
 625            if($libcurl =~ /libidn2/) {
 626                $feature{"libidn2"} = 1;
 627            }
 628            if($libcurl =~ /libssh2/i) {
 629                $feature{"libssh2"} = 1;
 630                $feature{"sshkeyalgo"} = ($ENV{'CURL_TEST_SSH_KEYALGO'} and
 631                    $ENV{'CURL_TEST_SSH_KEYALGO'} =~ /^(?:rsa|ecdsa|ed25519)$/) ? $ENV{'CURL_TEST_SSH_KEYALGO'} : 'rsa';
 632            }
 633            if($libcurl =~ /libssh\/([0-9.]*)\//i) {
 634                $feature{"libssh"} = 1;
 635                $feature{"sshkeyalgo"} = ($ENV{'CURL_TEST_SSH_KEYALGO'} and
 636                    $ENV{'CURL_TEST_SSH_KEYALGO'} =~ /^(?:rsa|ecdsa|ed25519)$/) ? $ENV{'CURL_TEST_SSH_KEYALGO'} : 'rsa';
 637                # Detect simple cases of default libssh configuration files ending up
 638                # setting `StrictHostKeyChecking no`. include files, quoted values,
 639                # '=value' format not implemented.
 640                $feature{"badlibssh"} = 0;
 641                foreach my $libssh_configfile (('/etc/ssh/ssh_config', $ENV{'HOME'} . '/.ssh/config')) {
 642                    if(open(my $fd, '<', $libssh_configfile)) {
 643                        while(my $line = <$fd>) {
 644                            chomp $line;
 645                            if($line =~ /^\s*StrictHostKeyChecking\s+(yes|no)\s*$/) {
 646                                $feature{"badlibssh"} = ($1 eq 'no' ? 1 : 0);
 647                                last;  # Do as openssh and libssh
 648                            }
 649                        }
 650                        close($fd);
 651                    }
 652                }
 653            }
 654        }
 655        elsif($_ =~ /^Protocols: (.*)/i) {
 656            $proto = $1;
 657            # these are the protocols compiled in to this libcurl
 658            parseprotocols($proto);
 659        }
 660        elsif($_ =~ /^Features: (.*)/i) {
 661            $feat = $1;
 662
 663            # built with memory tracking support (--enable-debug); may be disabled later
 664            $feature{"TrackMemory"} = $feat =~ /\bDebug/;
 665            # curl was built with --enable-debug
 666            $feature{"Debug"} = $feat =~ /\bDebug/;
 667            # ssl enabled
 668            $feature{"SSL"} = $feat =~ /SSL/i;
 669            # multiple ssl backends available.
 670            $feature{"MultiSSL"} = $feat =~ /MultiSSL/i;
 671            # large file support
 672            $feature{"Largefile"} = $feat =~ /Largefile/i;
 673            # IDN support
 674            $feature{"IDN"} = $feat =~ /IDN/i;
 675            # IPv6 support
 676            $feature{"IPv6"} = $feat =~ /IPv6/i;
 677            # Unix sockets support
 678            $feature{"UnixSockets"} = $feat =~ /UnixSockets/i;
 679            # libz compression
 680            $feature{"libz"} = $feat =~ /libz/i;
 681            # Brotli compression
 682            $feature{"brotli"} = $feat =~ /brotli/i;
 683            # Zstd compression
 684            $feature{"zstd"} = $feat =~ /zstd/i;
 685            # NTLM enabled
 686            $feature{"NTLM"} = $feat =~ /NTLM/i;
 687            # NTLM delegation to winbind daemon ntlm_auth helper enabled
 688            $feature{"NTLM_WB"} = $feat =~ /NTLM_WB/i;
 689            # SSPI enabled
 690            $feature{"SSPI"} = $feat =~ /SSPI/i;
 691            # GSS-API enabled
 692            $feature{"GSS-API"} = $feat =~ /GSS-API/i;
 693            # Kerberos enabled
 694            $feature{"Kerberos"} = $feat =~ /Kerberos/i;
 695            # SPNEGO enabled
 696            $feature{"SPNEGO"} = $feat =~ /SPNEGO/i;
 697            # TLS-SRP enabled
 698            $feature{"TLS-SRP"} = $feat =~ /TLS-SRP/i;
 699            # PSL enabled
 700            $feature{"PSL"} = $feat =~ /PSL/i;
 701            # alt-svc enabled
 702            $feature{"alt-svc"} = $feat =~ /alt-svc/i;
 703            # HSTS support
 704            $feature{"HSTS"} = $feat =~ /HSTS/i;
 705            $feature{"asyn-rr"} = $feat =~ /asyn-rr/;
 706            if($feat =~ /AsynchDNS/i) {
 707                if(!$feature{"c-ares"} || $feature{"asyn-rr"}) {
 708                    # this means threaded resolver
 709                    $feature{"threaded-resolver"} = 1;
 710                    $resolver="threaded";
 711
 712                    # does not count as "real" c-ares
 713                    $feature{"c-ares"} = 0;
 714                }
 715            }
 716            # http2 enabled
 717            $feature{"http/2"} = $feat =~ /HTTP2/;
 718            if($feature{"http/2"}) {
 719                push @protocols, 'http/2';
 720            }
 721            # http3 enabled
 722            $feature{"http/3"} = $feat =~ /HTTP3/;
 723            if($feature{"http/3"}) {
 724                push @protocols, 'http/3';
 725            }
 726            # https proxy support
 727            $feature{"HTTPS-proxy"} = $feat =~ /HTTPS-proxy/;
 728            if($feature{"HTTPS-proxy"}) {
 729                # 'https-proxy' is used as "server" so consider it a protocol
 730                push @protocols, 'https-proxy';
 731            }
 732            # Unicode support
 733            $feature{"Unicode"} = $feat =~ /Unicode/i;
 734            # Thread-safe init
 735            $feature{"threadsafe"} = $feat =~ /threadsafe/i;
 736            $feature{"HTTPSRR"} = $feat =~ /HTTPSRR/;
 737            $feature{"ECH"} = $feat =~ /ECH/;
 738        }
 739        #
 740        # Test harness currently uses a non-stunnel server in order to
 741        # run HTTP TLS-SRP tests required when curl is built with https
 742        # protocol support and TLS-SRP feature enabled. For convenience
 743        # 'httptls' may be included in the test harness protocols array
 744        # to differentiate this from classic stunnel based 'https' test
 745        # harness server.
 746        #
 747        if($feature{"TLS-SRP"}) {
 748            my $add_httptls;
 749            for(@protocols) {
 750                if($_ =~ /^https(-ipv6|)$/) {
 751                    $add_httptls=1;
 752                    last;
 753                }
 754            }
 755            if($add_httptls && (! grep /^httptls$/, @protocols)) {
 756                push @protocols, 'httptls';
 757                push @protocols, 'httptls-ipv6';
 758            }
 759        }
 760    }
 761
 762    if(!$curl) {
 763        logmsg "unable to get curl's version, further details are:\n";
 764        logmsg "issued command: \n";
 765        logmsg "$versioncmd \n";
 766        if($versretval == -1) {
 767            logmsg "command failed with: \n";
 768            logmsg "$versnoexec \n";
 769        }
 770        elsif($versretval & 127) {
 771            logmsg sprintf("command died with signal %d, and %s coredump.\n",
 772                           ($versretval & 127), ($versretval & 128) ? "a" : "no");
 773        }
 774        else {
 775            logmsg sprintf("command exited with value %d \n", $versretval >> 8);
 776        }
 777        logmsg "contents of $curlverout: \n";
 778        displaylogcontent($curlverout);
 779        logmsg "contents of $curlvererr: \n";
 780        displaylogcontent($curlvererr);
 781        die "Could not get curl's version";
 782    }
 783
 784    if(-r "../lib/curl_config.h") {
 785        open(my $conf, "<", "../lib/curl_config.h");
 786        while(<$conf>) {
 787            if($_ =~ /^\#define HAVE_GETRLIMIT/) {
 788                # set if system has getrlimit()
 789                $feature{"getrlimit"} = 1;
 790            }
 791        }
 792        close($conf);
 793    }
 794
 795    if($feature{"IPv6"}) {
 796        # client has IPv6 support
 797
 798        # check if the HTTP server has it!
 799        my $cmd = server_exe('sws')." --version";
 800        my @sws = `$cmd`;
 801        if($sws[0] =~ /IPv6/) {
 802            # HTTP server has IPv6 support!
 803            $http_ipv6 = 1;
 804        }
 805
 806        # check if the FTP server has it!
 807        $cmd = server_exe('sockfilt')." --version";
 808        @sws = `$cmd`;
 809        if($sws[0] =~ /IPv6/) {
 810            # FTP server has IPv6 support!
 811            $ftp_ipv6 = 1;
 812        }
 813    }
 814
 815    if($feature{"UnixSockets"}) {
 816        # client has Unix sockets support, check whether the HTTP server has it
 817        my $cmd = server_exe('sws')." --version";
 818        my @sws = `$cmd`;
 819        $http_unix = 1 if($sws[0] =~ /unix/);
 820    }
 821
 822    open(my $manh, "-|", shell_quote($CURL) . " -M 2>&1");
 823    while(my $s = <$manh>) {
 824        if($s =~ /built-in manual was disabled at build-time/) {
 825            $feature{"manual"} = 0;
 826            last;
 827        }
 828        $feature{"manual"} = 1;
 829        last;
 830    }
 831    close($manh);
 832
 833    $feature{"unittest"} = 1;
 834    $feature{"nghttpx"} = !!$ENV{'NGHTTPX'};
 835    $feature{"nghttpx-h3"} = !!$nghttpx_h3;
 836
 837    # Use this as a proxy for any cryptographic authentication
 838    $feature{"crypto"} = $feature{"NTLM"} || $feature{"Kerberos"} || $feature{"SPNEGO"};
 839    $feature{"local-http"} = servers::localhttp();
 840    $feature{"codeset-utf8"} = is_utf8_supported();
 841    if($feature{"codeset-utf8"}) {
 842        $ENV{'CURL_TEST_HAVE_CODESET_UTF8'} = 1;
 843    }
 844
 845    # make each protocol an enabled "feature"
 846    for my $p (@protocols) {
 847        $feature{$p} = 1;
 848    }
 849    # 'socks' was once here but is now removed
 850
 851    if($torture && !$feature{"TrackMemory"}) {
 852        die "cannot run torture tests since curl was built without ".
 853            "TrackMemory feature (--enable-debug)";
 854    }
 855
 856    my $hostname=join(' ', runclientoutput("hostname"));
 857    chomp $hostname;
 858    my $hosttype=join(' ', runclientoutput("uname -a"));
 859    chomp $hosttype;
 860    my $hostos=$^O;
 861
 862    # display summary information about curl and the test host
 863    logmsg("********* System characteristics ******** \n",
 864           "* $curl\n",
 865           "* $libcurl\n",
 866           "* Protocols: $proto\n",
 867           "* Features: $feat\n",
 868           "* Disabled: $dis\n",
 869           "* Host: $hostname\n",
 870           "* System: $hosttype\n",
 871           "* OS: $hostos\n",
 872           "* Perl: $^V ($^X)\n",
 873           "* Args: $args\n");
 874    if($jobs) {
 875        # Only show if not the default for now
 876        logmsg "* Jobs: $jobs\n";
 877    }
 878
 879    my $env = sprintf("%s%s%s%s%s",
 880                      $valgrind ? "Valgrind " : "",
 881                      $run_duphandle ? "test-duphandle " : "",
 882                      $run_event_based ? "event-based " : "",
 883                      $nghttpx_h3 ? "nghttpx-h3 " : "",
 884                      $libtool ? "Libtool " : "");
 885    if($env) {
 886        logmsg "* Env: $env\n";
 887    }
 888    logmsg "* Seed: $randseed\n";
 889    if(system("diff $TESTDIR/DISABLED $TESTDIR/DISABLED 2>$dev_null") != 0) {
 890        logmsg "* diff: missing\n";
 891    }
 892    if($mintotal) {
 893        logmsg "* Min tests: $mintotal\n";
 894    }
 895}
 896
 897#######################################################################
 898# display information about server features
 899#
 900sub displayserverfeatures {
 901    logmsg sprintf("* Servers: %s", $stunnel ? "SSL " : "");
 902    logmsg sprintf("%s", $http_ipv6 ? "HTTP-IPv6 " : "");
 903    logmsg sprintf("%s", $http_unix ? "HTTP-unix " : "");
 904    logmsg sprintf("%s\n", $ftp_ipv6 ? "FTP-IPv6 " : "");
 905    logmsg "***************************************** \n";
 906}
 907
 908#######################################################################
 909# Provide time stamps for single test skipped events
 910#
 911sub timestampskippedevents {
 912    my $testnum = $_[0];
 913
 914    return if((not defined($testnum)) || ($testnum < 1));
 915
 916    if($timestats) {
 917
 918        if($timevrfyend{$testnum}) {
 919            return;
 920        }
 921        elsif($timesrvrlog{$testnum}) {
 922            $timevrfyend{$testnum} = $timesrvrlog{$testnum};
 923            return;
 924        }
 925        elsif($timetoolend{$testnum}) {
 926            $timevrfyend{$testnum} = $timetoolend{$testnum};
 927            $timesrvrlog{$testnum} = $timetoolend{$testnum};
 928        }
 929        elsif($timetoolini{$testnum}) {
 930            $timevrfyend{$testnum} = $timetoolini{$testnum};
 931            $timesrvrlog{$testnum} = $timetoolini{$testnum};
 932            $timetoolend{$testnum} = $timetoolini{$testnum};
 933        }
 934        elsif($timesrvrend{$testnum}) {
 935            $timevrfyend{$testnum} = $timesrvrend{$testnum};
 936            $timesrvrlog{$testnum} = $timesrvrend{$testnum};
 937            $timetoolend{$testnum} = $timesrvrend{$testnum};
 938            $timetoolini{$testnum} = $timesrvrend{$testnum};
 939        }
 940        elsif($timesrvrini{$testnum}) {
 941            $timevrfyend{$testnum} = $timesrvrini{$testnum};
 942            $timesrvrlog{$testnum} = $timesrvrini{$testnum};
 943            $timetoolend{$testnum} = $timesrvrini{$testnum};
 944            $timetoolini{$testnum} = $timesrvrini{$testnum};
 945            $timesrvrend{$testnum} = $timesrvrini{$testnum};
 946        }
 947        elsif($timeprepini{$testnum}) {
 948            $timevrfyend{$testnum} = $timeprepini{$testnum};
 949            $timesrvrlog{$testnum} = $timeprepini{$testnum};
 950            $timetoolend{$testnum} = $timeprepini{$testnum};
 951            $timetoolini{$testnum} = $timeprepini{$testnum};
 952            $timesrvrend{$testnum} = $timeprepini{$testnum};
 953            $timesrvrini{$testnum} = $timeprepini{$testnum};
 954        }
 955    }
 956}
 957
 958# Setup CI Test Run
 959sub citest_starttestrun {
 960    if(azure_check_environment()) {
 961        $AZURE_RUN_ID = azure_create_test_run($ACURL);
 962        logmsg "Azure Run ID: $AZURE_RUN_ID\n" if($verbose);
 963    }
 964    # Appveyor does not require anything here
 965}
 966
 967# Register the test case with the CI runner
 968sub citest_starttest {
 969    my $testnum = $_[0];
 970
 971    # get the name of the test early
 972    my $testname= (getpart("client", "name"))[0];
 973    chomp $testname;
 974
 975    # create test result in CI services
 976    if(azure_check_environment() && $AZURE_RUN_ID) {
 977        $AZURE_RESULT_ID = azure_create_test_result($ACURL, $AZURE_RUN_ID, $testnum, $testname);
 978    }
 979    elsif(appveyor_check_environment()) {
 980        appveyor_create_test_result($ACURL, $testnum, $testname);
 981    }
 982}
 983
 984# Submit the test case result with the CI runner
 985sub citest_finishtest {
 986    my ($testnum, $error) = @_;
 987    # update test result in CI services
 988    if(azure_check_environment() && $AZURE_RUN_ID && $AZURE_RESULT_ID) {
 989        $AZURE_RESULT_ID = azure_update_test_result($ACURL, $AZURE_RUN_ID, $AZURE_RESULT_ID, $testnum, $error,
 990                                                    $timeprepini{$testnum}, $timevrfyend{$testnum});
 991    }
 992    elsif(appveyor_check_environment()) {
 993        appveyor_update_test_result($ACURL, $testnum, $error, $timeprepini{$testnum}, $timevrfyend{$testnum});
 994    }
 995}
 996
 997# Complete CI test run
 998sub citest_finishtestrun {
 999    if(azure_check_environment() && $AZURE_RUN_ID) {
1000        $AZURE_RUN_ID = azure_update_test_run($ACURL, $AZURE_RUN_ID);
1001    }
1002    # Appveyor does not require anything here
1003}
1004
1005# add one set of test timings from the runner to global set
1006sub updatetesttimings {
1007    my ($testnum, %testtimings)=@_;
1008
1009    if(defined $testtimings{"timeprepini"}) {
1010        $timeprepini{$testnum} = $testtimings{"timeprepini"};
1011    }
1012    if(defined $testtimings{"timesrvrini"}) {
1013        $timesrvrini{$testnum} = $testtimings{"timesrvrini"};
1014    }
1015    if(defined $testtimings{"timesrvrend"}) {
1016        $timesrvrend{$testnum} = $testtimings{"timesrvrend"};
1017    }
1018    if(defined $testtimings{"timetoolini"}) {
1019        $timetoolini{$testnum} = $testtimings{"timetoolini"};
1020    }
1021    if(defined $testtimings{"timetoolend"}) {
1022        $timetoolend{$testnum} = $testtimings{"timetoolend"};
1023    }
1024    if(defined $testtimings{"timesrvrlog"}) {
1025        $timesrvrlog{$testnum} = $testtimings{"timesrvrlog"};
1026    }
1027}
1028
1029#######################################################################
1030# Return the log directory for the given test runner
1031sub getrunnernumlogdir {
1032    my $runnernum = $_[0];
1033    return $jobs > 1 ? "$LOGDIR/$runnernum" : $LOGDIR;
1034}
1035
1036#######################################################################
1037# Return the log directory for the given test runner ID
1038sub getrunnerlogdir {
1039    my $runnerid = $_[0];
1040    if($jobs <= 1) {
1041        return $LOGDIR;
1042    }
1043    # TODO: speed up this O(n) operation
1044    for my $runnernum (keys %runnerids) {
1045        if($runnerid eq $runnerids{$runnernum}) {
1046            return "$LOGDIR/$runnernum";
1047        }
1048    }
1049    die "Internal error: runner ID $runnerid not found";
1050}
1051
1052#######################################################################
1053# Verify that this test case should be run
1054sub singletest_shouldrun {
1055    my $testnum = $_[0];
1056    my $why;   # why the test will not be run
1057    my $errorreturncode = 1; # 1 means normal error, 2 means ignored error
1058    my @what;  # what features are needed
1059
1060    if($disttests !~ /test$testnum(\W|\z)/ ) {
1061        logmsg "Warning: test$testnum not present in tests/data/Makefile.am\n";
1062    }
1063    if($disabled{$testnum}) {
1064        if(!$run_disabled) {
1065            $why = "listed in DISABLED";
1066        }
1067        else {
1068            logmsg "Warning: test$testnum is explicitly disabled\n";
1069        }
1070    }
1071    if($ignored{$testnum}) {
1072        logmsg "Warning: test$testnum result is ignored\n";
1073        $errorreturncode = 2;
1074    }
1075
1076    if(loadtest("${TESTDIR}/test${testnum}", 1)) {
1077        if($verbose) {
1078            # this is not a test
1079            logmsg "RUN: $testnum does not look like a test case\n";
1080        }
1081        $why = "no test";
1082    }
1083    else {
1084        @what = getpart("client", "features");
1085    }
1086
1087    # We require a feature to be present
1088    for(@what) {
1089        my $f = $_;
1090        $f =~ s/\s//g;
1091
1092        if($f =~ /^([^!].*)$/) {
1093            if($feature{$1}) {
1094                next;
1095            }
1096
1097            $why = "curl lacks $1 support";
1098            last;
1099        }
1100    }
1101
1102    # We require a feature to not be present
1103    if(!$why) {
1104        for(@what) {
1105            my $f = $_;
1106            $f =~ s/\s//g;
1107
1108            if($f =~ /^!(.*)$/) {
1109                if(!$feature{$1}) {
1110                    next;
1111                }
1112            }
1113            else {
1114                next;
1115            }
1116
1117            $why = "curl has $1 support";
1118            last;
1119        }
1120    }
1121
1122    my @info_keywords;
1123    if(!$why) {
1124        @info_keywords = getpart("info", "keywords");
1125
1126        if(!$info_keywords[0]) {
1127            $why = "missing the <keywords> section!";
1128        }
1129        # Only evaluate keywords if the section is present.
1130        else {
1131            # Prefix features with "feat:" and add to keywords list.
1132            push @info_keywords, map { "feat:" . lc($_) } getpart("client", "features");
1133
1134            my $match;
1135            for my $k (@info_keywords) {
1136                chomp $k;
1137                if($disabled_keywords{lc($k)}) {
1138                    if($k =~ /^feat:/) {
1139                        $why = "disabled by feature";
1140                    }
1141                    else {
1142                        $why = "disabled by keyword";
1143                    }
1144                }
1145                elsif($enabled_keywords{lc($k)}) {
1146                    $match = 1;
1147                }
1148                if($ignored_keywords{lc($k)}) {
1149                    logmsg "Warning: test$testnum result is ignored due to $k\n";
1150                    $errorreturncode = 2;
1151                }
1152            }
1153
1154            if(!$why && !$match && %enabled_keywords) {
1155                if(grep { /^feat:/ } keys %enabled_keywords) {
1156                    $why = "disabled by missing feature";
1157                }
1158                else {
1159                    $why = "disabled by missing keyword";
1160                }
1161            }
1162        }
1163    }
1164
1165    if(!$why && defined $custom_skip_reasons{test}{$testnum}) {
1166        $why = $custom_skip_reasons{test}{$testnum};
1167    }
1168
1169    if(!$why && defined $custom_skip_reasons{tool}) {
1170        foreach my $tool (getpart("client", "tool")) {
1171            foreach my $tool_skip_pattern (keys %{$custom_skip_reasons{tool}}) {
1172                if($tool =~ /$tool_skip_pattern/i) {
1173                    $why = $custom_skip_reasons{tool}{$tool_skip_pattern};
1174                }
1175            }
1176        }
1177    }
1178
1179    if(!$why && defined $custom_skip_reasons{keyword}) {
1180        foreach my $keyword (@info_keywords) {
1181            foreach my $keyword_skip_pattern (keys %{$custom_skip_reasons{keyword}}) {
1182                if($keyword =~ /$keyword_skip_pattern/i) {
1183                    $why = $custom_skip_reasons{keyword}{$keyword_skip_pattern};
1184                }
1185            }
1186        }
1187    }
1188
1189    if($why && checktest("${TESTDIR}/test${testnum}")) {
1190        logmsg "Warning: issue(s) found in test data: ${TESTDIR}/test${testnum}\n";
1191    }
1192
1193    return ($why, $errorreturncode);
1194}
1195
1196#######################################################################
1197# Print the test name and count tests
1198sub singletest_count {
1199    my ($testnum, $why) = @_;
1200
1201    if($why && !$listonly) {
1202        # there is a problem, count it as "skipped"
1203        $skipped{$why}++;
1204        $teststat[$testnum]=$why; # store reason for this test case
1205
1206        if(!$short) {
1207            if($skipped{$why} <= 3) {
1208                # show only the first three skips for each reason
1209                logmsg sprintf("test %04d SKIPPED: $why\n", $testnum);
1210            }
1211        }
1212
1213        timestampskippedevents($testnum);
1214        return -1;
1215    }
1216
1217    # At this point we have committed to run this test
1218    logmsg sprintf("test %04d...", $testnum) if(!$automakestyle);
1219
1220    # name of the test
1221    my $testname= (getpart("client", "name"))[0];
1222    chomp $testname;
1223    logmsg "[$testname]\n" if(!$short);
1224
1225    if($listonly) {
1226        timestampskippedevents($testnum);
1227    }
1228    return 0;
1229}
1230
1231# Make sure all line endings in the array are the same: CRLF
1232sub normalize_text {
1233    my ($ref) = @_;
1234    s/\r\n/\n/g for @$ref;
1235    s/\n/\r\n/g for @$ref;
1236}
1237
1238#######################################################################
1239# Verify test succeeded
1240sub singletest_check {
1241    my ($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind)=@_;
1242
1243    # Skip all the verification on torture tests
1244    if($torture) {
1245        # timestamp test result verification end
1246        $timevrfyend{$testnum} = Time::HiRes::time();
1247        return -2;
1248    }
1249
1250    my $logdir = getrunnerlogdir($runnerid);
1251    my @err = getpart("verify", "errorcode");
1252    my $errorcode = $err[0] || "0";
1253    my $ok="";
1254    my $res;
1255    chomp $errorcode;
1256    my $testname= (getpart("client", "name"))[0];
1257    chomp $testname;
1258    # what parts to cut off from stdout/stderr
1259    my @stripfile = getpart("verify", "stripfile");
1260
1261    my @validstdout = getpart("verify", "stdout");
1262    # get all attributes
1263    my %hash = getpartattr("verify", "stdout");
1264
1265    my $loadfile = $hash{'loadfile'};
1266    if($loadfile) {
1267        open(my $tmp, "<", $loadfile) || die "Cannot open file $loadfile: $!";
1268        @validstdout = <$tmp>;
1269        close($tmp);
1270
1271        # Enforce LF newlines on load
1272        s/\r\n/\n/g for @validstdout;
1273    }
1274
1275    if(@validstdout) {
1276        $validstdout[0] =~ s/^%EMPTY[\r\n]*//;
1277
1278        # verify redirected stdout
1279        my @actual = loadarray(stdoutfilename($logdir, $testnum));
1280
1281        foreach my $strip (@stripfile) {
1282            chomp $strip;
1283            my @newgen;
1284            for(@actual) {
1285                eval $strip;
1286                if($_) {
1287                    push @newgen, $_;
1288                }
1289            }
1290            # this is to get rid of array entries that vanished (zero
1291            # length) because of replacements
1292            @actual = @newgen;
1293        }
1294
1295        # get the mode attribute
1296        my $filemode=$hash{'mode'};
1297        if($filemode && ($filemode eq "text")) {
1298            normalize_text(\@validstdout);
1299            normalize_text(\@actual);
1300        }
1301
1302        if($hash{'nonewline'}) {
1303            # Yes, we must cut off the final newline from the final line
1304            # of the protocol data
1305            chomp($validstdout[-1]);
1306        }
1307
1308        if($hash{'crlf'}) {
1309            if($hash{'crlf'} eq "headers") {
1310                subnewlines(0, \$_) for @validstdout;
1311            }
1312            else {
1313                subnewlines(1, \$_) for @validstdout;
1314            }
1315        }
1316
1317        $res = compare($runnerid, $testnum, $testname, "stdout", \@actual, \@validstdout);
1318        if($res) {
1319            return -1;
1320        }
1321        $ok .= "s";
1322    }
1323    else {
1324        $ok .= "-"; # stdout not checked
1325    }
1326
1327    my @validstderr = getpart("verify", "stderr");
1328    if(@validstderr) {
1329        # verify redirected stderr
1330        my @actual = loadarray(stderrfilename($logdir, $testnum));
1331
1332        foreach my $strip (@stripfile) {
1333            chomp $strip;
1334            my @newgen;
1335            for(@actual) {
1336                eval $strip;
1337                if($_) {
1338                    push @newgen, $_;
1339                }
1340            }
1341            # this is to get rid of array entries that vanished (zero
1342            # length) because of replacements
1343            @actual = @newgen;
1344        }
1345
1346        # get all attributes
1347        my %hash = getpartattr("verify", "stderr");
1348
1349        # get the mode attribute
1350        my $filemode=$hash{'mode'};
1351        if($filemode && ($filemode eq "text")) {
1352            normalize_text(\@validstderr);
1353            normalize_text(\@actual);
1354        }
1355        if($filemode && ($filemode eq "warn")) {
1356            for(@validstderr) {
1357                s/Warning: //;
1358                s/\r//;
1359                s/\n/ /;
1360            }
1361            for(@actual) {
1362                s/Warning: //;
1363                s/\r//;
1364                s/\n/ /;
1365            }
1366            my $v = join(@validstderr, "");
1367            my $a = join(@actual, "");
1368            @validstderr = $v;
1369            @actual = $a;
1370        }
1371
1372        if($hash{'nonewline'}) {
1373            # Yes, we must cut off the final newline from the final line
1374            # of the protocol data
1375            chomp($validstderr[-1]);
1376        }
1377
1378        if($hash{'crlf'}) {
1379            if($hash{'crlf'} eq "headers") {
1380                subnewlines(0, \$_) for @validstderr;
1381            }
1382            else {
1383                subnewlines(1, \$_) for @validstderr;
1384            }
1385        }
1386
1387        $res = compare($runnerid, $testnum, $testname, "stderr", \@actual, \@validstderr);
1388        if($res) {
1389            return -1;
1390        }
1391        $ok .= "r";
1392    }
1393    else {
1394        $ok .= "-"; # stderr not checked
1395    }
1396
1397    # what to cut off from the live protocol sent by curl
1398    my @strip = getpart("verify", "strip");
1399
1400    # what parts to cut off from the protocol & upload
1401    my @strippart = getpart("verify", "strippart");
1402
1403    # this is the valid protocol blurb curl should generate
1404    my @protocol= getpart("verify", "protocol");
1405    if(@protocol) {
1406        # Verify the sent request
1407        my @out = loadarray("$logdir/$SERVERIN");
1408
1409        # check if there is any attributes on the verify/protocol section
1410        my %hash = getpartattr("verify", "protocol");
1411
1412        if($hash{'nonewline'}) {
1413            # Yes, we must cut off the final newline from the final line
1414            # of the protocol data
1415            chomp($protocol[-1]);
1416        }
1417
1418        for(@strip) {
1419            # strip off all lines that match the patterns from both arrays
1420            chomp $_;
1421            @out = striparray( $_, \@out);
1422            @protocol= striparray( $_, \@protocol);
1423        }
1424
1425        for my $strip (@strippart) {
1426            chomp $strip;
1427            for(@out) {
1428                eval $strip;
1429            }
1430        }
1431
1432        if($hash{'crlf'}) {
1433            if($hash{'crlf'} eq "headers") {
1434                subnewlines(0, \$_) for @protocol;
1435            }
1436            else {
1437                subnewlines(1, \$_) for @protocol;
1438            }
1439        }
1440
1441        if((!$out[0] || ($out[0] eq "")) && $protocol[0]) {
1442            logmsg "\n $testnum: protocol FAILED!\n".
1443                " There was no content at all in the file $logdir/$SERVERIN.\n".
1444                " Server glitch? Total curl failure? Returned: $cmdres\n";
1445            # timestamp test result verification end
1446            $timevrfyend{$testnum} = Time::HiRes::time();
1447            return -1;
1448        }
1449
1450        $res = compare($runnerid, $testnum, $testname, "protocol", \@out, \@protocol);
1451        if($res) {
1452            return -1;
1453        }
1454
1455        $ok .= "p";
1456    }
1457    else {
1458        $ok .= "-"; # protocol not checked
1459    }
1460
1461    my %replyattr = getpartattr("reply", "data");
1462    my @reply;
1463    if(partexists("reply", "datacheck")) {
1464        for my $partsuffix (('', '1', '2', '3', '4')) {
1465            my @replycheckpart = getpart("reply", "datacheck".$partsuffix);
1466            if(@replycheckpart) {
1467                my %replycheckpartattr = getpartattr("reply", "datacheck".$partsuffix);
1468                # get the mode attribute
1469                my $filemode=$replycheckpartattr{'mode'};
1470                if($filemode && ($filemode eq "text")) {
1471                    normalize_text(\@replycheckpart);
1472                }
1473                if($replycheckpartattr{'nonewline'}) {
1474                    # Yes, we must cut off the final newline from the final line
1475                    # of the datacheck
1476                    chomp($replycheckpart[-1]);
1477                }
1478                if($replycheckpartattr{'crlf'}) {
1479                    if($replycheckpartattr{'crlf'} eq "headers") {
1480                        subnewlines(0, \$_) for @replycheckpart;
1481                    }
1482                    else {
1483                        subnewlines(1, \$_) for @replycheckpart;
1484                    }
1485                }
1486                push(@reply, @replycheckpart);
1487            }
1488        }
1489    }
1490    else {
1491        # check against the data section
1492        @reply = getpart("reply", "data");
1493        if(@reply) {
1494            if($replyattr{'nonewline'}) {
1495                # cut off the final newline from the final line of the data
1496                chomp($reply[-1]);
1497            }
1498        }
1499        # get the mode attribute
1500        my $filemode=$replyattr{'mode'};
1501        if($filemode && ($filemode eq "text")) {
1502            normalize_text(\@reply);
1503        }
1504        if($replyattr{'crlf'}) {
1505            if($replyattr{'crlf'} eq "headers") {
1506                subnewlines(0, \$_) for @reply;
1507            }
1508            else {
1509                subnewlines(1, \$_) for @reply;
1510            }
1511        }
1512    }
1513
1514    if(!$replyattr{'nocheck'} && (@reply || $replyattr{'sendzero'})) {
1515        # verify the received data
1516        my @out = loadarray($CURLOUT);
1517
1518        # get the mode attribute
1519        my $filemode=$replyattr{'mode'};
1520        if($filemode && ($filemode eq "text")) {
1521            normalize_text(\@out);
1522        }
1523        $res = compare($runnerid, $testnum, $testname, "data", \@out, \@reply);
1524        if($res) {
1525            return -1;
1526        }
1527        $ok .= "d";
1528    }
1529    else {
1530        $ok .= "-"; # data not checked
1531    }
1532
1533    # if this section exists, we verify upload
1534    my @upload = getpart("verify", "upload");
1535    if(@upload) {
1536        my %hash = getpartattr("verify", "upload");
1537        if($hash{'nonewline'}) {
1538            # cut off the final newline from the final line of the upload data
1539            chomp($upload[-1]);
1540        }
1541        for my $line (@upload) {
1542            subbase64(\$line);
1543            subsha256base64file(\$line);
1544            substrippemfile(\$line);
1545        }
1546
1547        # verify uploaded data
1548        my @out = loadarray("$logdir/upload.$testnum");
1549        for my $strip (@strippart) {
1550            chomp $strip;
1551            for(@out) {
1552                eval $strip;
1553            }
1554        }
1555        if($hash{'crlf'}) {
1556            subnewlines(1, \$_) for @upload;
1557        }
1558        if($hash{'nonewline'}) {
1559            # Yes, we must cut off the final newline from the final line
1560            # of the upload data
1561            chomp($upload[-1]);
1562        }
1563
1564        $res = compare($runnerid, $testnum, $testname, "upload", \@out, \@upload);
1565        if($res) {
1566            return -1;
1567        }
1568        $ok .= "u";
1569    }
1570    else {
1571        $ok .= "-"; # upload not checked
1572    }
1573
1574    # this is the valid protocol blurb curl should generate to a proxy
1575    my @proxyprot = getpart("verify", "proxy");
1576    if(@proxyprot) {
1577        # Verify the sent proxy request
1578        # check if there is any attributes on the verify/protocol section
1579        my %hash = getpartattr("verify", "proxy");
1580
1581        if($hash{'nonewline'}) {
1582            # Yes, we must cut off the final newline from the final line
1583            # of the protocol data
1584            chomp($proxyprot[-1]);
1585        }
1586
1587        my @out = loadarray("$logdir/$PROXYIN");
1588        for(@strip) {
1589            # strip off all lines that match the patterns from both arrays
1590            chomp $_;
1591            @out = striparray( $_, \@out);
1592            @proxyprot= striparray( $_, \@proxyprot);
1593        }
1594
1595        for my $strip (@strippart) {
1596            chomp $strip;
1597            for(@out) {
1598                eval $strip;
1599            }
1600        }
1601
1602        if($hash{'crlf'}) {
1603            if($hash{'crlf'} eq "headers") {
1604                subnewlines(0, \$_) for @proxyprot;
1605            }
1606            else {
1607                subnewlines(1, \$_) for @proxyprot;
1608            }
1609        }
1610
1611        $res = compare($runnerid, $testnum, $testname, "proxy", \@out, \@proxyprot);
1612        if($res) {
1613            return -1;
1614        }
1615
1616        $ok .= "P";
1617    }
1618    else {
1619        $ok .= "-"; # proxy not checked
1620    }
1621
1622    my $outputok;
1623    for my $partsuffix (('', '1', '2', '3', '4')) {
1624        my @outfile=getpart("verify", "file".$partsuffix);
1625        if(@outfile || partexists("verify", "file".$partsuffix) ) {
1626            # we are supposed to verify a dynamically generated file!
1627            my %hash = getpartattr("verify", "file".$partsuffix);
1628
1629            my $filename=$hash{'name'};
1630            if(!$filename) {
1631                logmsg " $testnum: IGNORED: section verify=>file$partsuffix ".
1632                       "has no name attribute\n";
1633                if(runnerac_stopservers($runnerid)) {
1634                    logmsg "ERROR: runner $runnerid seems to have died\n";
1635                } else {
1636
1637                    # TODO: this is a blocking call that will stall the controller,
1638                    if($verbose) {
1639                        logmsg "WARNING: blocking call in async function\n";
1640                    }
1641                    # but this error condition should never happen except during
1642                    # development.
1643                    my ($rid, $unexpected, $logs) = runnerar($runnerid);
1644                    if(!$rid) {
1645                        logmsg "ERROR: runner $runnerid seems to have died\n";
1646                    } else {
1647                        logmsg $logs;
1648                    }
1649                }
1650                # timestamp test result verification end
1651                $timevrfyend{$testnum} = Time::HiRes::time();
1652                return -1;
1653            }
1654            my @generated=loadarray($filename);
1655
1656            # what parts to cut off from the file
1657            my @stripfilepar = getpart("verify", "stripfile".$partsuffix);
1658
1659            my $filemode=$hash{'mode'};
1660            if($filemode && ($filemode eq "text")) {
1661                normalize_text(\@outfile);
1662                normalize_text(\@generated);
1663            }
1664            if($hash{'crlf'}) {
1665                if($hash{'crlf'} eq "headers") {
1666                    subnewlines(0, \$_) for @outfile;
1667                }
1668                else {
1669                    subnewlines(1, \$_) for @outfile;
1670                }
1671            }
1672
1673            for my $strip (@stripfilepar) {
1674                chomp $strip;
1675                my @newgen;
1676                for(@generated) {
1677                    eval $strip;
1678                    if($_) {
1679                        push @newgen, $_;
1680                    }
1681                }
1682                # this is to get rid of array entries that vanished (zero
1683                # length) because of replacements
1684                @generated = @newgen;
1685            }
1686
1687            if($hash{'nonewline'}) {
1688                # cut off the final newline from the final line of the
1689                # output data
1690                chomp($outfile[-1]);
1691            }
1692
1693            $res = compare($runnerid, $testnum, $testname, "output ($filename)",
1694                           \@generated, \@outfile);
1695            if($res) {
1696                return -1;
1697            }
1698
1699            $outputok = 1; # output checked
1700        }
1701    }
1702    $ok .= ($outputok) ? "o" : "-"; # output checked or not
1703
1704    # verify SOCKS proxy details
1705    my @socksprot = getpart("verify", "socks");
1706    if(@socksprot) {
1707        # Verify the sent SOCKS proxy details
1708        my @out = loadarray("$logdir/$SOCKSIN");
1709        $res = compare($runnerid, $testnum, $testname, "socks", \@out, \@socksprot);
1710        if($res) {
1711            return -1;
1712        }
1713    }
1714
1715    my @dnsd = getpart("verify", "dns");
1716    if(@dnsd) {
1717        # we are supposed to verify a dynamically generated file!
1718        my %hash = getpartattr("verify", "dns");
1719        my $hostname=$hash{'host'};
1720
1721        # Verify the sent DNS requests
1722        my @out = loadarray("$logdir/dnsd.input");
1723        my @sverify = sort @dnsd;
1724        my @sout = sort @out;
1725
1726        if($hostname) {
1727            # when a hostname is set, we filter out requests to just this
1728            # pattern
1729            @sout = grep {/$hostname/} @sout;
1730        }
1731
1732        $res = compare($runnerid, $testnum, $testname, "DNS", \@sout, \@sverify);
1733        if($res) {
1734            return -1;
1735        }
1736    }
1737
1738    # accept multiple comma-separated error codes
1739    my @splerr = split(/ *, */, $errorcode);
1740    my $errok;
1741    foreach my $e (@splerr) {
1742        if($e == $cmdres) {
1743            # a fine error code
1744            $errok = 1;
1745            last;
1746        }
1747    }
1748
1749    if($errok) {
1750        $ok .= "e";
1751    }
1752    else {
1753        if(!$short) {
1754            logmsg sprintf("\n%s returned $cmdres, when expecting %s\n",
1755                           (!$tool) ? "curl" : $tool, $errorcode);
1756        }
1757        logmsg " $testnum: exit FAILED\n";
1758        # timestamp test result verification end
1759        $timevrfyend{$testnum} = Time::HiRes::time();
1760        return -1;
1761    }
1762
1763    if($feature{"TrackMemory"}) {
1764        if(! -f "$logdir/$MEMDUMP") {
1765            my %cmdhash = getpartattr("client", "command");
1766            my $cmdtype = $cmdhash{'type'} || "default";
1767            if($cmdhash{'option'} && $cmdhash{'option'} !~ /no-memdebug/) {
1768                logmsg "\n** ALERT! memory tracking with no output file?\n"
1769                    if($cmdtype ne "perl");
1770            }
1771            $ok .= "-"; # problem with memory checking
1772        }
1773        else {
1774            my @memdata = memanalyze("$logdir/$MEMDUMP", 0, 0, 0);
1775            my $leak=0;
1776            for(@memdata) {
1777                if($_ ne "") {
1778                    # well it could be other memory problems as well, but
1779                    # we call it leak for short here
1780                    $leak=1;
1781                }
1782            }
1783            if($leak) {
1784                logmsg "\n** MEMORY FAILURE\n";
1785                logmsg @memdata;
1786                logmsg " $testnum: memory FAILED\n";
1787                # timestamp test result verification end
1788                $timevrfyend{$testnum} = Time::HiRes::time();
1789                return -1;
1790            }
1791            else {
1792                $ok .= "m";
1793            }
1794            my @more = memanalyze("$logdir/$MEMDUMP", 1, 0, 0);
1795            my $allocs = 0;
1796            my $max = 0;
1797            for(@more) {
1798                if(/^Allocations: (\d+)/) {
1799                    $allocs = $1;
1800                }
1801                elsif(/^Maximum allocated: (\d+)/) {
1802                    $max = $1;
1803                }
1804            }
1805            my @limits = getpart("verify", "limits");
1806            my $lim_allocs = 1000; # high default values
1807            my $lim_max = 1000000;
1808            for(@limits) {
1809                if(/^Allocations: (\d+)/i) {
1810                    $lim_allocs = $1;
1811                }
1812                elsif(/^Maximum allocated: (\d+)/i) {
1813                    $lim_max = $1;
1814                }
1815            }
1816            logmsg "did $allocs allocations, $lim_allocs allowed\n"
1817                if($verbose);
1818
1819            logmsg "allocated $max maximum, $lim_max allowed\n"
1820                if($verbose);
1821
1822            if($allocs > $lim_allocs) {
1823                logmsg "\n** TOO MANY ALLOCS\n";
1824                logmsg "$lim_allocs allocations allowed, did $allocs\n";
1825                logmsg " $testnum: allocs FAILED\n";
1826                # timestamp test result verification end
1827                $timevrfyend{$testnum} = Time::HiRes::time();
1828                return -1;
1829            }
1830            if($max > $lim_max) {
1831                logmsg "\n** TOO MUCH TOTAL ALLOCATION\n";
1832                logmsg "$lim_max maximum allocation allowed, did $max\n";
1833                logmsg " $testnum: allocsize FAILED\n";
1834                # timestamp test result verification end
1835                $timevrfyend{$testnum} = Time::HiRes::time();
1836                return -1;
1837            }
1838        }
1839    }
1840    else {
1841        $ok .= "-"; # memory not checked
1842    }
1843
1844    my @notexists = getpart("verify", "notexists");
1845    if(@notexists) {
1846        # a list of directory entries that must not exist
1847        my $err;
1848        while(@notexists) {
1849            my $fname = shift @notexists;
1850            chomp $fname;
1851            if(-e $fname) {
1852                logmsg "ERROR: Found '$fname' when not supposed to exist.\n";
1853                $err++;
1854            }
1855            elsif($verbose) {
1856                logmsg "Found '$fname' confirmed to not exist.\n";
1857            }
1858        }
1859        if($err) {
1860            return -1;
1861        }
1862    }
1863    if($valgrind) {
1864        if($usedvalgrind) {
1865            if(!opendir(DIR, $logdir)) {
1866                logmsg "ERROR: unable to read $logdir\n";
1867                # timestamp test result verification end
1868                $timevrfyend{$testnum} = Time::HiRes::time();
1869                return -1;
1870            }
1871            my @files = readdir(DIR);
1872            closedir(DIR);
1873            my $vgfile;
1874            foreach my $file (@files) {
1875                if($file =~ /^valgrind$testnum(\..*|)$/) {
1876                    $vgfile = $file;
1877                    last;
1878                }
1879            }
1880            if(!$vgfile) {
1881                logmsg "ERROR: valgrind log file missing for test $testnum\n";
1882                # timestamp test result verification end
1883                $timevrfyend{$testnum} = Time::HiRes::time();
1884                return -1;
1885            }
1886            my @e = valgrindparse("$logdir/$vgfile");
1887            if(@e && $e[0]) {
1888                if($automakestyle) {
1889                    logmsg "FAIL: $testnum - $testname - valgrind\n";
1890                }
1891                else {
1892                    logmsg " valgrind ERROR ";
1893                    logmsg @e;
1894                }
1895                # timestamp test result verification end
1896                $timevrfyend{$testnum} = Time::HiRes::time();
1897                return -1;
1898            }
1899            $ok .= "v";
1900        }
1901        else {
1902            if($verbose) {
1903                logmsg " valgrind SKIPPED\n";
1904            }
1905            $ok .= "-"; # skipped
1906        }
1907    }
1908    else {
1909        $ok .= "-"; # valgrind not checked
1910    }
1911    # add 'E' for event-based
1912    $ok .= $run_event_based ? "E" : "-";
1913
1914    logmsg "$ok " if(!$short);
1915
1916    # timestamp test result verification end
1917    $timevrfyend{$testnum} = Time::HiRes::time();
1918
1919    return 0;
1920}
1921
1922#######################################################################
1923# Report a successful test
1924sub singletest_success {
1925    my ($testnum, $count, $total, $errorreturncode)=@_;
1926
1927    my $sofar= time()-$start;
1928    my $esttotal = $sofar/$count * $total;
1929    my $estleft = $esttotal - $sofar;
1930    my $timeleft=sprintf("remaining: %02d:%02d",
1931                     $estleft/60,
1932                     $estleft%60);
1933    my $took = $timevrfyend{$testnum} - $timeprepini{$testnum};
1934    my $duration = sprintf("duration: %02d:%02d",
1935                           $sofar/60, $sofar%60);
1936    if(!$automakestyle) {
1937        logmsg sprintf("OK (%-3d out of %-3d, %s, took %.3fs, %s)\n",
1938                       $count, $total, $timeleft, $took, $duration);
1939    }
1940    else {
1941        my $testname= (getpart("client", "name"))[0];
1942        chomp $testname;
1943        logmsg "PASS: $testnum - $testname\n";
1944    }
1945
1946    if($errorreturncode==2) {
1947        # ignored test success
1948        $passedign .= "$testnum ";
1949        logmsg "Warning: test$testnum result is ignored, but passed!\n";
1950    }
1951}
1952
1953#######################################################################
1954# Run a single specified test case
1955# This is structured as a state machine which changes state after an
1956# asynchronous call is made that awaits a response. The function returns with
1957# an error code and a flag that indicates if the state machine has completed,
1958# which means (if not) the function must be called again once the response has
1959# arrived.
1960#
1961sub singletest {
1962    my ($runnerid, $testnum, $count, $total)=@_;
1963
1964    # start buffering logmsg; stop it on return
1965    logmsg_bufferfortest($runnerid);
1966    if(!exists $singletest_state{$runnerid}) {
1967        # First time in singletest() for this test
1968        $singletest_state{$runnerid} = ST_INIT;
1969    }
1970
1971    if($singletest_state{$runnerid} == ST_INIT) {
1972        my $logdir = getrunnerlogdir($runnerid);
1973        # first, remove all lingering log & lock files
1974        if(!cleardir($logdir)) {
1975            #logmsg "Warning: $runnerid: cleardir($logdir) failed\n";
1976        }
1977        if(!cleardir("$logdir/$LOCKDIR")) {
1978            logmsg "Warning: $runnerid: cleardir($logdir/$LOCKDIR) failed\n";
1979        }
1980
1981        $singletest_state{$runnerid} = ST_INITED;
1982        # Recursively call the state machine again because there is no
1983        # event expected that would otherwise trigger a new call.
1984        return singletest(@_);
1985
1986    } elsif($singletest_state{$runnerid} == ST_INITED) {
1987        ###################################################################
1988        # Restore environment variables that were modified in a previous run.
1989        # Test definition may instruct to (un)set environment vars.
1990        # This is done this early so that leftover variables do not affect
1991        # starting servers or CI registration.
1992        # restore_test_env(1);
1993
1994        ###################################################################
1995        # Load test file so CI registration can get the right data before the
1996        # runner is called
1997        loadtest("${TESTDIR}/test${testnum}", 1);
1998
1999        ###################################################################
2000        # Register the test case with the CI environment
2001        citest_starttest($testnum);
2002
2003        if(runnerac_test_preprocess($runnerid, $testnum)) {
2004            logmsg "ERROR: runner $runnerid seems to have died\n";
2005            $singletest_state{$runnerid} = ST_INIT;
2006            return (-1, 0);
2007        }
2008        $singletest_state{$runnerid} = ST_PREPROCESS;
2009
2010    } elsif($singletest_state{$runnerid} == ST_PREPROCESS) {
2011        my ($rid, $why, $error, $logs, $testtimings) = runnerar($runnerid);
2012        if(!$rid) {
2013            logmsg "ERROR: runner $runnerid seems to have died\n";
2014            $singletest_state{$runnerid} = ST_INIT;
2015            return (-1, 0);
2016        }
2017        logmsg $logs;
2018        updatetesttimings($testnum, %$testtimings);
2019        if($error == -2) {
2020            if($postmortem) {
2021                # Error indicates an actual problem starting the server, so
2022                # display the server logs
2023                displaylogs($rid, $testnum);
2024            }
2025        }
2026
2027        #######################################################################
2028        # Load test file for this test number
2029        my $logdir = getrunnerlogdir($runnerid);
2030        loadtest("${logdir}/test${testnum}");
2031
2032        #######################################################################
2033        # Print the test name and count tests
2034        $error = singletest_count($testnum, $why);
2035        if($error) {
2036            # Submit the test case result with the CI environment
2037            citest_finishtest($testnum, $error);
2038            $singletest_state{$runnerid} = ST_INIT;
2039            logmsg singletest_dumplogs();
2040            return ($error, 0);
2041        }
2042
2043        #######################################################################
2044        # Execute this test number
2045        my $cmdres;
2046        my $CURLOUT;
2047        my $tool;
2048        my $usedvalgrind;
2049        if(runnerac_test_run($runnerid, $testnum)) {
2050            logmsg "ERROR: runner $runnerid seems to have died\n";
2051            $singletest_state{$runnerid} = ST_INIT;
2052            return (-1, 0);
2053        }
2054        $singletest_state{$runnerid} = ST_RUN;
2055
2056    } elsif($singletest_state{$runnerid} == ST_RUN) {
2057        my ($rid, $error, $logs, $testtimings, $cmdres, $CURLOUT, $tool, $usedvalgrind) = runnerar($runnerid);
2058        if(!$rid) {
2059            logmsg "ERROR: runner $runnerid seems to have died\n";
2060            $singletest_state{$runnerid} = ST_INIT;
2061            return (-1, 0);
2062        }
2063        logmsg $logs;
2064        updatetesttimings($testnum, %$testtimings);
2065        if($error == -1) {
2066            # no further verification will occur
2067            $timevrfyend{$testnum} = Time::HiRes::time();
2068            my $err = ignoreresultcode($testnum);
2069            # Submit the test case result with the CI environment
2070            citest_finishtest($testnum, $err);
2071            $singletest_state{$runnerid} = ST_INIT;
2072            logmsg singletest_dumplogs();
2073            # return a test failure, either to be reported or to be ignored
2074            return ($err, 0);
2075        }
2076        elsif($error == -2) {
2077            # fill in the missing timings on error
2078            timestampskippedevents($testnum);
2079            # Submit the test case result with the CI environment
2080            citest_finishtest($testnum, $error);
2081            $singletest_state{$runnerid} = ST_INIT;
2082            logmsg singletest_dumplogs();
2083            return ($error, 0);
2084        }
2085        elsif($error > 0) {
2086            # no further verification will occur
2087            $timevrfyend{$testnum} = Time::HiRes::time();
2088            # Submit the test case result with the CI environment
2089            citest_finishtest($testnum, $error);
2090            $singletest_state{$runnerid} = ST_INIT;
2091            logmsg singletest_dumplogs();
2092            return ($error, 0);
2093        }
2094
2095        #######################################################################
2096        # Verify that the test succeeded
2097        #
2098        # Load test file for this test number
2099        my $logdir = getrunnerlogdir($runnerid);
2100        loadtest("${logdir}/test${testnum}");
2101        readtestkeywords();
2102
2103        $error = singletest_check($runnerid, $testnum, $cmdres, $CURLOUT, $tool, $usedvalgrind);
2104        if($error == -1) {
2105            my $err = ignoreresultcode($testnum);
2106            # Submit the test case result with the CI environment
2107            citest_finishtest($testnum, $err);
2108            $singletest_state{$runnerid} = ST_INIT;
2109            logmsg singletest_dumplogs();
2110            # return a test failure, either to be reported or to be ignored
2111            return ($err, 0);
2112        }
2113        elsif($error == -2) {
2114            # torture test; there is no verification, so the run result holds the
2115            # test success code
2116            # Submit the test case result with the CI environment
2117            citest_finishtest($testnum, $cmdres);
2118            $singletest_state{$runnerid} = ST_INIT;
2119            logmsg singletest_dumplogs();
2120            return ($cmdres, 0);
2121        }
2122
2123        #######################################################################
2124        # Report a successful test
2125        singletest_success($testnum, $count, $total, ignoreresultcode($testnum));
2126
2127        # Submit the test case result with the CI environment
2128        citest_finishtest($testnum, 0);
2129        $singletest_state{$runnerid} = ST_INIT;
2130
2131        logmsg singletest_dumplogs();
2132        return (0, 0);  # state machine is finished
2133    }
2134    singletest_unbufferlogs();
2135    return (0, 1);  # state machine must be called again on event
2136}
2137
2138#######################################################################
2139# runtimestats displays test-suite run time statistics
2140#
2141sub runtimestats {
2142    my $lasttest = $_[0];
2143
2144    return if(not $timestats);
2145
2146    logmsg "::group::Run Time Stats\n";
2147
2148    logmsg "\nTest suite total running time breakdown per task...\n\n";
2149
2150    my @timesrvr;
2151    my @timeprep;
2152    my @timetool;
2153    my @timelock;
2154    my @timevrfy;
2155    my @timetest;
2156    my $timesrvrtot = 0.0;
2157    my $timepreptot = 0.0;
2158    my $timetooltot = 0.0;
2159    my $timelocktot = 0.0;
2160    my $timevrfytot = 0.0;
2161    my $timetesttot = 0.0;
2162    my $counter;
2163
2164    for my $testnum (1 .. $lasttest) {
2165        if($timesrvrini{$testnum}) {
2166            $timesrvrtot += $timesrvrend{$testnum} - $timesrvrini{$testnum};
2167            $timepreptot +=
2168                (($timetoolini{$testnum} - $timeprepini{$testnum}) -
2169                 ($timesrvrend{$testnum} - $timesrvrini{$testnum}));
2170            $timetooltot += $timetoolend{$testnum} - $timetoolini{$testnum};
2171            $timelocktot += $timesrvrlog{$testnum} - $timetoolend{$testnum};
2172            $timevrfytot += $timevrfyend{$testnum} - $timesrvrlog{$testnum};
2173            $timetesttot += $timevrfyend{$testnum} - $timeprepini{$testnum};
2174            push @timesrvr, sprintf("%06.3f  %04d",
2175                $timesrvrend{$testnum} - $timesrvrini{$testnum}, $testnum);
2176            push @timeprep, sprintf("%06.3f  %04d",
2177                ($timetoolini{$testnum} - $timeprepini{$testnum}) -
2178                ($timesrvrend{$testnum} - $timesrvrini{$testnum}), $testnum);
2179            push @timetool, sprintf("%06.3f  %04d",
2180                $timetoolend{$testnum} - $timetoolini{$testnum}, $testnum);
2181            push @timelock, sprintf("%06.3f  %04d",
2182                $timesrvrlog{$testnum} - $timetoolend{$testnum}, $testnum);
2183            push @timevrfy, sprintf("%06.3f  %04d",
2184                $timevrfyend{$testnum} - $timesrvrlog{$testnum}, $testnum);
2185            push @timetest, sprintf("%06.3f  %04d",
2186                $timevrfyend{$testnum} - $timeprepini{$testnum}, $testnum);
2187        }
2188    }
2189
2190    {
2191        no warnings 'numeric';
2192        @timesrvr = sort { $b <=> $a } @timesrvr;
2193        @timeprep = sort { $b <=> $a } @timeprep;
2194        @timetool = sort { $b <=> $a } @timetool;
2195        @timelock = sort { $b <=> $a } @timelock;
2196        @timevrfy = sort { $b <=> $a } @timevrfy;
2197        @timetest = sort { $b <=> $a } @timetest;
2198    }
2199
2200    logmsg "Spent ". sprintf("%08.3f ", $timesrvrtot) .
2201           "seconds starting and verifying test harness servers.\n";
2202    logmsg "Spent ". sprintf("%08.3f ", $timepreptot) .
2203           "seconds reading definitions and doing test preparations.\n";
2204    logmsg "Spent ". sprintf("%08.3f ", $timetooltot) .
2205           "seconds actually running test tools.\n";
2206    logmsg "Spent ". sprintf("%08.3f ", $timelocktot) .
2207           "seconds awaiting server logs lock removal.\n";
2208    logmsg "Spent ". sprintf("%08.3f ", $timevrfytot) .
2209           "seconds verifying test results.\n";
2210    logmsg "Spent ". sprintf("%08.3f ", $timetesttot) .
2211           "seconds doing all of the above.\n";
2212
2213    $counter = 25;
2214    logmsg "\nTest server starting and verification time per test ".
2215        sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2216    logmsg "-time-  test\n";
2217    logmsg "------  ----\n";
2218    foreach my $txt (@timesrvr) {
2219        last if((not $fullstats) && (not $counter--));
2220        logmsg "$txt\n";
2221    }
2222
2223    $counter = 10;
2224    logmsg "\nTest definition reading and preparation time per test ".
2225        sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2226    logmsg "-time-  test\n";
2227    logmsg "------  ----\n";
2228    foreach my $txt (@timeprep) {
2229        last if((not $fullstats) && (not $counter--));
2230        logmsg "$txt\n";
2231    }
2232
2233    $counter = 25;
2234    logmsg "\nTest tool execution time per test ".
2235        sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2236    logmsg "-time-  test\n";
2237    logmsg "------  ----\n";
2238    foreach my $txt (@timetool) {
2239        last if((not $fullstats) && (not $counter--));
2240        logmsg "$txt\n";
2241    }
2242
2243    $counter = 15;
2244    logmsg "\nTest server logs lock removal time per test ".
2245        sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2246    logmsg "-time-  test\n";
2247    logmsg "------  ----\n";
2248    foreach my $txt (@timelock) {
2249        last if((not $fullstats) && (not $counter--));
2250        logmsg "$txt\n";
2251    }
2252
2253    $counter = 10;
2254    logmsg "\nTest results verification time per test ".
2255        sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2256    logmsg "-time-  test\n";
2257    logmsg "------  ----\n";
2258    foreach my $txt (@timevrfy) {
2259        last if((not $fullstats) && (not $counter--));
2260        logmsg "$txt\n";
2261    }
2262
2263    $counter = 50;
2264    logmsg "\nTotal time per test ".
2265        sprintf("(%s)...\n\n", (not $fullstats) ? "top $counter" : "full");
2266    logmsg "-time-  test\n";
2267    logmsg "------  ----\n";
2268    foreach my $txt (@timetest) {
2269        last if((not $fullstats) && (not $counter--));
2270        logmsg "$txt\n";
2271    }
2272
2273    logmsg "\n";
2274
2275    logmsg "::endgroup::\n";
2276}
2277
2278#######################################################################
2279# returns code indicating why a test was skipped
2280# 0=unknown test, 1=use test result, 2=ignore test result
2281#
2282sub ignoreresultcode {
2283    my ($testnum)=@_;
2284    if(defined $ignoretestcodes{$testnum}) {
2285        return $ignoretestcodes{$testnum};
2286    }
2287    return 0;
2288}
2289
2290#######################################################################
2291# Put the given runner ID onto the queue of runners ready for a new task
2292#
2293sub runnerready {
2294    my ($runnerid)=@_;
2295    push @runnersidle, $runnerid;
2296}
2297
2298#######################################################################
2299# Create test runners
2300#
2301sub createrunners {
2302    my ($numrunners)=@_;
2303    if(! $numrunners) {
2304        $numrunners++;
2305    }
2306    # create $numrunners runners with minimum 1
2307    for my $runnernum (1..$numrunners) {
2308        my $dir = getrunnernumlogdir($runnernum);
2309        cleardir($dir);
2310        mkdir($dir, 0777);
2311        $runnerids{$runnernum} = runner_init($dir, $jobs);
2312        runnerready($runnerids{$runnernum});
2313    }
2314}
2315
2316#######################################################################
2317# Pick a test runner for the given test
2318#
2319sub pickrunner {
2320    my ($testnum)=@_;
2321    scalar(@runnersidle) || die "No runners available";
2322
2323    return pop @runnersidle;
2324}
2325
2326#######################################################################
2327# Check options to this test program
2328#
2329
2330# Special case for CMake: replace '$TFLAGS' by the contents of the
2331# environment variable (if any).
2332if(@ARGV && $ARGV[-1] eq '$TFLAGS') {
2333    pop @ARGV;
2334    push(@ARGV, split(' ', $ENV{'TFLAGS'})) if defined($ENV{'TFLAGS'});
2335}
2336
2337$args = join(' ', @ARGV);
2338
2339$valgrind = checktestcmd("valgrind");
2340my $number=0;
2341my $fromnum=-1;
2342my @testthis;
2343while(@ARGV) {
2344    if($ARGV[0] eq "-v") {
2345        # verbose output
2346        $verbose=1;
2347    }
2348    elsif($ARGV[0] eq "-c") {
2349        # use this path to curl instead of default
2350        $DBGCURL=$CURL=$ARGV[1];
2351        shift @ARGV;
2352    }
2353    elsif($ARGV[0] eq "-vc") {
2354        # use this path to a curl used to verify servers
2355
2356        # Particularly useful when you introduce a crashing bug somewhere in
2357        # the development version as then it will not be able to run any tests
2358        # since it cannot verify the servers!
2359
2360        $VCURL=shell_quote($ARGV[1]);
2361        shift @ARGV;
2362    }
2363    elsif($ARGV[0] eq "-ac") {
2364        # use this curl only to talk to APIs (currently only CI test APIs)
2365        $ACURL=shell_quote($ARGV[1]);
2366        shift @ARGV;
2367    }
2368    elsif($ARGV[0] eq "-d") {
2369        # have the servers display protocol output
2370        $debugprotocol=1;
2371    }
2372    elsif(($ARGV[0] eq "-e") || ($ARGV[0] eq "--test-event")) {
2373        # run the tests cases event based if possible
2374        $run_event_based=1;
2375    }
2376    elsif($ARGV[0] eq "--test-duphandle") {
2377        # run the tests with --test-duphandle
2378        $run_duphandle=1;
2379    }
2380    elsif($ARGV[0] eq "-f") {
2381        # force - run the test case even if listed in DISABLED
2382        $run_disabled=1;
2383    }
2384    elsif($ARGV[0] eq "-E") {
2385        # load additional reasons to skip tests
2386        shift @ARGV;
2387        my $exclude_file = $ARGV[0];
2388        open(my $fd, "<", $exclude_file) or die "Could not open '$exclude_file': $!";
2389        while(my $line = <$fd>) {
2390            next if($line =~ /^#/);
2391            chomp $line;
2392            my ($type, $patterns, $skip_reason) = split(/\s*:\s*/, $line, 3);
2393
2394            die "Unsupported type: $type\n" if($type !~ /^keyword|test|tool$/);
2395
2396            foreach my $pattern (split(/,/, $patterns)) {
2397                if($type eq "test") {
2398                    # Strip leading zeros in the test number
2399                    $pattern = int($pattern);
2400                }
2401                $custom_skip_reasons{$type}{$pattern} = $skip_reason;
2402            }
2403        }
2404        close($fd);
2405    }
2406    elsif($ARGV[0] eq "-g") {
2407        # run this test with gdb
2408        $gdbthis=1;
2409    }
2410    elsif($ARGV[0] eq "-gl") {
2411        # run this test with lldb
2412        $gdbthis=2;
2413    }
2414    elsif($ARGV[0] eq "-gw") {
2415        # run this test with windowed gdb
2416        $gdbthis=1;
2417        $gdbxwin=1;
2418    }
2419    elsif($ARGV[0] eq "-s") {
2420        # short output
2421        $short=1;
2422    }
2423    elsif($ARGV[0] eq "-am") {
2424        # automake-style output
2425        $short=1;
2426        $automakestyle=1;
2427    }
2428    elsif($ARGV[0] =~ /-m=(\d+)/) {
2429        my ($num)=($1);
2430        $maxtime=$num;
2431    }
2432    elsif($ARGV[0] =~ /--min=(\d+)/) {
2433        my ($num)=($1);
2434        $mintotal=$num;
2435    }
2436    elsif($ARGV[0] eq "-n") {
2437        # no valgrind
2438        undef $valgrind;
2439    }
2440    elsif($ARGV[0] eq "--no-debuginfod") {
2441        # disable the valgrind debuginfod functionality
2442        $no_debuginfod = 1;
2443    }
2444    elsif($ARGV[0] eq "-R") {
2445        # execute in scrambled order
2446        $scrambleorder=1;
2447    }
2448    elsif($ARGV[0] =~ /^-t(.*)/) {
2449        # torture
2450        $torture=1;
2451        my $xtra = $1;
2452
2453        if($xtra =~ s/(\d+)$//) {
2454            $tortalloc = $1;
2455        }
2456    }
2457    elsif($ARGV[0] =~ /--shallow=(\d+)/) {
2458        # Fail no more than this amount per tests when running
2459        # torture.
2460        my ($num)=($1);
2461        $shallow=$num;
2462    }
2463    elsif($ARGV[0] =~ /--repeat=(\d+)/) {
2464        # Repeat-run the given tests this many times
2465        $repeat = $1;
2466    }
2467    elsif($ARGV[0] =~ /--retry=(\d+)/) {
2468        # Number of attempts for the whole test run to retry failed tests
2469        $retry = $1;
2470    }
2471    elsif($ARGV[0] =~ /--seed=(\d+)/) {
2472        # Set a fixed random seed (used for -R and --shallow)
2473        $randseed = $1;
2474    }
2475    elsif($ARGV[0] eq "-a") {
2476        # continue anyway, even if a test fail
2477        $anyway=1;
2478    }
2479    elsif($ARGV[0] eq "-o") {
2480        shift @ARGV;
2481        if($ARGV[0] =~ /^(\w+)=([\w.:\/\[\]-]+)$/) {
2482            my ($variable, $value) = ($1, $2);
2483            eval "\$$variable='$value'" or die "Failed to set \$$variable to $value: $@";
2484        } else {
2485            die "Failed to parse '-o $ARGV[0]'. May contain unexpected characters.\n";
2486        }
2487    }
2488    elsif($ARGV[0] eq "-p") {
2489        $postmortem=1;
2490    }
2491    elsif($ARGV[0] eq "-P") {
2492        shift @ARGV;
2493        $proxy_address=$ARGV[0];
2494    }
2495    elsif($ARGV[0] eq "-L") {
2496        # require additional library file
2497        shift @ARGV;
2498        require $ARGV[0];
2499    }
2500    elsif($ARGV[0] eq "-l") {
2501        # lists the test case names only
2502        $listonly=1;
2503    }
2504    elsif($ARGV[0] eq "--buildinfo") {
2505        $buildinfo=1;
2506    }
2507    elsif($ARGV[0] =~ /^-j(.*)/) {
2508        # parallel jobs
2509        $jobs=1;
2510        my $xtra = $1;
2511        if($xtra =~ s/(\d+)$//) {
2512            $jobs = $1;
2513        }
2514    }
2515    elsif($ARGV[0] eq "-k") {
2516        # keep stdout and stderr files after tests
2517        $keepoutfiles=1;
2518    }
2519    elsif($ARGV[0] eq "-r") {
2520        # run time statistics needs Time::HiRes
2521        if($Time::HiRes::VERSION) {
2522            # presize hashes appropriately to hold an entire test run
2523            keys(%timeprepini) = 2000;
2524            keys(%timesrvrini) = 2000;
2525            keys(%timesrvrend) = 2000;
2526            keys(%timetoolini) = 2000;
2527            keys(%timetoolend) = 2000;
2528            keys(%timesrvrlog) = 2000;
2529            keys(%timevrfyend) = 2000;
2530            $timestats=1;
2531            $fullstats=0;
2532        }
2533    }
2534    elsif($ARGV[0] eq "-rf") {
2535        # run time statistics needs Time::HiRes
2536        if($Time::HiRes::VERSION) {
2537            # presize hashes appropriately to hold an entire test run
2538            keys(%timeprepini) = 2000;
2539            keys(%timesrvrini) = 2000;
2540            keys(%timesrvrend) = 2000;
2541            keys(%timetoolini) = 2000;
2542            keys(%timetoolend) = 2000;
2543            keys(%timesrvrlog) = 2000;
2544            keys(%timevrfyend) = 2000;
2545            $timestats=1;
2546            $fullstats=1;
2547        }
2548    }
2549    elsif($ARGV[0] eq "-u") {
2550        # error instead of warning on server unexpectedly alive
2551        $err_unexpected=1;
2552    }
2553    elsif(($ARGV[0] eq "-h") || ($ARGV[0] eq "--help")) {
2554        # show help text
2555        print <<"EOHELP"
2556Usage: runtests.pl [options] [test selection(s)]
2557  -a       continue even if a test fails
2558  -ac path use this curl only to talk to APIs (currently only CI test APIs)
2559  -am      automake style output PASS/FAIL: [number] [name]
2560  --buildinfo dump buildinfo.txt
2561  -c path  use this curl executable
2562  -d       display server debug info
2563  -e, --test-event  event-based execution
2564  --test-duphandle  duplicate handles before use
2565  -E file  load the specified file to exclude certain tests
2566  -f       forcibly run even if disabled
2567  -g       run the test case with gdb
2568  -gw      run the test case with gdb as a windowed application
2569  -h       this help text
2570  -j[N]    spawn this number of processes to run tests (default 0)
2571  -k       keep stdout and stderr files present after tests
2572  -L path  require an additional perl library file to replace certain functions
2573  -l       list all test case names/descriptions
2574  -m=[seconds] set timeout for curl commands in tests
2575  --min=[count] minimum number of tests to run.
2576  -n       no valgrind
2577  --no-debuginfod disable the valgrind debuginfod functionality
2578  -o variable=value set internal variable to the specified value
2579  -P proxy use the specified proxy
2580  -p       print log file contents when a test fails
2581  -R       scrambled order (uses the random seed, see --seed)
2582  -r       run time statistics
2583  -rf      full run time statistics
2584  --repeat=[num] run the given tests this many times
2585  --retry=[num] number of attempts for the whole test run to retry failed tests
2586  -s       short output
2587  --seed=[num] set the random seed to a fixed number
2588  --shallow=[num] randomly makes the torture tests "thinner"
2589  -t[N]    torture (simulate function failures); N means fail Nth function
2590  -u       error instead of warning on server unexpectedly alive
2591  -v       verbose output
2592  -vc path use this curl only to verify the existing servers
2593  [num]    like "5 6 9" or " 5 to 22 " to run those tests only
2594  [!num]   like "!5 !6 !9" to disable those tests
2595  [~num]   like "~5 ~6 ~9" to ignore the result of those tests
2596  [keyword] like "IPv6" to select only tests containing the key word
2597  [!keyword] like "!cookies" to disable any tests containing the key word
2598  [~keyword] like "~cookies" to ignore results of tests containing key word
2599EOHELP
2600    ;
2601        exit;
2602    }
2603    elsif($ARGV[0] =~ /^(\d+)/) {
2604        $number = $1;
2605        if($fromnum >= 0) {
2606            for my $n ($fromnum .. $number) {
2607                push @testthis, $n;
2608            }
2609            $fromnum = -1;
2610        }
2611        else {
2612            push @testthis, $1;
2613        }
2614    }
2615    elsif($ARGV[0] =~ /^to$/i) {
2616        $fromnum = $number+1;
2617    }
2618    elsif($ARGV[0] =~ /^!(\d+)/) {
2619        $fromnum = -1;
2620        $disabled{$1}=$1;
2621    }
2622    elsif($ARGV[0] =~ /^~(\d+)/) {
2623        $fromnum = -1;
2624        $ignored{$1}=$1;
2625    }
2626    elsif($ARGV[0] =~ /^!(.+)/) {
2627        $disabled_keywords{lc($1)}=$1;
2628    }
2629    elsif($ARGV[0] =~ /^~(.+)/) {
2630        $ignored_keywords{lc($1)}=$1;
2631    }
2632    elsif($ARGV[0] =~ /^([-[{a-zA-Z].*)/) {
2633        $enabled_keywords{lc($1)}=$1;
2634    }
2635    else {
2636        print "Unknown option: $ARGV[0]\n";
2637        exit;
2638    }
2639    shift @ARGV;
2640}
2641
2642delete $ENV{'DEBUGINFOD_URLS'} if($ENV{'DEBUGINFOD_URLS'} && $no_debuginfod);
2643
2644if(!$randseed) {
2645    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
2646        localtime(time);
2647    # seed of the month. December 2019 becomes 201912
2648    $randseed = ($year+1900)*100 + $mon+1;
2649    print "Using curl: $CURL\n";
2650    open(my $curlvh, "-|", exerunner() . shell_quote($CURL) . " --version 2>$dev_null") ||
2651        die "could not get curl version!";
2652    my @c = <$curlvh>;
2653    close($curlvh) || die "could not get curl version!";
2654    # use the first line of output and get the md5 out of it
2655    my $str = md5($c[0]);
2656    $randseed += unpack('S', $str);  # unsigned 16-bit value
2657}
2658srand $randseed;
2659
2660if(@testthis && ($testthis[0] ne "")) {
2661    $TESTCASES=join(" ", @testthis);
2662}
2663
2664if($valgrind) {
2665    # we have found valgrind on the host, use it
2666
2667    # verify that we can invoke it fine
2668    my $code = runclient("valgrind >$dev_null 2>&1");
2669
2670    if(($code>>8) != 1) {
2671        #logmsg "Valgrind failure, disable it\n";
2672        undef $valgrind;
2673    } else {
2674
2675        # since valgrind 2.1.x, '--tool' option is mandatory
2676        # use it, if it is supported by the version installed on the system
2677        # (this happened in 2003, so we could probably do not need to care about
2678        # that old version any longer and just delete this check)
2679        runclient("valgrind --help 2>&1 | grep -- --tool >$dev_null 2>&1");
2680        if(($? >> 8)) {
2681            $valgrind_tool="";
2682        }
2683        open(my $curlh, "<", $CURL);
2684        my $l = <$curlh>;
2685        if($l =~ /^\#\!/) {
2686            # A shell script. This is typically when built with libtool,
2687            $valgrind="../libtool --mode=execute $valgrind";
2688        }
2689        close($curlh);
2690
2691        # valgrind 3 renamed the --logfile option to --log-file!!!
2692        # (this happened in 2005, so we could probably do not need to care about
2693        # that old version any longer and just delete this check)
2694        my $ver=join(' ', runclientoutput("valgrind --version"));
2695        # cut off all but digits and dots
2696        $ver =~ s/[^0-9.]//g;
2697
2698        if($ver =~ /^(\d+)/) {
2699            $ver = $1;
2700            if($ver < 3) {
2701                $valgrind_logfile="--logfile";
2702            }
2703        }
2704    }
2705}
2706
2707if($gdbthis) {
2708    # open the executable curl and read the first 4 bytes of it
2709    open(my $check, "<", $CURL);
2710    my $c;
2711    sysread $check, $c, 4;
2712    close($check);
2713    if($c eq "#! /") {
2714        # A shell script. This is typically when built with libtool,
2715        $libtool = 1;
2716        $gdb = "../libtool --mode=execute gdb";
2717    }
2718}
2719
2720#######################################################################
2721# clear and create logging directory:
2722#
2723
2724# TODO: figure how to get around this. This directory is needed for checksystemfeatures()
2725# Maybe create & use & delete a temporary directory in that function
2726cleardir($LOGDIR);
2727mkdir($LOGDIR, 0777);
2728mkdir("$LOGDIR/$LOCKDIR", 0777);
2729
2730#######################################################################
2731# initialize some variables
2732#
2733
2734get_disttests();
2735if(!$jobs) {
2736    # Disable buffered logging with only one test job
2737    setlogfunc(\&logmsg);
2738}
2739
2740if(!$mintotal && $ENV{"CURL_TEST_MIN"}) {
2741    $mintotal = $ENV{"CURL_TEST_MIN"};
2742}
2743
2744#######################################################################
2745# Output curl version and host info being tested
2746#
2747
2748if(!$listonly) {
2749    checksystemfeatures();
2750}
2751
2752#######################################################################
2753# Output information about the curl build
2754#
2755if(!$listonly && $buildinfo) {
2756    if(open(my $fd, "<", "../buildinfo.txt")) {
2757        while(my $line = <$fd>) {
2758            chomp $line;
2759            if($line && $line !~ /^#/) {
2760                logmsg("* $line\n");
2761            }
2762        }
2763        close($fd);
2764    }
2765}
2766
2767#######################################################################
2768# initialize configuration needed to set up servers
2769# TODO: rearrange things so this can be called only in runner_init()
2770#
2771initserverconfig();
2772
2773if(!$listonly) {
2774    # these can only be displayed after initserverconfig() has been called
2775    displayserverfeatures();
2776
2777    # globally disabled tests
2778    disabledtests("$TESTDIR/DISABLED");
2779}
2780
2781#######################################################################
2782# Fetch all disabled tests, if there are any
2783#
2784sub disabledtests {
2785    my ($file) = @_;
2786    my @input;
2787
2788    if(open(my $disabledh, "<", $file)) {
2789        while(<$disabledh>) {
2790            if(/^ *\#/) {
2791                # allow comments
2792                next;
2793            }
2794            push @input, $_;
2795        }
2796        close($disabledh);
2797
2798        # preprocess the input to make conditionally disabled tests depending
2799        # on variables
2800        my @pp = prepro(0, @input);
2801        for my $t (@pp) {
2802            if($t =~ /(\d+)/) {
2803                my ($n) = $1;
2804                $disabled{$n}=$n; # disable this test number
2805                if(! -f "$srcdir/data/test$n") {
2806                    print STDERR "WARNING! Non-existing test $n in $file!\n";
2807                    # fail hard to make user notice
2808                    exit 1;
2809                }
2810                logmsg "DISABLED: test $n\n" if($verbose);
2811            }
2812            else {
2813                print STDERR "$file: rubbish content: $t\n";
2814                exit 2;
2815            }
2816        }
2817    }
2818    else {
2819        print STDERR "Cannot open $file, exiting\n";
2820        exit 3;
2821    }
2822}
2823
2824#######################################################################
2825# If 'all' tests are requested, find out all test numbers
2826#
2827
2828if($TESTCASES eq "all") {
2829    # Get all commands and find out their test numbers
2830    opendir(DIR, $TESTDIR) || die "cannot opendir $TESTDIR: $!";
2831    my @cmds = grep { /^test([0-9]+)$/ && -f "$TESTDIR/$_" } readdir(DIR);
2832    closedir(DIR);
2833
2834    $TESTCASES=""; # start with no test cases
2835
2836    # cut off everything but the digits
2837    for(@cmds) {
2838        $_ =~ s/[a-z\/\.]*//g;
2839    }
2840    # sort the numbers from low to high
2841    foreach my $n (sort { $a <=> $b } @cmds) {
2842        if($disabled{$n}) {
2843            # skip disabled test cases
2844            my $why = "configured as DISABLED";
2845            $skipped{$why}++;
2846            $teststat[$n]=$why; # store reason for this test case
2847            next;
2848        }
2849        $TESTCASES .= " $n";
2850    }
2851}
2852else {
2853    my $verified="";
2854    for(split(" ", $TESTCASES)) {
2855        if(-e "$TESTDIR/test$_") {
2856            $verified.="$_ ";
2857        }
2858    }
2859    if($verified eq "") {
2860        print "No existing test cases were specified\n";
2861        exit;
2862    }
2863    $TESTCASES = $verified;
2864}
2865if($repeat) {
2866    my $s;
2867    for(1 .. $repeat) {
2868        $s .= $TESTCASES;
2869    }
2870    $TESTCASES = $s;
2871}
2872
2873if($scrambleorder) {
2874    # scramble the order of the test cases
2875    my @rand;
2876    while($TESTCASES) {
2877        my @all = split(/ +/, $TESTCASES);
2878        if(!$all[0]) {
2879            # if the first is blank, shift away it
2880            shift @all;
2881        }
2882        my $r = rand @all;
2883        push @rand, $all[$r];
2884        $all[$r]="";
2885        $TESTCASES = join(" ", @all);
2886    }
2887    $TESTCASES = join(" ", @rand);
2888}
2889
2890# Display the contents of the given file.  Line endings are canonicalized
2891# and excessively long files are elided
2892sub displaylogcontent {
2893    my ($file)=@_;
2894    if(open(my $single, "<", $file)) {
2895        my $linecount = 0;
2896        my $truncate;
2897        my @tail;
2898        while(my $string = <$single>) {
2899            $string =~ s/\r\n/\n/g;
2900            $string =~ s/[\r\f\032]/\n/g;
2901            $string .= "\n" unless ($string =~ /\n$/);
2902            $string =~ tr/\n//;
2903            for my $line (split(m/\n/, $string)) {
2904                $line =~ s/\s*\!$//;
2905                if($truncate) {
2906                    push @tail, " $line\n";
2907                } else {
2908                    logmsg " $line\n";
2909                }
2910                $linecount++;
2911                $truncate = $linecount > 1200;
2912            }
2913        }
2914        close($single);
2915        if(@tail) {
2916            my $tailshow = 200;
2917            my $tailskip = 0;
2918            my $tailtotal = scalar @tail;
2919            if($tailtotal > $tailshow) {
2920                $tailskip = $tailtotal - $tailshow;
2921                logmsg "=== File too long: $tailskip lines omitted here\n";
2922            }
2923            for($tailskip .. $tailtotal-1) {
2924                logmsg "$tail[$_]";
2925            }
2926        }
2927    }
2928}
2929
2930sub displaylogs {
2931    my ($runnerid, $testnum)=@_;
2932    my $logdir = getrunnerlogdir($runnerid);
2933    opendir(DIR, $logdir) ||
2934        die "cannot open dir: $!";
2935    my @logs = readdir(DIR);
2936    closedir(DIR);
2937
2938    logmsg "== Contents of files in the $logdir/ directory after test $testnum\n";
2939    foreach my $log (sort @logs) {
2940        if($log =~ /\.(\.|)$/) {
2941            next; # skip "." and ".."
2942        }
2943        if($log =~ /^\.nfs/) {
2944            next; # skip ".nfs"
2945        }
2946        if(($log eq "memdump") || ($log eq "core")) {
2947            next; # skip "memdump" and  "core"
2948        }
2949        if((-d "$logdir/$log") || (! -s "$logdir/$log")) {
2950            next; # skip directory and empty files
2951        }
2952        if(($log =~ /^stdout\d+/) && ($log !~ /^stdout$testnum/)) {
2953            next; # skip stdoutNnn of other tests
2954        }
2955        if(($log =~ /^stderr\d+/) && ($log !~ /^stderr$testnum/)) {
2956            next; # skip stderrNnn of other tests
2957        }
2958        if(($log =~ /^upload\d+/) && ($log !~ /^upload$testnum/)) {
2959            next; # skip uploadNnn of other tests
2960        }
2961        if(($log =~ /^curl\d+\.out/) && ($log !~ /^curl$testnum\.out/)) {
2962            next; # skip curlNnn.out of other tests
2963        }
2964        if(($log =~ /^test\d+\.txt/) && ($log !~ /^test$testnum\.txt/)) {
2965            next; # skip testNnn.txt of other tests
2966        }
2967        if(($log =~ /^file\d+\.txt/) && ($log !~ /^file$testnum\.txt/)) {
2968            next; # skip fileNnn.txt of other tests
2969        }
2970        if(($log =~ /^netrc\d+/) && ($log !~ /^netrc$testnum/)) {
2971            next; # skip netrcNnn of other tests
2972        }
2973        if(($log =~ /^trace\d+/) && ($log !~ /^trace$testnum/)) {
2974            next; # skip traceNnn of other tests
2975        }
2976        if(($log =~ /^valgrind\d+/) && ($log !~ /^valgrind$testnum(?:\..*)?$/)) {
2977            next; # skip valgrindNnn of other tests
2978        }
2979        if(($log =~ /^test$testnum$/)) {
2980            next; # skip test$testnum since it can be big
2981        }
2982        logmsg "=== Start of file $log\n";
2983        displaylogcontent("$logdir/$log");
2984        logmsg "=== End of file $log\n";
2985    }
2986}
2987
2988#######################################################################
2989# Scan tests to find suitable candidates
2990#
2991
2992my $failed;
2993my $failedign;
2994my $failedre;
2995my $ok=0;
2996my $ign=0;
2997my $total=0;
2998my $executed=0;
2999my $retry_done=0;
3000my $lasttest=0;
3001my @at = split(" ", $TESTCASES);
3002my $count=0;
3003my $endwaitcnt=0;
3004
3005$start = time();
3006
3007# scan all tests to find ones we should try to run
3008my @runtests;
3009foreach my $testnum (@at) {
3010    $lasttest = $testnum if($testnum > $lasttest);
3011    my ($why, $errorreturncode) = singletest_shouldrun($testnum);
3012    if($why || $listonly) {
3013        # Display test name now--test will be completely skipped later
3014        my $error = singletest_count($testnum, $why);
3015        next;
3016    }
3017    $ignoretestcodes{$testnum} = $errorreturncode;
3018    push(@runtests, $testnum);
3019}
3020my $totaltests = scalar(@runtests);
3021
3022if($listonly) {
3023    exit(0);
3024}
3025
3026#######################################################################
3027# Setup CI Test Run
3028citest_starttestrun();
3029
3030#######################################################################
3031# Start test runners
3032#
3033my $numrunners = $jobs < scalar(@runtests) ? $jobs : scalar(@runtests);
3034createrunners($numrunners);
3035
3036#######################################################################
3037# The main test-loop
3038#
3039# Every iteration through the loop consists of these steps:
3040#   - if the global abort flag is set, exit the loop; we are done
3041#   - if a runner is idle, start a new test on it
3042#   - if all runners are idle, exit the loop; we are done
3043#   - if a runner has a response for us, process the response
3044
3045# run through each candidate test and execute it
3046my $runner_wait_cnt = 0;
3047
3048# number of retry attempts for the whole test run
3049my $retry_left;
3050if($torture) {
3051    $retry_left = 0;  # No use of retrying torture tests
3052}
3053else {
3054    $retry_left = $retry;
3055}
3056
3057while() {
3058    # check the abort flag
3059    if($globalabort) {
3060        logmsg singletest_dumplogs();
3061        logmsg "Aborting tests\n";
3062        logmsg "Waiting for " . scalar((keys %runnersrunning)) . " outstanding test(s) to finish...\n";
3063        # Wait for the last requests to complete and throw them away so
3064        # that IPC calls & responses stay in sync
3065        # TODO: send a signal to the runners to interrupt a long test
3066        foreach my $rid (keys %runnersrunning) {
3067            runnerar($rid);
3068            delete $runnersrunning{$rid};
3069            logmsg ".";
3070            $| = 1;
3071        }
3072        logmsg "\n";
3073        last;
3074    }
3075
3076    # Start a new test if possible
3077    if(scalar(@runnersidle) && scalar(@runtests)) {
3078        # A runner is ready to run a test, and tests are still available to run
3079        # so start a new test.
3080        $count++;
3081        my $testnum = shift(@runtests);
3082
3083        # pick a runner for this new test
3084        my $runnerid = pickrunner($testnum);
3085        $countforrunner{$runnerid} = $count;
3086
3087        # Start the test
3088        my ($error, $again) = singletest($runnerid, $testnum, $countforrunner{$runnerid}, $totaltests);
3089        if($again) {
3090            # this runner is busy running a test
3091            $runnersrunning{$runnerid} = $testnum;
3092        } else {
3093            runnerready($runnerid);
3094            if($error >= 0) {
3095                # We make this simplifying assumption to avoid having to handle
3096                # $error properly here, but we must handle the case of runner
3097                # death without abending here.
3098                die "Internal error: test must not complete on first call";
3099            }
3100        }
3101    }
3102
3103    # See if we have completed all the tests
3104    if(!scalar(%runnersrunning)) {
3105        # No runners are running; we must be done
3106        scalar(@runtests) && die 'Internal error: still have tests to run';
3107        last;
3108    }
3109
3110    # See if a test runner needs attention
3111    # If we could be running more tests, do not wait so we can schedule a new
3112    # one immediately. If all runners are busy, wait a fraction of a second
3113    # for one to finish so we can still loop around to check the abort flag.
3114    my $runnerwait = scalar(@runnersidle) && scalar(@runtests) ? 0.1 : 1.0;
3115    my (@ridsready, $riderror) = runnerar_ready($runnerwait);
3116    if(@ridsready) {
3117        for my $ridready (@ridsready) {
3118            if($ridready && ! defined $runnersrunning{$ridready}) {
3119                # On Linux, a closed pipe still shows up as ready instead of error.
3120                # Detect this here by seeing if we are expecting it to be ready and
3121                # treat it as an error if not.
3122                logmsg "ERROR: Runner $ridready is unexpectedly ready; is probably actually dead\n";
3123                $riderror = $ridready;
3124                undef $ridready;
3125            }
3126            if($ridready) {
3127                $endwaitcnt = 0;
3128                # This runner is ready to be serviced
3129                my $testnum = $runnersrunning{$ridready};
3130                defined $testnum ||  die "Internal error: test for runner $ridready unknown";
3131                delete $runnersrunning{$ridready};
3132                my ($error, $again) = singletest($ridready, $testnum, $countforrunner{$ridready}, $totaltests);
3133                if($again) {
3134                    # this runner is busy running a test
3135                    $runnersrunning{$ridready} = $testnum;
3136                } else {
3137                    # Test is complete
3138                    $runner_wait_cnt = 0;
3139                    runnerready($ridready);
3140
3141                    if($error < 0) {
3142                        # not a test we can run
3143                        next;
3144                    }
3145
3146                    $total++; # number of tests we have run
3147                    $executed++;
3148
3149                    if($error > 0) {
3150                        if($error == 2) {
3151                            # ignored test failures
3152                            $failedign .= "$testnum ";
3153                        }
3154                        else {
3155                            # make another attempt to counteract flaky failures
3156                            if($retry_left > 0) {
3157                                $retry_left--;
3158                                $retry_done++;
3159                                $total--;
3160                                push(@runtests, $testnum);
3161                                $failedre .= "$testnum ";
3162                            }
3163                            else {
3164                                $failed.= "$testnum ";
3165                            }
3166                        }
3167                        if($postmortem) {
3168                            # display all files in $LOGDIR/ in a nice way
3169                            displaylogs($ridready, $testnum);
3170                        }
3171                        if($error == 2) {
3172                            $ign++; # ignored test result counter
3173                        }
3174                        elsif(!$anyway) {
3175                            # a test failed, abort
3176                            logmsg "\n - abort tests\n";
3177                            undef @runtests;  # empty out the remaining tests
3178                        }
3179                    }
3180                    elsif(!$error) {
3181                        $ok++; # successful test counter
3182                    }
3183                }
3184            }
3185        }
3186    }
3187    if(!@ridsready && $runnerwait && !$torture && scalar(%runnersrunning)) {
3188        $runner_wait_cnt++;
3189        if($runner_wait_cnt >= 5) {
3190            my $msg = "waiting for " . scalar(%runnersrunning) . " results:";
3191            my $sep = " ";
3192            foreach my $rid (keys %runnersrunning) {
3193                $msg .= $sep . $runnersrunning{$rid} . "[$rid]";
3194                $sep = ", "
3195            }
3196            logmsg "$msg\n";
3197        }
3198        if($runner_wait_cnt >= 10) {
3199            $runner_wait_cnt = 0;
3200            foreach my $rid (keys %runnersrunning) {
3201                my $testnum = $runnersrunning{$rid};
3202                logmsg "current state of test $testnum in [$rid]:\n";
3203                displaylogs($rid, $testnum);
3204            }
3205        }
3206    }
3207    if($riderror) {
3208        logmsg "ERROR: runner $riderror is dead! aborting test run\n";
3209        delete $runnersrunning{$riderror} if(defined $runnersrunning{$riderror});
3210        $globalabort = 1;
3211    }
3212    $endwaitcnt += $runnerwait;
3213    if($endwaitcnt >= 10) {
3214        # Once all tests have been scheduled on a runner at the end of a test
3215        # run, we just wait for their results to come in. If we are still
3216        # waiting after a couple of minutes ($endwaitcnt multiplied by
3217        # $runnerwait, plus $jobs because that number will not time out), display
3218        # the same test runner status as we give with a SIGUSR1. This will
3219        # likely point to a single test that has hung.
3220        logmsg "Hmmm, the tests are taking a while to finish. Here is the status:\n";
3221        catch_usr1();
3222        $endwaitcnt = 0;
3223    }
3224}
3225
3226my $sofar = time() - $start;
3227
3228#######################################################################
3229# Finish CI Test Run
3230citest_finishtestrun();
3231
3232# Tests done, stop the servers
3233foreach my $runnerid (values %runnerids) {
3234    runnerac_stopservers($runnerid);
3235}
3236
3237# Wait for servers to stop
3238my $unexpected;
3239foreach my $runnerid (values %runnerids) {
3240    my ($rid, $unexpected_for_runner, $logs) = runnerar($runnerid);
3241    $unexpected ||= $unexpected_for_runner;
3242    logmsg $logs;
3243}
3244
3245# Kill the runners
3246# There is a race condition here since we do not know exactly when the runners
3247# have each finished shutting themselves down, but we are about to exit so it
3248# does not make much difference.
3249foreach my $runnerid (values %runnerids) {
3250    runnerac_shutdown($runnerid);
3251    sleep 0;  # give runner a context switch so it can shut itself down
3252}
3253
3254my $numskipped = %skipped ? sum values %skipped : 0;
3255my $all = $total + $numskipped;
3256
3257runtimestats($lasttest);
3258
3259if($all) {
3260    logmsg "TESTDONE: $all tests were considered during ".
3261        sprintf("%.0f", $sofar) ." seconds.\n";
3262}
3263
3264if(%skipped && !$short) {
3265    my $s=0;
3266    # Temporary hash to print the restraints sorted by the number
3267    # of their occurrences
3268    my %restraints;
3269    logmsg "TESTINFO: $numskipped tests were skipped due to these restraints:\n";
3270
3271    for(keys %skipped) {
3272        my $r = $_;
3273        my $skip_count = $skipped{$r};
3274        my $log_line = sprintf("TESTINFO: \"%s\" %d time%s (", $r, $skip_count,
3275                               ($skip_count == 1) ? "" : "s");
3276
3277        # now gather all test case numbers that had this reason for being
3278        # skipped
3279        my $c=0;
3280        my $max = 9;
3281        for(0 .. scalar @teststat) {
3282            my $t = $_;
3283            if($teststat[$t] && ($teststat[$t] eq $r)) {
3284                if($c < $max) {
3285                    $log_line .= ", " if($c);
3286                    $log_line .= $t;
3287                }
3288                $c++;
3289            }
3290        }
3291        if($c > $max) {
3292            $log_line .= " and ".($c-$max)." more";
3293        }
3294        $log_line .= ")\n";
3295        $restraints{$log_line} = $skip_count;
3296    }
3297    foreach my $log_line (sort {$restraints{$b} <=> $restraints{$a} || uc($a) cmp uc($b)} keys %restraints) {
3298        logmsg $log_line;
3299    }
3300}
3301
3302sub testnumdetails {
3303    my ($desc, $numlist) = @_;
3304    foreach my $testnum (split(' ', $numlist)) {
3305        if(!loadtest("${TESTDIR}/test${testnum}", 1)) {
3306            my @info_keywords = getpart("info", "keywords");
3307            my $testname = (getpart("client", "name"))[0];
3308            chomp $testname;
3309            logmsg "$desc $testnum: '$testname'";
3310            my $first = 1;
3311            for my $k (@info_keywords) {
3312                chomp $k;
3313                my $sep = ($first == 1) ? " " : ", ";
3314                logmsg "$sep$k";
3315                $first = 0;
3316            }
3317            logmsg "\n";
3318        }
3319    }
3320}
3321
3322if($executed) {
3323    if($failedre) {
3324        my $sorted = numsortwords($failedre);
3325        logmsg "::group::Failed Retried Test details\n";
3326        testnumdetails("FAIL-RETRIED", $sorted);
3327        logmsg "RETRIED: failed tests: $sorted\n";
3328        logmsg "::endgroup::\n";
3329    }
3330
3331    if($passedign) {
3332        my $sorted = numsortwords($passedign);
3333        logmsg "::group::Passed Ignored Test details\n";
3334        testnumdetails("PASSED-IGNORED", $sorted);
3335        logmsg "IGNORED: passed tests: $sorted\n";
3336        logmsg "::endgroup::\n";
3337    }
3338
3339    if($failedign) {
3340        my $sorted = numsortwords($failedign);
3341        testnumdetails("FAIL-IGNORED", $sorted);
3342        logmsg "IGNORED: failed tests: $sorted\n";
3343    }
3344    logmsg sprintf("TESTDONE: $ok tests out of $total reported OK: %d%%\n",
3345                   $ok/$total*100);
3346
3347    if($failed && ($ok != $total)) {
3348        my $failedsorted = numsortwords($failed);
3349        logmsg "\n";
3350        testnumdetails("FAIL", $failedsorted);
3351        logmsg "\nTESTFAIL: These test cases failed: $failedsorted\n\n";
3352    }
3353}
3354else {
3355    logmsg "\nTESTFAIL: No tests were performed\n\n";
3356    if(scalar(keys %enabled_keywords)) {
3357        logmsg "TESTFAIL: Nothing matched these keywords: ";
3358        for(keys %enabled_keywords) {
3359            logmsg "$_ ";
3360        }
3361        logmsg "\n";
3362    }
3363}
3364
3365if($mintotal) {
3366    if($total < $mintotal) {
3367        logmsg "TESTFAIL: number of tests run ($total) was below the minimum of: $mintotal\n";
3368        exit 1;
3369    }
3370    else {
3371        logmsg "TESTDONE: minimum number of tests was met: $mintotal\n";
3372    }
3373}
3374
3375if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) {
3376    exit 1;
3377}