luna - curl/src/tool

   1/***************************************************************************
   2 *                                  _   _ ____  _
   3 *  Project                     ___| | | |  _ \| |
   4 *                             / __| | | | |_) | |
   5 *                            | (__| |_| |  _ <| |___
   6 *                             \___|\___/|_| \_\_____|
   7 *
   8 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
   9 *
  10 * This software is licensed as described in the file COPYING, which
  11 * you should have received as part of this distribution. The terms
  12 * are also available at https://curl.se/docs/copyright.html.
  13 *
  14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
  15 * copies of the Software, and permit persons to whom the Software is
  16 * furnished to do so, under the terms of the COPYING file.
  17 *
  18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  19 * KIND, either express or implied.
  20 *
  21 * SPDX-License-Identifier: curl
  22 *
  23 ***************************************************************************/
  24#include "tool_setup.h"
  25
  26#include "curlx/nonblock.h" /* for curlx_nonblock() */
  27
  28#ifdef HAVE_LOCALE_H
  29#  include <locale.h>
  30#endif
  31
  32#ifdef HAVE_SYS_SELECT_H
  33#  include <sys/select.h>
  34#elif defined(HAVE_UNISTD_H)
  35#  include <unistd.h>
  36#endif
  37
  38#ifdef __VMS
  39#  include <fabdef.h>
  40#endif
  41
  42#ifdef __AMIGA__
  43#  include <proto/dos.h>
  44#endif
  45
  46#ifdef HAVE_NETINET_IN_H
  47#  include <netinet/in.h>
  48#endif
  49
  50#ifdef HAVE_UV_H
  51/* this is for libuv-enabled debug builds only */
  52#include <uv.h>
  53#endif
  54
  55#include "tool_cfgable.h"
  56#include "tool_cb_hdr.h"
  57#include "tool_cb_prg.h"
  58#include "tool_cb_wrt.h"
  59#include "tool_dirhie.h"
  60#include "tool_doswin.h"
  61#include "tool_easysrc.h"
  62#include "tool_filetime.h"
  63#include "tool_getparam.h"
  64#include "tool_helpers.h"
  65#include "tool_libinfo.h"
  66#include "tool_main.h"
  67#include "tool_msgs.h"
  68#include "tool_operate.h"
  69#include "tool_operhlp.h"
  70#include "tool_paramhlp.h"
  71#include "tool_parsecfg.h"
  72#include "tool_setopt.h"
  73#include "tool_ssls.h"
  74#include "tool_urlglob.h"
  75#include "tool_util.h"
  76#include "tool_writeout.h"
  77#include "tool_xattr.h"
  78#include "tool_vms.h"
  79#include "tool_help.h"
  80#include "tool_hugehelp.h"
  81#include "tool_progress.h"
  82#include "tool_ipfs.h"
  83#include "config2setopts.h"
  84#include "var.h"
  85
  86#ifdef DEBUGBUILD
  87/* libcurl's debug-only curl_easy_perform_ev() */
  88CURL_EXTERN CURLcode curl_easy_perform_ev(CURL *easy);
  89#endif
  90
  91#define CURL_CA_CERT_ERRORMSG                                              \
  92  "More details here: https://curl.se/docs/sslcerts.html\n\n"              \
  93  "curl failed to verify the legitimacy of the server and therefore "      \
  94  "could not\nestablish a secure connection to it. To learn more about "   \
  95  "this situation and\nhow to fix it, please visit the webpage mentioned " \
  96  "above.\n"
  97
  98static CURLcode create_transfer(CURLSH *share,
  99                                bool *added,
 100                                bool *skipped);
 101
 102static bool is_fatal_error(CURLcode code)
 103{
 104  switch(code) {
 105  case CURLE_FAILED_INIT:
 106  case CURLE_OUT_OF_MEMORY:
 107  case CURLE_UNKNOWN_OPTION:
 108  case CURLE_BAD_FUNCTION_ARGUMENT:
 109    /* critical error */
 110    return TRUE;
 111  default:
 112    break;
 113  }
 114
 115  /* no error or not critical */
 116  return FALSE;
 117}
 118
 119/*
 120 * Check if a given string is a PKCS#11 URI
 121 */
 122static bool is_pkcs11_uri(const char *string)
 123{
 124  if(curl_strnequal(string, "pkcs11:", 7)) {
 125    return TRUE;
 126  }
 127  else {
 128    return FALSE;
 129  }
 130}
 131
 132#ifdef __VMS
 133/*
 134 * get_vms_file_size does what it takes to get the real size of the file
 135 *
 136 * For fixed files, find out the size of the EOF block and adjust.
 137 *
 138 * For all others, have to read the entire file in, discarding the contents.
 139 * Most posted text files are small, and binary files like zlib archives
 140 * and CD/DVD images should be either a STREAM_LF format or a fixed format.
 141 *
 142 */
 143static curl_off_t vms_realfilesize(const char *name,
 144                                   const curlx_struct_stat *stat_buf)
 145{
 146  char buffer[8192];
 147  curl_off_t count;
 148  int ret_stat;
 149  FILE *file;
 150
 151  /* !checksrc! disable FOPENMODE 1 */
 152  file = curlx_fopen(name, "r"); /* VMS */
 153  if(!file) {
 154    return 0;
 155  }
 156  count = 0;
 157  ret_stat = 1;
 158  while(ret_stat > 0) {
 159    ret_stat = fread(buffer, 1, sizeof(buffer), file);
 160    if(ret_stat)
 161      count += ret_stat;
 162  }
 163  curlx_fclose(file);
 164
 165  return count;
 166}
 167
 168/*
 169 *
 170 *  VmsSpecialSize checks to see if the stat st_size can be trusted and
 171 *  if not to call a routine to get the correct size.
 172 *
 173 */
 174static curl_off_t VmsSpecialSize(const char *name,
 175                                 const curlx_struct_stat *stat_buf)
 176{
 177  switch(stat_buf->st_fab_rfm) {
 178  case FAB$C_VAR:
 179  case FAB$C_VFC:
 180    return vms_realfilesize(name, stat_buf);
 181    break;
 182  default:
 183    return stat_buf->st_size;
 184  }
 185}
 186#endif /* __VMS */
 187
 188struct per_transfer *transfers; /* first node */
 189static struct per_transfer *transfersl; /* last node */
 190
 191/* add_per_transfer creates a new 'per_transfer' node in the linked
 192   list of transfers */
 193static CURLcode add_per_transfer(struct per_transfer **per)
 194{
 195  struct per_transfer *p;
 196  p = curlx_calloc(1, sizeof(struct per_transfer));
 197  if(!p)
 198    return CURLE_OUT_OF_MEMORY;
 199  if(!transfers)
 200    /* first entry */
 201    transfersl = transfers = p;
 202  else {
 203    /* make the last node point to the new node */
 204    transfersl->next = p;
 205    /* make the new node point back to the formerly last node */
 206    p->prev = transfersl;
 207    /* move the last node pointer to the new entry */
 208    transfersl = p;
 209  }
 210  *per = p;
 211
 212  return CURLE_OK;
 213}
 214
 215/* Remove the specified transfer from the list (and free it), return the next
 216   in line */
 217static struct per_transfer *del_per_transfer(struct per_transfer *per)
 218{
 219  struct per_transfer *n;
 220  struct per_transfer *p;
 221  DEBUGASSERT(transfers);
 222  DEBUGASSERT(transfersl);
 223  DEBUGASSERT(per);
 224
 225  n = per->next;
 226  p = per->prev;
 227
 228  if(p)
 229    p->next = n;
 230  else
 231    transfers = n;
 232
 233  if(n)
 234    n->prev = p;
 235  else
 236    transfersl = p;
 237
 238  curlx_free(per->uploadfile);
 239  curlx_free(per->outfile);
 240  curlx_free(per->url);
 241  curl_easy_cleanup(per->curl);
 242  curlx_free(per);
 243
 244  return n;
 245}
 246
 247static CURLcode pre_transfer(struct per_transfer *per)
 248{
 249  curl_off_t uploadfilesize = -1;
 250  curlx_struct_stat fileinfo;
 251  CURLcode result = CURLE_OK;
 252
 253  if(per->uploadfile && !stdin_upload(per->uploadfile)) {
 254    /* VMS Note:
 255     *
 256     * Reading binary from files can be a problem... Only FIXED, VAR
 257     * etc WITHOUT implied CC do work. Others need a \n appended to
 258     * a line
 259     *
 260     * - Stat gives a size but this is UNRELIABLE in VMS. E.g.
 261     * a fixed file with implied CC needs to have a byte added for every
 262     * record processed, this can be derived from Filesize & recordsize
 263     * for VARiable record files the records need to be counted!  for
 264     * every record add 1 for linefeed and subtract 2 for the record
 265     * header for VARIABLE header files only the bare record data needs
 266     * to be considered with one appended if implied CC
 267     */
 268#ifdef __VMS
 269    /* Calculate the real upload size for VMS */
 270    per->infd = -1;
 271    if(curlx_stat(per->uploadfile, &fileinfo) == 0) {
 272      fileinfo.st_size = VmsSpecialSize(per->uploadfile, &fileinfo);
 273      switch(fileinfo.st_fab_rfm) {
 274      case FAB$C_VAR:
 275      case FAB$C_VFC:
 276      case FAB$C_STMCR:
 277        per->infd = curlx_open(per->uploadfile, O_RDONLY | CURL_O_BINARY);
 278        break;
 279      default:
 280        per->infd = curlx_open(per->uploadfile, O_RDONLY | CURL_O_BINARY,
 281                               "rfm=stmlf", "ctx=stm");
 282      }
 283    }
 284    if(per->infd == -1)
 285#else
 286      per->infd = curlx_open(per->uploadfile, O_RDONLY | CURL_O_BINARY);
 287    if((per->infd == -1) || curlx_fstat(per->infd, &fileinfo))
 288#endif
 289    {
 290      helpf("cannot open '%s'", per->uploadfile);
 291      if(per->infd != -1) {
 292        curlx_close(per->infd);
 293        per->infd = STDIN_FILENO;
 294      }
 295      return CURLE_READ_ERROR;
 296    }
 297    per->infdopen = TRUE;
 298
 299    /* we ignore file size for char/block devices, sockets, etc. */
 300    if(S_ISREG(fileinfo.st_mode))
 301      uploadfilesize = fileinfo.st_size;
 302
 303#ifdef DEBUGBUILD
 304    /* allow dedicated test cases to override */
 305    {
 306      const char *ev = getenv("CURL_UPLOAD_SIZE");
 307      if(ev) {
 308        curl_off_t sz;
 309        curlx_str_number(&ev, &sz, CURL_OFF_T_MAX);
 310        uploadfilesize = sz;
 311      }
 312    }
 313#endif
 314
 315    if(uploadfilesize != -1)
 316      my_setopt_offt(per->curl, CURLOPT_INFILESIZE_LARGE, uploadfilesize);
 317  }
 318  per->uploadfilesize = uploadfilesize;
 319  per->start = curlx_now();
 320  return result;
 321}
 322
 323void single_transfer_cleanup(void)
 324{
 325  struct State *state = &global->state;
 326  /* Free list of remaining URLs */
 327  glob_cleanup(&state->urlglob);
 328  curlx_safefree(state->uploadfile);
 329  /* Free list of globbed upload files */
 330  glob_cleanup(&state->inglob);
 331}
 332
 333/* Helper function for retrycheck.
 334 *
 335 * This function is a prerequisite check used to determine whether or not some
 336 * already downloaded data (ie out->bytes written) can be safely resumed in a
 337 * subsequent transfer. The conditions are somewhat pedantic to avoid any risk
 338 * of data corruption.
 339 *
 340 * Specific HTTP limitations (scheme, method, response code, etc) are checked
 341 * in retrycheck if this prerequisite check is met.
 342 */
 343static bool is_outfile_auto_resumable(struct OperationConfig *config,
 344                                      struct per_transfer *per,
 345                                      CURLcode result)
 346{
 347  struct OutStruct *outs = &per->outs;
 348  return config->use_resume && config->resume_from_current &&
 349    config->resume_from >= 0 && outs->init == config->resume_from &&
 350    outs->bytes > 0 && outs->filename && outs->regular_file && outs->fopened &&
 351    outs->stream && !ferror(outs->stream) &&
 352    !config->customrequest && !per->uploadfile &&
 353    (config->httpreq == TOOL_HTTPREQ_UNSPEC ||
 354     config->httpreq == TOOL_HTTPREQ_GET) &&
 355    /* CURLE_WRITE_ERROR could mean outs->bytes is not accurate */
 356    result != CURLE_WRITE_ERROR && result != CURLE_RANGE_ERROR;
 357}
 358
 359enum retryreason {
 360  RETRY_NO,
 361  RETRY_ALL_ERRORS,
 362  RETRY_TIMEOUT,
 363  RETRY_CONNREFUSED,
 364  RETRY_HTTP,
 365  RETRY_FTP
 366};
 367
 368/* figure out how long to wait until retry */
 369static uint32_t retry_sleep(struct OperationConfig *config,
 370                            struct per_transfer *per,
 371                            enum retryreason retry,
 372                            bool *retryp)
 373{
 374  static const char * const m[] = {
 375    "(retrying all errors)",
 376    ": timeout",
 377    ": connection refused",
 378    ": HTTP error",
 379    ": FTP error"
 380  };
 381  CURL *curl = per->curl;
 382  uint32_t sleeptime = 0;
 383  if(RETRY_HTTP == retry) {
 384    curl_off_t retry_after = 0;
 385    curl_easy_getinfo(curl, CURLINFO_RETRY_AFTER, &retry_after);
 386    if(retry_after) {
 387      /* make sure it does not overflow */
 388      if(retry_after > (curl_off_t)(UINT32_MAX / 1000))
 389        sleeptime = UINT32_MAX;
 390      else
 391        sleeptime = (uint32_t)retry_after * 1000U; /* milliseconds */
 392
 393      /* if adding retry_after seconds to the process would exceed the
 394         maximum time allowed for retrying, then exit the retries right
 395         away */
 396      if(config->retry_maxtime_ms) {
 397        timediff_t ms = curlx_timediff_ms(curlx_now(), per->retrystart);
 398
 399        if((CURL_OFF_T_MAX - sleeptime < ms) ||
 400           (ms + sleeptime > config->retry_maxtime_ms)) {
 401          warnf("The Retry-After: time would "
 402                "make this command line exceed the maximum allowed time "
 403                "for retries.");
 404          *retryp = FALSE;
 405          return 0; /* no retry */
 406        }
 407      }
 408    }
 409  }
 410  if(!sleeptime && !config->retry_delay_ms) {
 411    if(!per->retry_sleep)
 412      per->retry_sleep = RETRY_SLEEP_DEFAULT;
 413    else
 414      per->retry_sleep *= 2;
 415    if(per->retry_sleep > RETRY_SLEEP_MAX)
 416      per->retry_sleep = RETRY_SLEEP_MAX;
 417  }
 418  if(!sleeptime)
 419    sleeptime = per->retry_sleep;
 420
 421  warnf("Problem %s. Retrying in %u%s%.*u second%s. "
 422        "%ld retr%s left.",
 423        m[retry - 1], sleeptime / 1000,
 424        (sleeptime % 1000 ? "." : ""),
 425        (sleeptime % 1000 ? 3 : 0),
 426        sleeptime % 1000,
 427        (sleeptime == 1000 ? "" : "s"),
 428        per->retry_remaining,
 429        (per->retry_remaining > 1 ? "ies" : "y"));
 430
 431  return sleeptime;
 432}
 433
 434static CURLcode retrycheck(struct OperationConfig *config,
 435                           struct per_transfer *per,
 436                           CURLcode result,
 437                           bool *retryp,
 438                           uint32_t *delayms)
 439{
 440  CURL *curl = per->curl;
 441  struct OutStruct *outs = &per->outs;
 442  enum retryreason reason = RETRY_NO;
 443  if((result == CURLE_OPERATION_TIMEDOUT) ||
 444     (result == CURLE_COULDNT_RESOLVE_HOST) ||
 445     (result == CURLE_COULDNT_RESOLVE_PROXY) ||
 446     (result == CURLE_FTP_ACCEPT_TIMEOUT))
 447    /* retry timeout always */
 448    reason = RETRY_TIMEOUT;
 449  else if(config->retry_connrefused &&
 450          (result == CURLE_COULDNT_CONNECT)) {
 451    long oserrno = 0;
 452    curl_easy_getinfo(curl, CURLINFO_OS_ERRNO, &oserrno);
 453    if(SOCKECONNREFUSED == oserrno)
 454      reason = RETRY_CONNREFUSED;
 455  }
 456  else if((result == CURLE_OK) ||
 457          (config->fail && (result == CURLE_HTTP_RETURNED_ERROR))) {
 458    /* If it returned OK. _or_ failonerror was enabled and it
 459       returned due to such an error, check for HTTP transient
 460       errors to retry on. */
 461    const char *scheme;
 462    curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme);
 463    scheme = proto_token(scheme);
 464    if(scheme == proto_http || scheme == proto_https) {
 465      /* This was HTTP(S) */
 466      long response = 0;
 467      curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
 468
 469      switch(response) {
 470      case 408: /* Request Timeout */
 471      case 429: /* Too Many Requests (RFC6585) */
 472      case 500: /* Internal Server Error */
 473      case 502: /* Bad Gateway */
 474      case 503: /* Service Unavailable */
 475      case 504: /* Gateway Timeout */
 476      case 522: /* Connection Timed Out (Cloudflare) */
 477      case 524: /* Proxy Read Timeout (Cloudflare) */
 478        reason = RETRY_HTTP;
 479        /*
 480         * At this point, we have already written data to the output
 481         * file (or terminal). If we write to a file, we must rewind
 482         * or close/re-open the file so that the next attempt starts
 483         * over from the beginning.
 484         *
 485         * For the upload case, we might need to start over reading from a
 486         * previous point if we have uploaded something when this was
 487         * returned.
 488         */
 489        break;
 490      }
 491    }
 492  } /* if CURLE_OK */
 493  else if(result) {
 494    const char *scheme;
 495    long response = 0;
 496
 497    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
 498    curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme);
 499    scheme = proto_token(scheme);
 500
 501    if((scheme == proto_ftp || scheme == proto_ftps) && response / 100 == 4)
 502      /*
 503       * This is typically when the FTP server only allows a certain
 504       * amount of users and we are not one of them. All 4xx codes
 505       * are transient.
 506       */
 507      reason = RETRY_FTP;
 508  }
 509
 510  if(result && !reason && config->retry_all_errors)
 511    reason = RETRY_ALL_ERRORS;
 512
 513  if(reason) {
 514    bool truncate = TRUE; /* truncate output file */
 515
 516    /* how long to wait until next attempt */
 517    uint32_t sleeptime;
 518
 519    *retryp = TRUE;
 520
 521    sleeptime = retry_sleep(config, per, reason, retryp);
 522    if(!*retryp)
 523      /* no retry */
 524      return CURLE_OK;
 525    per->retry_remaining--;
 526
 527    /* Skip truncation of outfile if auto-resume is enabled for download and
 528       the partially received data is good. Only for HTTP GET requests in
 529       limited circumstances. */
 530    if(is_outfile_auto_resumable(config, per, result)) {
 531      long response = 0;
 532      struct curl_header *header = NULL;
 533      const char *method = NULL, *scheme = NULL;
 534
 535      curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_METHOD, &method);
 536      curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
 537      curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme);
 538      scheme = proto_token(scheme);
 539
 540      if((scheme == proto_http || scheme == proto_https) &&
 541         method && !strcmp(method, "GET") &&
 542         ((response == 206 && config->resume_from) ||
 543          (response == 200 &&
 544           !curl_easy_header(curl, "Accept-Ranges", 0,
 545                             CURLH_HEADER, -1, &header) &&
 546           !strcmp(header->value, "bytes")))) {
 547
 548        notef("Keeping %" CURL_FORMAT_CURL_OFF_T " bytes", outs->bytes);
 549        if(fflush(outs->stream)) {
 550          errorf("Failed to flush output file stream");
 551          return CURLE_WRITE_ERROR;
 552        }
 553        if(outs->bytes >= CURL_OFF_T_MAX - outs->init) {
 554          errorf("Exceeded maximum supported file size ("
 555                 "%" CURL_FORMAT_CURL_OFF_T " + "
 556                 "%" CURL_FORMAT_CURL_OFF_T ")",
 557                 outs->init, outs->bytes);
 558          return CURLE_WRITE_ERROR;
 559        }
 560        truncate = FALSE;
 561        outs->init += outs->bytes;
 562        outs->bytes = 0;
 563        config->resume_from = outs->init;
 564        curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, config->resume_from);
 565      }
 566    }
 567
 568    if(truncate && outs->bytes && outs->filename && outs->stream) {
 569      curlx_struct_stat fileinfo;
 570
 571      /* The output can be a named pipe or a character device etc that
 572         cannot be truncated. Only truncate regular files. */
 573      if(!curlx_fstat(fileno(outs->stream), &fileinfo) &&
 574         S_ISREG(fileinfo.st_mode)) {
 575        int rc;
 576        /* We have written data to an output file, we truncate file */
 577        fflush(outs->stream);
 578        notef("Throwing away %" CURL_FORMAT_CURL_OFF_T " bytes", outs->bytes);
 579
 580        /* truncate file at the position where we started appending */
 581        if(toolx_ftruncate(fileno(outs->stream), outs->init)) {
 582          /* when truncate fails, we cannot append as then we
 583             create something strange, bail out */
 584          errorf("Failed to truncate file");
 585          return CURLE_WRITE_ERROR;
 586        }
 587        /* now seek to the end of the file, the position where we
 588           truncated the file in a large file-safe way */
 589        rc = fseek(outs->stream, 0, SEEK_END);
 590
 591        if(rc) {
 592          errorf("Failed seeking to end of file");
 593          return CURLE_WRITE_ERROR;
 594        }
 595        outs->bytes = 0; /* clear for next round */
 596      }
 597    }
 598    per->num_retries++;
 599    *delayms = sleeptime;
 600    result = CURLE_OK;
 601  }
 602  return result;
 603}
 604
 605static CURLcode post_check_result(struct per_transfer *per, CURLcode result)
 606{
 607  struct OperationConfig *config = per->config;
 608
 609#ifdef __VMS
 610  if(is_vms_shell()) {
 611    /* VMS DCL shell behavior */
 612    if(global->silent && !global->showerror)
 613      vms_show = VMSSTS_HIDE;
 614  }
 615#endif
 616  if(!config->synthetic_error && result &&
 617     (!global->silent || global->showerror)) {
 618    const char *msg = per->errorbuffer;
 619    curl_mfprintf(tool_stderr, "curl: (%d) %s\n", result,
 620                  msg[0] ? msg : curl_easy_strerror(result));
 621    if(result == CURLE_PEER_FAILED_VERIFICATION)
 622      fputs(CURL_CA_CERT_ERRORMSG, tool_stderr);
 623  }
 624  else if(config->fail == FAIL_WITH_BODY) {
 625    /* if HTTP response >= 400, return error */
 626    long code = 0;
 627    curl_easy_getinfo(per->curl, CURLINFO_RESPONSE_CODE, &code);
 628    if(code >= 400) {
 629      if(!global->silent || global->showerror)
 630        curl_mfprintf(tool_stderr,
 631                      "curl: (%d) The requested URL returned error: %ld\n",
 632                      CURLE_HTTP_RETURNED_ERROR, code);
 633      return CURLE_HTTP_RETURNED_ERROR;
 634    }
 635  }
 636  return result;
 637}
 638
 639static CURLcode post_output_handling(struct per_transfer *per,
 640                                     CURLcode result)
 641{
 642  struct OutStruct *outs = &per->outs;
 643  struct OperationConfig *config = per->config;
 644  CURL *curl = per->curl;
 645  int rc;
 646
 647#ifdef _WIN32
 648  /* Discard incomplete UTF-8 sequence buffered from body */
 649  memset(outs->utf8seq, 0, sizeof(outs->utf8seq));
 650#endif
 651
 652  /* Set file extended attributes */
 653  if(!result && config->xattr && outs->fopened && outs->stream) {
 654    rc = fwrite_xattr(curl, per->url, fileno(outs->stream));
 655    if(rc) {
 656      char errbuf[STRERROR_LEN];
 657      warnf("Error setting extended attributes on '%s': %s", outs->filename,
 658            curlx_strerror(errno, errbuf, sizeof(errbuf)));
 659    }
 660  }
 661
 662  if(!result && !outs->stream && !outs->bytes) {
 663    /* we have received no data despite the transfer was successful
 664       ==> force creation of an empty output file (if an output file
 665       was specified) */
 666    long cond_unmet = 0L;
 667    /* do not create (or even overwrite) the file in case we get no
 668       data because of unmet condition */
 669    curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &cond_unmet);
 670    if(!cond_unmet && !tool_create_output_file(outs, config))
 671      return CURLE_WRITE_ERROR;
 672  }
 673
 674  if(!outs->regular_file && outs->stream) {
 675    /* Dump standard stream buffered data */
 676    rc = fflush(outs->stream);
 677    if(!result && rc) {
 678      /* something went wrong in the writing process */
 679      errorf("Failed writing body");
 680      return CURLE_WRITE_ERROR;
 681    }
 682  }
 683
 684  return result;
 685}
 686
 687static CURLcode post_close_output(struct per_transfer *per,
 688                                  CURLcode result)
 689{
 690  struct OutStruct *outs = &per->outs;
 691  struct OperationConfig *config = per->config;
 692  int rc;
 693
 694  /* Close the outs file */
 695  if(outs->fopened && outs->stream) {
 696    rc = curlx_fclose(outs->stream);
 697    if(!result && rc) {
 698      /* something went wrong in the writing process */
 699      result = CURLE_WRITE_ERROR;
 700      errorf("curl: (%d) Failed writing body", result);
 701    }
 702    if(result && config->rm_partial) {
 703      curlx_struct_stat st;
 704      if(!curlx_stat(outs->filename, &st) && S_ISREG(st.st_mode)) {
 705        if(!unlink(outs->filename))
 706          notef("Removed output file: %s", outs->filename);
 707        else
 708          warnf("Failed removing: %s", outs->filename);
 709      }
 710      else
 711        warnf("Skipping removal; not a regular file: %s", outs->filename);
 712    }
 713  }
 714
 715  /* File time can only be set _after_ the file has been closed */
 716  if(!result && config->remote_time && outs->regular_file && outs->filename) {
 717    /* Ask libcurl if we got a remote file time */
 718    curl_off_t filetime = -1;
 719    curl_easy_getinfo(per->curl, CURLINFO_FILETIME_T, &filetime);
 720    if(filetime != -1)
 721      setfiletime(filetime, outs->filename);
 722  }
 723  return result;
 724}
 725/*
 726 * Call this after a transfer has completed.
 727 */
 728static CURLcode post_per_transfer(struct per_transfer *per,
 729                                  CURLcode result,
 730                                  bool *retryp,
 731                                  uint32_t *delay) /* milliseconds! */
 732{
 733  struct OutStruct *outs = &per->outs;
 734  struct OperationConfig *config = per->config;
 735
 736  *retryp = FALSE;
 737  *delay = 0; /* for no retry, keep it zero */
 738
 739  if(!per->curl || !config)
 740    return result;
 741
 742#ifdef _WIN32
 743  if(per->uploadfile) {
 744    if(!strcmp(per->uploadfile, ".") && per->infd > 0) {
 745#ifndef CURL_WINDOWS_UWP
 746      sclose(per->infd);
 747#else
 748      warnf("Closing per->infd != 0: FD == %d. "
 749            "This behavior is only supported on desktop Windows", per->infd);
 750#endif
 751    }
 752  }
 753  else
 754#endif
 755    if(per->infdopen)
 756      curlx_close(per->infd);
 757
 758  if(!per->skip) {
 759    result = post_check_result(per, result);
 760    result = post_output_handling(per, result);
 761
 762    /* if retry-max-time is non-zero, make sure we have not exceeded the
 763       time */
 764    if(per->retry_remaining &&
 765       (!config->retry_maxtime_ms ||
 766        (curlx_timediff_ms(curlx_now(), per->retrystart) <
 767         config->retry_maxtime_ms))) {
 768      result = retrycheck(config, per, result, retryp, delay);
 769      if(!result && *retryp)
 770        return CURLE_OK; /* retry! */
 771    }
 772
 773    if((global->progressmode == CURL_PROGRESS_BAR) &&
 774       per->progressbar.calls)
 775      /* if the custom progress bar has been displayed, we output a
 776         newline here */
 777      fputs("\n", per->progressbar.out);
 778
 779    result = post_close_output(per, result);
 780  }
 781  /* Write the --write-out data before cleanup but after result is final */
 782  if(config->writeout)
 783    ourWriteOut(config, per, result);
 784
 785  /* Close function-local opened file descriptors */
 786  if(per->heads.fopened && per->heads.stream)
 787    curlx_fclose(per->heads.stream);
 788
 789  if(per->heads.alloc_filename)
 790    curlx_safefree(per->heads.filename);
 791
 792  if(per->etag_save.fopened && per->etag_save.stream)
 793    curlx_fclose(per->etag_save.stream);
 794
 795  if(per->etag_save.alloc_filename)
 796    curlx_safefree(per->etag_save.filename);
 797
 798  if(outs->alloc_filename)
 799    curlx_safefree(outs->filename);
 800  curl_slist_free_all(per->hdrcbdata.headlist);
 801  per->hdrcbdata.headlist = NULL;
 802  return result;
 803}
 804
 805static CURLcode set_cert_types(struct OperationConfig *config)
 806{
 807  if(feature_ssl) {
 808    /* Check if config->cert is a PKCS#11 URI and set the config->cert_type if
 809     * necessary */
 810    if(config->cert && !config->cert_type && is_pkcs11_uri(config->cert)) {
 811      config->cert_type = curlx_strdup("ENG");
 812      if(!config->cert_type)
 813        return CURLE_OUT_OF_MEMORY;
 814    }
 815
 816    /* Check if config->key is a PKCS#11 URI and set the config->key_type if
 817     * necessary */
 818    if(config->key && !config->key_type && is_pkcs11_uri(config->key)) {
 819      config->key_type = curlx_strdup("ENG");
 820      if(!config->key_type)
 821        return CURLE_OUT_OF_MEMORY;
 822    }
 823
 824    /* Check if config->proxy_cert is a PKCS#11 URI and set the
 825     * config->proxy_type if necessary */
 826    if(config->proxy_cert && !config->proxy_cert_type &&
 827       is_pkcs11_uri(config->proxy_cert)) {
 828      config->proxy_cert_type = curlx_strdup("ENG");
 829      if(!config->proxy_cert_type)
 830        return CURLE_OUT_OF_MEMORY;
 831    }
 832
 833    /* Check if config->proxy_key is a PKCS#11 URI and set the
 834     * config->proxy_key_type if necessary */
 835    if(config->proxy_key && !config->proxy_key_type &&
 836       is_pkcs11_uri(config->proxy_key)) {
 837      config->proxy_key_type = curlx_strdup("ENG");
 838      if(!config->proxy_key_type)
 839        return CURLE_OUT_OF_MEMORY;
 840    }
 841  }
 842  return CURLE_OK;
 843}
 844
 845static CURLcode append2query(struct OperationConfig *config,
 846                             struct per_transfer *per,
 847                             const char *q)
 848{
 849  CURLcode result = CURLE_OK;
 850  if(q) {
 851    CURLU *uh = curl_url();
 852    if(uh) {
 853      CURLUcode uerr;
 854      uerr = curl_url_set(uh, CURLUPART_URL, per->url, CURLU_GUESS_SCHEME);
 855      if(uerr) {
 856        result = urlerr_cvt(uerr);
 857        errorf("(%d) Could not parse the URL, "
 858               "failed to set query", result);
 859        config->synthetic_error = TRUE;
 860      }
 861      else {
 862        char *updated = NULL;
 863        uerr = curl_url_set(uh, CURLUPART_QUERY, q, CURLU_APPENDQUERY);
 864        if(!uerr)
 865          uerr = curl_url_get(uh, CURLUPART_URL, &updated,
 866                              CURLU_NO_GUESS_SCHEME);
 867        if(uerr)
 868          result = urlerr_cvt(uerr);
 869        else {
 870          /* use our new URL instead! */
 871          curlx_free(per->url); /* free previous URL */
 872          /* make it allocated by tool memory */
 873          per->url = curlx_strdup(updated);
 874          curl_free(updated);
 875          if(!per->url)
 876            result = CURLE_OUT_OF_MEMORY;
 877        }
 878      }
 879      curl_url_cleanup(uh);
 880    }
 881    else
 882      result = CURLE_OUT_OF_MEMORY;
 883  }
 884  return result;
 885}
 886
 887static CURLcode etag_compare(struct OperationConfig *config)
 888{
 889  CURLcode result = CURLE_OK;
 890  char *etag_from_file = NULL;
 891  char *header = NULL;
 892  ParameterError pe;
 893
 894  /* open file for reading: */
 895  FILE *file = curlx_fopen(config->etag_compare_file, FOPEN_READTEXT);
 896  if(!file) {
 897    char errbuf[STRERROR_LEN];
 898    warnf("Failed to open %s: %s", config->etag_compare_file,
 899          curlx_strerror(errno, errbuf, sizeof(errbuf)));
 900  }
 901
 902  if((PARAM_OK == file2string(&etag_from_file, file)) &&
 903     etag_from_file) {
 904    char *h = curl_maprintf("If-None-Match: %s", etag_from_file);
 905    curlx_safefree(etag_from_file);
 906    if(h) {
 907      /* move it to the right memory */
 908      header = curlx_strdup(h);
 909      curl_free(h);
 910    }
 911  }
 912  else
 913    header = curlx_strdup("If-None-Match: \"\"");
 914
 915  if(!header) {
 916    if(file)
 917      curlx_fclose(file);
 918    errorf("Failed to allocate memory for custom etag header");
 919    return CURLE_OUT_OF_MEMORY;
 920  }
 921
 922  /* add Etag from file to list of custom headers */
 923  pe = add2list(&config->headers, header);
 924  curlx_free(header);
 925
 926  if(file)
 927    curlx_fclose(file);
 928  if(pe != PARAM_OK)
 929    result = CURLE_OUT_OF_MEMORY;
 930  return result;
 931}
 932
 933static CURLcode etag_store(struct OperationConfig *config,
 934                           struct OutStruct *etag_save,
 935                           bool *skip)
 936{
 937  if(config->create_dirs) {
 938    CURLcode result = create_dir_hierarchy(config->etag_save_file);
 939    if(result)
 940      return result;
 941  }
 942
 943  /* open file for output: */
 944  if(strcmp(config->etag_save_file, "-")) {
 945    FILE *newfile = curlx_fopen(config->etag_save_file, "ab");
 946    if(!newfile) {
 947      warnf("Failed creating file for saving etags: \"%s\". "
 948            "Skip this transfer", config->etag_save_file);
 949      *skip = TRUE;
 950      return CURLE_OK;
 951    }
 952    else {
 953      etag_save->filename = config->etag_save_file;
 954      etag_save->regular_file = TRUE;
 955      etag_save->fopened = TRUE;
 956      etag_save->stream = newfile;
 957    }
 958  }
 959  else {
 960    /* always use binary mode for protocol header output */
 961    CURL_BINMODE(etag_save->stream);
 962  }
 963  return CURLE_OK;
 964}
 965
 966static CURLcode setup_headerfile(struct OperationConfig *config,
 967                                 struct per_transfer *per,
 968                                 struct OutStruct *heads)
 969{
 970  if(config->headerfile) {
 971    /* open file for output: */
 972    if(!strcmp(config->headerfile, "%")) {
 973      heads->stream = stderr;
 974      /* use binary mode for protocol header output */
 975      CURL_BINMODE(heads->stream);
 976    }
 977    else if(strcmp(config->headerfile, "-")) {
 978      FILE *newfile;
 979
 980      /*
 981       * Since every transfer has its own file handle for dumping
 982       * the headers, we need to open it in append mode, since transfers
 983       * might finish in any order.
 984       * The first transfer clears the file.
 985       *
 986       * Consider placing the file handle inside the OperationConfig, so
 987       * that it does not need to be opened/closed for every transfer.
 988       */
 989      if(config->create_dirs) {
 990        CURLcode result = create_dir_hierarchy(config->headerfile);
 991        /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */
 992        if(result)
 993          return result;
 994      }
 995      if(!per->prev || per->prev->config != config) {
 996        newfile = curlx_fopen(config->headerfile, "wb");
 997        if(newfile)
 998          curlx_fclose(newfile);
 999      }
1000      newfile = curlx_fopen(config->headerfile, "ab");
1001
1002      if(!newfile) {
1003        errorf("Failed to open %s", config->headerfile);
1004        return CURLE_WRITE_ERROR;
1005      }
1006      else {
1007        heads->filename = config->headerfile;
1008        heads->regular_file = TRUE;
1009        heads->fopened = TRUE;
1010        heads->stream = newfile;
1011      }
1012    }
1013    else {
1014      /* always use binary mode for protocol header output */
1015      CURL_BINMODE(heads->stream);
1016    }
1017  }
1018  return CURLE_OK;
1019}
1020
1021/* figure out and populate per->outfile */
1022static CURLcode setup_outfile(struct OperationConfig *config,
1023                              struct per_transfer *per,
1024                              struct getout *u,
1025                              struct OutStruct *outs,
1026                              bool *skipped)
1027{
1028  struct State *state = &global->state;
1029
1030  outs->out_null = u->out_null;
1031  /* if --out-null or output to stdout, return */
1032  if(outs->out_null ||
1033     (!u->useremote && (!u->outfile || !strcmp("-", u->outfile)))) {
1034    per->outfile = u->outfile;
1035    u->outfile = NULL;
1036    return CURLE_OK;
1037  }
1038
1039  if(!u->outfile) {
1040    SANITIZEcode sc;
1041    /* extract the filename from the URL */
1042    CURLcode result = get_url_file_name(&per->outfile, per->url, &sc);
1043    if(sc) {
1044      if(sc == SANITIZE_ERR_OUT_OF_MEMORY)
1045        return CURLE_OUT_OF_MEMORY;
1046      if(sc == SANITIZE_ERR_INVALID_PATH)
1047        errorf("bad output filename");
1048      return result;
1049    }
1050    else if(result) {
1051      errorf("Failed to extract a filename from the URL to use for storage");
1052      return result;
1053    }
1054  }
1055  else if(glob_inuse(&state->urlglob) || glob_inuse(&state->inglob)) {
1056    /* expand '#1' ... '#9' references from URL pattern and named references
1057       from the upload file glob */
1058    SANITIZEcode sc;
1059    CURLcode result =
1060      glob_match_url(&per->outfile, u->outfile, &state->urlglob,
1061                     glob_inuse(&state->inglob) ? &state->inglob : NULL, &sc);
1062
1063    if(sc) {
1064      if(sc == SANITIZE_ERR_OUT_OF_MEMORY)
1065        return CURLE_OUT_OF_MEMORY;
1066      if(sc == SANITIZE_ERR_INVALID_PATH)
1067        warnf("bad output filename");
1068      return result;
1069    }
1070    else if(result) {
1071      /* bad globbing */
1072      if(state->urlglob.error) {
1073        glob_show_error(&state->urlglob, u->outfile, tool_stderr, result);
1074        config->synthetic_error = TRUE;
1075      }
1076      else
1077        warnf("bad output glob");
1078      return result;
1079    }
1080    if(!*per->outfile) {
1081      warnf("output glob produces empty string");
1082      return CURLE_WRITE_ERROR;
1083    }
1084  }
1085  else {
1086    per->outfile = curlx_strdup(u->outfile);
1087    if(!per->outfile)
1088      return CURLE_OUT_OF_MEMORY;
1089  }
1090
1091  DEBUGASSERT(per->outfile);
1092
1093  if(config->output_dir && *config->output_dir) {
1094    char *d = curl_maprintf("%s/%s", config->output_dir, per->outfile);
1095    curlx_safefree(per->outfile);
1096    if(d) {
1097      per->outfile = curlx_strdup(d); /* move to right memory */
1098      curl_free(d);
1099    }
1100    if(!d || !per->outfile)
1101      return CURLE_WRITE_ERROR;
1102  }
1103  /* Create the directory hierarchy, if not pre-existent to a multiple
1104     file output call */
1105
1106  if(config->create_dirs) {
1107    CURLcode result = create_dir_hierarchy(per->outfile);
1108    /* create_dir_hierarchy shows error upon CURLE_WRITE_ERROR */
1109    if(result)
1110      return result;
1111  }
1112
1113  if(config->skip_existing) {
1114    curlx_struct_stat fileinfo;
1115    if(!curlx_stat(per->outfile, &fileinfo)) {
1116      /* file is present */
1117      notef("skips transfer, \"%s\" exists locally", per->outfile);
1118      per->skip = TRUE;
1119      *skipped = TRUE;
1120    }
1121  }
1122
1123  if(config->resume_from_current) {
1124    /* We are told to continue from where we are now. Get the size
1125       of the file as it is now and open it for append instead */
1126    curlx_struct_stat fileinfo;
1127    /* VMS -- Danger, the filesize is only valid for stream files */
1128    if(curlx_stat(per->outfile, &fileinfo) == 0)
1129      /* set offset to current file size: */
1130      config->resume_from = fileinfo.st_size;
1131    else
1132      /* let offset be 0 */
1133      config->resume_from = 0;
1134  }
1135
1136  if(config->resume_from) {
1137#ifdef __VMS
1138    /* open file for output, forcing VMS output format into stream
1139       mode which is needed for stat() call above to always work. */
1140    FILE *file = curlx_fopen(per->outfile, "ab",
1141                             "ctx=stm", "rfm=stmlf", "rat=cr", "mrs=0");
1142#else
1143    /* open file for output: */
1144    FILE *file = curlx_fopen(per->outfile, "ab");
1145#endif
1146    if(!file) {
1147      errorf("cannot open '%s'", per->outfile);
1148      return CURLE_WRITE_ERROR;
1149    }
1150    outs->fopened = TRUE;
1151    outs->stream = file;
1152    outs->init = config->resume_from;
1153  }
1154  else {
1155    outs->stream = NULL; /* open when needed */
1156  }
1157  outs->filename = per->outfile;
1158  outs->regular_file = TRUE;
1159  return CURLE_OK;
1160}
1161
1162static void check_stdin_upload(struct OperationConfig *config,
1163                               struct per_transfer *per)
1164{
1165  /* count to see if there are more than one auth bit set
1166     in the authtype field */
1167  int authbits = 0;
1168  int bitcheck = 0;
1169  while(bitcheck < 32) {
1170    if(config->authtype & (1UL << bitcheck++)) {
1171      authbits++;
1172      if(authbits > 1) {
1173        /* more than one, we are done! */
1174        break;
1175      }
1176    }
1177  }
1178
1179  /*
1180   * If the user has also selected --anyauth or --proxy-anyauth
1181   * we should warn them.
1182   */
1183  if(config->proxyanyauth || (authbits > 1)) {
1184    warnf("Using --anyauth or --proxy-anyauth with upload from stdin"
1185          " involves a big risk of it not working. Use a temporary"
1186          " file or a fixed auth type instead");
1187  }
1188
1189  DEBUGASSERT(per->infdopen == FALSE);
1190  DEBUGASSERT(per->infd == STDIN_FILENO);
1191
1192  CURL_BINMODE(stdin);
1193  if(!strcmp(per->uploadfile, ".")) {
1194#if defined(USE_WINSOCK) && !defined(CURL_WINDOWS_UWP)
1195    /* non-blocking stdin behavior on Windows is challenging
1196       Spawn a new thread that reads from stdin and write
1197       out to a socket */
1198    curl_socket_t f = win32_stdin_read_thread();
1199
1200    if(f == CURL_SOCKET_BAD)
1201      warnf("win32_stdin_read_thread returned INVALID_SOCKET "
1202            "falling back to blocking mode");
1203    else if(f > INT_MAX) {
1204      warnf("win32_stdin_read_thread returned identifier "
1205            "larger than INT_MAX. This should not happen unless "
1206            "the upper 32 bits of a Windows socket have started "
1207            "being used for something... falling back to blocking "
1208            "mode");
1209      sclose(f);
1210    }
1211    else
1212      per->infd = (int)f;
1213#endif
1214    if(curlx_nonblock((curl_socket_t)per->infd, TRUE) < 0) {
1215      char errbuf[STRERROR_LEN];
1216      warnf("fcntl failed on fd=%d: %s", per->infd,
1217            curlx_strerror(errno, errbuf, sizeof(errbuf)));
1218    }
1219  }
1220}
1221
1222static void setup_header_cb(struct OperationConfig *config,
1223                            struct HdrCbData *hdrcbdata,
1224                            struct getout *u,
1225                            struct OutStruct *outs,
1226                            struct OutStruct *heads,
1227                            struct OutStruct *etag)
1228{
1229  hdrcbdata->honor_cd_filename = (config->content_disposition && u->useremote);
1230  hdrcbdata->outs = outs;
1231  hdrcbdata->heads = heads;
1232  hdrcbdata->etag_save = etag;
1233  hdrcbdata->config = config;
1234}
1235
1236static CURLcode setup_input_file(struct OperationConfig *config,
1237                                 struct State *state, struct getout *u,
1238                                 FILE *err)
1239{
1240  CURLcode result = CURLE_OK;
1241  if(u->infile) {
1242    if(!config->globoff && !glob_inuse(&state->inglob))
1243      result = glob_url(&state->inglob, u->infile, &state->upnum, err);
1244    if(!result && !state->uploadfile) {
1245      if(glob_inuse(&state->inglob))
1246        result = glob_next_url(&state->uploadfile, &state->inglob);
1247      else if(!state->upidx) {
1248        /* copy the allocated string */
1249        state->uploadfile = u->infile;
1250        u->infile = NULL;
1251      }
1252    }
1253  }
1254  return result;
1255}
1256
1257static CURLcode setup_url_pattern(struct OperationConfig *config,
1258                                  struct State *state, struct getout *u,
1259                                  FILE *err)
1260{
1261  CURLcode result = CURLE_OK;
1262  if(!state->urlnum) {
1263    if(!config->globoff && !u->noglob) {
1264      /* Unless explicitly shut off, we expand '{...}' and '[...]'
1265         expressions and return total number of URLs in pattern set */
1266      result = glob_url(&state->urlglob, u->url, &state->urlnum, err);
1267    }
1268    else
1269      state->urlnum = 1; /* without globbing, this is a single URL */
1270  }
1271  return result;
1272}
1273
1274static CURLcode setup_etag_files(struct OperationConfig *config,
1275                                 struct OutStruct *etag_save,
1276                                 bool *break_loop)
1277{
1278  CURLcode result = CURLE_OK;
1279  *break_loop = FALSE;
1280
1281  /* --etag-save */
1282  memset(etag_save, 0, sizeof(*etag_save));
1283  etag_save->stream = stdout;
1284
1285  /* --etag-compare */
1286  if(config->etag_compare_file) {
1287    result = etag_compare(config);
1288    if(result)
1289      return result;
1290  }
1291
1292  if(config->etag_save_file) {
1293    bool badetag = FALSE;
1294    result = etag_store(config, etag_save, &badetag);
1295    if(result || badetag) {
1296      *break_loop = TRUE;
1297      return result;
1298    }
1299  }
1300  return result;
1301}
1302
1303static CURLcode select_next_url(struct State *state,
1304                                struct getout *u,
1305                                char **url)
1306{
1307  CURLcode result = CURLE_OK;
1308  if(glob_inuse(&state->urlglob))
1309    result = glob_next_url(url, &state->urlglob);
1310  else if(!state->urlidx) {
1311    *url = curlx_strdup(u->url);
1312    if(!*url)
1313      result = CURLE_OUT_OF_MEMORY;
1314  }
1315  else
1316    *url = NULL;
1317  return result;
1318}
1319
1320static CURLcode setup_transfer_upload(struct OperationConfig *config,
1321                                      struct per_transfer *per)
1322{
1323  CURLcode result = CURLE_OK;
1324  if(per->uploadfile) {
1325    if(stdin_upload(per->uploadfile))
1326      check_stdin_upload(config, per);
1327    else {
1328      /* We have specified a file to upload and it is not "-" */
1329      result = add_file_name_to_url(per->curl, &per->url, per->uploadfile);
1330      if(result)
1331        return result;
1332    }
1333
1334    if(config->resume_from_current)
1335      config->resume_from = -1; /* -1 then forces get-it-yourself */
1336  }
1337  return result;
1338}
1339
1340/* create a transfer */
1341static CURLcode create_single(struct OperationConfig *config,
1342                              CURLSH *share, struct State *state,
1343                              bool *added, bool *skipped)
1344{
1345  const bool orig_isatty = (bool)global->isatty;
1346  const bool orig_noprogress = (bool)global->noprogress;
1347  CURLcode result = CURLE_OK;
1348  while(state->urlnode) {
1349    struct per_transfer *per = NULL;
1350    struct OutStruct *outs;
1351    struct OutStruct *heads;
1352    struct OutStruct etag_first;
1353    CURL *curl;
1354    struct getout *u = state->urlnode;
1355    FILE *err = (!global->silent || global->showerror) ? tool_stderr : NULL;
1356    bool break_loop = FALSE;
1357
1358    if(!u->url) {
1359      /* This node has no URL. End of the road. */
1360      warnf("Got more output options than URLs");
1361      break;
1362    }
1363
1364    result = setup_input_file(config, state, u, err);
1365    if(result)
1366      return result;
1367
1368    if(state->upidx >= state->upnum) {
1369      state->urlnum = 0;
1370      curlx_safefree(state->uploadfile);
1371      glob_cleanup(&state->inglob);
1372      state->upidx = 0;
1373      state->urlnode = u->next; /* next node */
1374      state->upnum = 1;
1375      continue;
1376    }
1377
1378    result = setup_url_pattern(config, state, u, err);
1379    if(result)
1380      return result;
1381
1382    result = setup_etag_files(config, &etag_first, &break_loop);
1383    if(result || break_loop)
1384      return result;
1385
1386    curl = curl_easy_init();
1387    if(curl)
1388      result = add_per_transfer(&per);
1389    else
1390      result = CURLE_OUT_OF_MEMORY;
1391    if(result) {
1392      curl_easy_cleanup(curl);
1393      if(etag_first.fopened)
1394        curlx_fclose(etag_first.stream);
1395      return result;
1396    }
1397    per->etag_save = etag_first; /* copy the whole struct */
1398    if(state->uploadfile) {
1399      per->uploadfile = curlx_strdup(state->uploadfile);
1400      if(!per->uploadfile ||
1401         SetHTTPrequest(TOOL_HTTPREQ_PUT, &config->httpreq)) {
1402        curlx_safefree(per->uploadfile);
1403        curl_easy_cleanup(curl);
1404        return CURLE_FAILED_INIT;
1405      }
1406    }
1407    per->config = config;
1408    per->curl = curl;
1409    per->urlnum = u->num;
1410
1411    /* default headers output stream is stdout */
1412    heads = &per->heads;
1413    heads->stream = stdout;
1414
1415    /* Single header file for all URLs */
1416    result = setup_headerfile(config, per, heads);
1417    if(result)
1418      return result;
1419    outs = &per->outs;
1420
1421    per->infdopen = FALSE;
1422    per->infd = STDIN_FILENO;
1423
1424    /* default output stream is stdout */
1425    outs->stream = stdout;
1426
1427    result = select_next_url(state, u, &per->url);
1428    if(result)
1429      return result;
1430    if(!per->url)
1431      break;
1432
1433    result = setup_outfile(config, per, u, outs, skipped);
1434    if(result)
1435      return result;
1436
1437    result = setup_transfer_upload(config, per);
1438    if(result)
1439      return result;
1440
1441    if(!outs->out_null && output_expected(per->url, per->uploadfile) &&
1442       outs->stream && isatty(fileno(outs->stream)))
1443      /* we send the output to a tty, therefore we switch off the progress
1444         meter */
1445      per->noprogress = global->noprogress = global->isatty = TRUE;
1446    else {
1447      /* progress meter is per download, so restore config values */
1448      per->noprogress = global->noprogress = orig_noprogress;
1449      global->isatty = orig_isatty;
1450    }
1451
1452    result = append2query(config, per, config->httpgetfields);
1453    if(result)
1454      return result;
1455
1456    if((!per->outfile || !strcmp(per->outfile, "-")) &&
1457       !config->use_ascii) {
1458      /* We get the output to stdout and we have not got the ASCII/text flag,
1459         then set stdout to be binary */
1460      CURL_BINMODE(stdout);
1461    }
1462
1463    /* explicitly passed to stdout means okaying binary gunk */
1464    config->terminal_binary_ok = (per->outfile && !strcmp(per->outfile, "-"));
1465
1466    setup_header_cb(config, &per->hdrcbdata, u, outs, heads, &etag_first);
1467
1468    result = config2setopts(config, per, curl, share);
1469    if(result)
1470      return result;
1471
1472    /* initialize retry vars for loop below */
1473    per->retry_sleep_default = config->retry_delay_ms;
1474    per->retry_remaining = config->req_retry;
1475    per->retry_sleep = per->retry_sleep_default; /* ms */
1476    per->retrystart = curlx_now();
1477
1478    /* Here's looping around each globbed URL */
1479    if(++state->urlidx >= state->urlnum) {
1480      state->urlidx = state->urlnum = 0;
1481      glob_cleanup(&state->urlglob);
1482      state->upidx++;
1483      curlx_safefree(state->uploadfile); /* clear it to get the next */
1484    }
1485    *added = TRUE;
1486    break;
1487  }
1488  return result;
1489}
1490
1491/* create the next (singular) transfer */
1492static CURLcode single_transfer(struct OperationConfig *config,
1493                                CURLSH *share, bool *added, bool *skipped)
1494{
1495  CURLcode result = CURLE_OK;
1496  struct State *state = &global->state;
1497  char *httpgetfields = config->httpgetfields;
1498
1499  *skipped = *added = FALSE; /* not yet */
1500
1501  if(config->postfields) {
1502    if(config->use_httpget) {
1503      if(!httpgetfields) {
1504        /* Use the postfields data for an HTTP get */
1505        httpgetfields = config->httpgetfields = config->postfields;
1506        config->postfields = NULL;
1507        if(SetHTTPrequest((config->no_body ? TOOL_HTTPREQ_HEAD :
1508                           TOOL_HTTPREQ_GET), &config->httpreq))
1509          return CURLE_FAILED_INIT;
1510      }
1511    }
1512    else if(SetHTTPrequest(TOOL_HTTPREQ_SIMPLEPOST, &config->httpreq))
1513      return CURLE_FAILED_INIT;
1514  }
1515  if(!httpgetfields)
1516    config->httpgetfields = config->query;
1517
1518  result = set_cert_types(config);
1519  if(result)
1520    return result;
1521
1522  if(!state->urlnode) {
1523    /* first time caller, setup things */
1524    state->urlnode = config->url_list;
1525    state->upnum = 1;
1526  }
1527
1528  return create_single(config, share, state, added, skipped);
1529}
1530
1531static long all_added; /* number of easy handles currently added */
1532
1533/*
1534 * add_parallel_transfers() sets 'morep' to TRUE if there are more transfers
1535 * to add even after this call returns. sets 'addedp' to TRUE if one or more
1536 * transfers were added.
1537 */
1538static CURLcode add_parallel_transfers(CURLM *multi, CURLSH *share,
1539                                       bool *morep, bool *addedp)
1540{
1541  struct per_transfer *per;
1542  CURLcode result = CURLE_OK;
1543  CURLMcode mcode;
1544  bool sleeping = FALSE;
1545  curl_off_t nxfers;
1546
1547  *addedp = FALSE;
1548  *morep = FALSE;
1549  mcode = curl_multi_get_offt(multi, CURLMINFO_XFERS_CURRENT, &nxfers);
1550  if(mcode) {
1551    DEBUGASSERT(0);
1552    return CURLE_UNKNOWN_OPTION;
1553  }
1554
1555  if(nxfers < (curl_off_t)(global->parallel_max * 2)) {
1556    bool skipped = FALSE;
1557    do {
1558      result = create_transfer(share, addedp, &skipped);
1559      if(result)
1560        return result;
1561    } while(skipped);
1562  }
1563  for(per = transfers; per && (all_added < global->parallel_max);
1564      per = per->next) {
1565    if(per->added || per->skip)
1566      /* already added or to be skipped */
1567      continue;
1568    if(per->startat && (time(NULL) < per->startat)) {
1569      /* this is still delaying */
1570      sleeping = TRUE;
1571      continue;
1572    }
1573    per->added = TRUE;
1574
1575    result = pre_transfer(per);
1576    if(result)
1577      return result;
1578
1579    /* parallel connect means that we do not set PIPEWAIT since pipewait
1580       makes libcurl prefer multiplexing */
1581    (void)curl_easy_setopt(per->curl, CURLOPT_PIPEWAIT,
1582                           global->parallel_connect ? 0L : 1L);
1583    (void)curl_easy_setopt(per->curl, CURLOPT_PRIVATE, per);
1584    /* curl does not use signals, switching this on saves some system calls */
1585    (void)curl_easy_setopt(per->curl, CURLOPT_NOSIGNAL, 1L);
1586    (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_cb);
1587    (void)curl_easy_setopt(per->curl, CURLOPT_XFERINFODATA, per);
1588    (void)curl_easy_setopt(per->curl, CURLOPT_NOPROGRESS, 0L);
1589    (void)curl_easy_setopt(per->curl, CURLOPT_ERRORBUFFER, per->errorbuffer);
1590#ifdef DEBUGBUILD
1591    if(getenv("CURL_FORBID_REUSE"))
1592      (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L);
1593#endif
1594
1595    mcode = curl_multi_add_handle(multi, per->curl);
1596    if(mcode) {
1597      DEBUGASSERT(mcode == CURLM_OUT_OF_MEMORY);
1598      result = CURLE_OUT_OF_MEMORY;
1599    }
1600
1601    if(!result) {
1602      bool getadded = FALSE;
1603      bool skipped = FALSE;
1604      do {
1605        result = create_transfer(share, &getadded, &skipped);
1606        if(result)
1607          break;
1608      } while(skipped);
1609    }
1610    if(result)
1611      return result;
1612
1613    per->errorbuffer[0] = 0;
1614    per->added = TRUE;
1615    all_added++;
1616    *addedp = TRUE;
1617  }
1618  *morep = (per || sleeping);
1619  return CURLE_OK;
1620}
1621
1622struct parastate {
1623  CURLM *multi;
1624  CURLSH *share;
1625  CURLMcode mcode;
1626  CURLcode result;
1627  int still_running;
1628  struct curltime start;
1629  bool more_transfers;
1630  bool added_transfers;
1631  /* wrapitup is set TRUE after a critical error occurs to end all transfers */
1632  bool wrapitup;
1633  /* wrapitup_processed is set TRUE after the per transfer abort flag is set */
1634  bool wrapitup_processed;
1635  time_t tick;
1636};
1637
1638#if defined(DEBUGBUILD) && defined(USE_LIBUV)
1639
1640#define DEBUG_UV 0
1641
1642/* object to pass to the callbacks */
1643struct datauv {
1644  uv_timer_t timeout;
1645  uv_loop_t *loop;
1646  struct parastate *s;
1647};
1648
1649struct contextuv {
1650  uv_poll_t poll_handle;
1651  curl_socket_t sockfd;
1652  struct datauv *uv;
1653};
1654
1655static void mnotify(CURLM *multi, unsigned int notification,
1656                    CURL *easy, void *user_data);
1657
1658/* callback from libuv on socket activity */
1659static void on_uv_socket(uv_poll_t *req, int status, int events)
1660{
1661  int flags = 0;
1662  struct contextuv *c = (struct contextuv *)req->data;
1663  (void)status;
1664  if(events & UV_READABLE)
1665    flags |= CURL_CSELECT_IN;
1666  if(events & UV_WRITABLE)
1667    flags |= CURL_CSELECT_OUT;
1668
1669  curl_multi_socket_action(c->uv->s->multi, c->sockfd, flags,
1670                           &c->uv->s->still_running);
1671  progress_meter(c->uv->s->multi, &c->uv->s->start, FALSE);
1672}
1673
1674/* callback from libuv when timeout expires */
1675static void on_uv_timeout(uv_timer_t *req)
1676{
1677  struct datauv *uv = (struct datauv *)req->data;
1678#if DEBUG_UV
1679  curl_mfprintf(tool_stderr, "parallel_event: on_uv_timeout\n");
1680#endif
1681  if(uv && uv->s) {
1682    curl_multi_socket_action(uv->s->multi, CURL_SOCKET_TIMEOUT, 0,
1683                             &uv->s->still_running);
1684    progress_meter(uv->s->multi, &uv->s->start, FALSE);
1685  }
1686}
1687
1688/* callback from libcurl to update the timeout expiry */
1689static int cb_timeout(CURLM *multi, long timeout_ms, void *userp)
1690{
1691  struct datauv *uv = userp;
1692  (void)multi;
1693#if DEBUG_UV
1694  curl_mfprintf(tool_stderr, "parallel_event: cb_timeout=%ld\n", timeout_ms);
1695#endif
1696  if(timeout_ms < 0)
1697    uv_timer_stop(&uv->timeout);
1698  else {
1699    if(timeout_ms == 0)
1700      timeout_ms = 1; /* 0 means call curl_multi_socket_action asap but NOT
1701                         within the callback itself */
1702    uv_timer_start(&uv->timeout, on_uv_timeout, timeout_ms,
1703                   0); /* do not repeat */
1704  }
1705  return 0;
1706}
1707
1708static struct contextuv *create_context(curl_socket_t sockfd,
1709                                        struct datauv *uv)
1710{
1711  struct contextuv *c;
1712
1713  c = (struct contextuv *)curlx_malloc(sizeof(*c));
1714
1715  c->sockfd = sockfd;
1716  c->uv = uv;
1717
1718  uv_poll_init_socket(uv->loop, &c->poll_handle, sockfd);
1719  c->poll_handle.data = c;
1720
1721  return c;
1722}
1723
1724static void close_cb(uv_handle_t *handle)
1725{
1726  struct contextuv *c = (struct contextuv *)handle->data;
1727  curlx_free(c);
1728}
1729
1730static void destroy_context(struct contextuv *c)
1731{
1732  uv_close((uv_handle_t *)&c->poll_handle, close_cb);
1733}
1734
1735/* callback from libcurl to update socket activity to wait for */
1736static int cb_socket(CURL *easy, curl_socket_t s, int action,
1737                     void *userp, void *socketp)
1738{
1739  struct contextuv *c;
1740  int events = 0;
1741  struct datauv *uv = userp;
1742  (void)easy;
1743
1744#if DEBUG_UV
1745  curl_mfprintf(tool_stderr, "parallel_event: cb_socket, "
1746                "fd=%" FMT_SOCKET_T ", action=%x, p=%p\n", s, action, socketp);
1747#endif
1748  switch(action) {
1749  case CURL_POLL_IN:
1750  case CURL_POLL_OUT:
1751  case CURL_POLL_INOUT:
1752    c = socketp ? (struct contextuv *)socketp : create_context(s, uv);
1753
1754    curl_multi_assign(uv->s->multi, s, c);
1755
1756    if(action != CURL_POLL_IN)
1757      events |= UV_WRITABLE;
1758    if(action != CURL_POLL_OUT)
1759      events |= UV_READABLE;
1760
1761    uv_poll_start(&c->poll_handle, events, on_uv_socket);
1762    break;
1763  case CURL_POLL_REMOVE:
1764    if(socketp) {
1765      c = (struct contextuv *)socketp;
1766      uv_poll_stop(&c->poll_handle);
1767      destroy_context(c);
1768      curl_multi_assign(uv->s->multi, s, NULL);
1769    }
1770    break;
1771  default:
1772    abort();
1773  }
1774
1775  return 0;
1776}
1777
1778static CURLcode parallel_event(struct parastate *s)
1779{
1780  CURLcode result = CURLE_OK;
1781  struct datauv uv = { 0 };
1782
1783  s->result = CURLE_OK;
1784  uv.s = s;
1785  uv.loop = uv_default_loop();
1786  uv_timer_init(uv.loop, &uv.timeout);
1787  uv.timeout.data = &uv;
1788
1789  /* setup event callbacks */
1790  curl_multi_setopt(s->multi, CURLMOPT_SOCKETFUNCTION, cb_socket);
1791  curl_multi_setopt(s->multi, CURLMOPT_SOCKETDATA, &uv);
1792  curl_multi_setopt(s->multi, CURLMOPT_TIMERFUNCTION, cb_timeout);
1793  curl_multi_setopt(s->multi, CURLMOPT_TIMERDATA, &uv);
1794  curl_multi_setopt(s->multi, CURLMOPT_MAX_HOST_CONNECTIONS, (long)
1795                    global->parallel_host);
1796
1797  /* kickstart the thing */
1798  curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0,
1799                           &s->still_running);
1800
1801  while(!s->mcode && (s->still_running || s->more_transfers)) {
1802
1803    if(!s->still_running && s->more_transfers) {
1804      curl_off_t nxfers;
1805      /* None are running, but there may be more ready to run. Try
1806       * to add more when possible. */
1807      s->result = add_parallel_transfers(s->multi, s->share,
1808                                         &s->more_transfers,
1809                                         &s->added_transfers);
1810      if(s->result)
1811        break;
1812      if(s->more_transfers) {
1813        /* There are still more transfers, so not all were added. If we
1814         * did not add *any*, it means there is nothing to do until time
1815         * passes and a transfer becomes ready to add.
1816         * Avoid busy looping by taking a small break. */
1817        CURLMcode mcode = curl_multi_get_offt(
1818          s->multi, CURLMINFO_XFERS_CURRENT, &nxfers);
1819        if(!mcode && !nxfers) {
1820          /* None have been added, must be waiting on a timeout */
1821          curlx_wait_ms(500);
1822        }
1823      }
1824    }
1825#if DEBUG_UV
1826    curl_mfprintf(tool_stderr, "parallel_event: uv_run(), "
1827                  "mcode=%d, %d running, %d more\n",
1828                  s->mcode, uv.s->still_running, s->more_transfers);
1829#endif
1830    uv_run(uv.loop, UV_RUN_DEFAULT);
1831#if DEBUG_UV
1832    curl_mfprintf(tool_stderr, "parallel_event: uv_run() returned\n");
1833#endif
1834
1835    /* early exit called */
1836    if(s->wrapitup) {
1837      if(s->still_running && !s->wrapitup_processed) {
1838        struct per_transfer *per;
1839        for(per = transfers; per; per = per->next) {
1840          if(per->added)
1841            per->abort = TRUE;
1842        }
1843        s->wrapitup_processed = TRUE;
1844      }
1845      break;
1846    }
1847  }
1848
1849  result = s->result;
1850
1851  /* Make sure to return some kind of error if there was a multi problem */
1852  if(s->mcode) {
1853    result = (s->mcode == CURLM_OUT_OF_MEMORY) ? CURLE_OUT_OF_MEMORY :
1854      /* The other multi errors should never happen, so return
1855         something suitably generic */
1856      CURLE_BAD_FUNCTION_ARGUMENT;
1857  }
1858
1859  /* We need to cleanup the multi here, since the uv context lives on the
1860   * stack and about to be gone. multi_cleanup can trigger events! */
1861  curl_multi_cleanup(s->multi);
1862
1863#if DEBUG_UV
1864  curl_mfprintf(tool_stderr, "DONE parallel_event -> %d, mcode=%d, "
1865          "%d running, %d more\n",
1866          result, s->mcode, uv.s->still_running, s->more_transfers);
1867#endif
1868  return result;
1869}
1870
1871#endif
1872
1873static CURLcode check_finished(struct parastate *s)
1874{
1875  CURLcode result = CURLE_OK;
1876  int rc;
1877  CURLMsg *msg;
1878  bool checkmore = FALSE;
1879  do {
1880    msg = curl_multi_info_read(s->multi, &rc);
1881    if(msg) {
1882      bool retry;
1883      uint32_t delay;
1884      struct per_transfer *ended;
1885      CURL *easy = msg->easy_handle;
1886      CURLcode tres = msg->data.result;
1887      curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ended);
1888      curl_multi_remove_handle(s->multi, easy);
1889
1890      if(ended->abort && (tres == CURLE_ABORTED_BY_CALLBACK)) {
1891        curl_msnprintf(ended->errorbuffer, CURL_ERROR_SIZE,
1892                       "Transfer aborted due to critical error "
1893                       "in another transfer");
1894      }
1895      tres = post_per_transfer(ended, tres, &retry, &delay);
1896      progress_finalize(ended); /* before it goes away */
1897      all_added--; /* one fewer added */
1898      checkmore = TRUE;
1899      if(retry) {
1900        ended->added = FALSE; /* add it again */
1901        /* we delay retries in full integer seconds only */
1902        ended->startat = delay ? time(NULL) + (delay / 1000) : 0;
1903      }
1904      else {
1905        /* result receives this transfer's error unless the transfer was
1906           marked for abort due to a critical error in another transfer */
1907        if(tres && (!ended->abort || !result))
1908          result = tres;
1909        if(is_fatal_error(result) || (result && global->fail_early))
1910          s->wrapitup = TRUE;
1911        (void)del_per_transfer(ended);
1912      }
1913    }
1914  } while(msg);
1915  if(!s->wrapitup) {
1916    if(!checkmore) {
1917      time_t tock = time(NULL);
1918      if(s->tick != tock) {
1919        checkmore = TRUE;
1920        s->tick = tock;
1921      }
1922    }
1923    if(checkmore) {
1924      /* one or more transfers completed, add more! */
1925      CURLcode tres = add_parallel_transfers(s->multi, s->share,
1926                                             &s->more_transfers,
1927                                             &s->added_transfers);
1928      if(tres)
1929        result = tres;
1930      if(s->added_transfers)
1931        /* we added new ones, make sure the loop does not exit yet */
1932        s->still_running = 1;
1933    }
1934    if(is_fatal_error(result) || (result && global->fail_early))
1935      s->wrapitup = TRUE;
1936  }
1937  return result;
1938}
1939
1940static void mnotify(CURLM *multi, unsigned int notification,
1941                    CURL *easy, void *user_data)
1942{
1943  struct parastate *s = user_data;
1944  CURLcode result;
1945
1946  (void)multi;
1947  (void)easy;
1948
1949  switch(notification) {
1950  case CURLMNOTIFY_INFO_READ:
1951    result = check_finished(s);
1952    /* remember first failure */
1953    if(result && !s->result)
1954      s->result = result;
1955    break;
1956  default:
1957    break;
1958  }
1959}
1960
1961static CURLcode parallel_transfers(CURLSH *share)
1962{
1963  CURLcode result;
1964  struct parastate p;
1965  struct parastate *s = &p;
1966  s->share = share;
1967  s->mcode = CURLM_OK;
1968  s->result = CURLE_OK;
1969  s->still_running = 1;
1970  s->start = curlx_now();
1971  s->wrapitup = FALSE;
1972  s->wrapitup_processed = FALSE;
1973  s->tick = time(NULL);
1974  s->multi = curl_multi_init();
1975  if(!s->multi)
1976    return CURLE_OUT_OF_MEMORY;
1977
1978  if(TRUE
1979#ifdef DEBUGBUILD
1980    && getenv("CURL_QUICK_EXIT")
1981#endif
1982    ) {
1983    /* QUICK_EXIT allows for running threads to be detached and not
1984     * joined. Preferably in non-debug runs. */
1985    (void)curl_multi_setopt(s->multi, CURLMOPT_QUICK_EXIT, 1L);
1986  }
1987  (void)curl_multi_setopt(s->multi, CURLMOPT_NOTIFYFUNCTION, mnotify);
1988  (void)curl_multi_setopt(s->multi, CURLMOPT_NOTIFYDATA, s);
1989  (void)curl_multi_setopt(s->multi, CURLMOPT_MAX_HOST_CONNECTIONS, (long)
1990                          global->parallel_host);
1991  (void)curl_multi_notify_enable(s->multi, CURLMNOTIFY_INFO_READ);
1992
1993  result = add_parallel_transfers(s->multi, s->share,
1994                                  &s->more_transfers, &s->added_transfers);
1995  if(result) {
1996    curl_multi_cleanup(s->multi);
1997    return result;
1998  }
1999
2000#ifdef DEBUGBUILD
2001  if(global->test_event_based)
2002#ifdef USE_LIBUV
2003    return parallel_event(s);
2004#else
2005    errorf("Testing --parallel event-based requires libuv");
2006#endif
2007  else
2008#endif
2009
2010  if(all_added) {
2011    while(!s->mcode && (s->still_running || s->more_transfers)) {
2012      /* If stopping prematurely (eg due to a --fail-early condition) then
2013         signal that any transfers in the multi should abort (via progress
2014         callback). */
2015      if(s->wrapitup) {
2016        if(!s->still_running)
2017          break;
2018        if(!s->wrapitup_processed) {
2019          struct per_transfer *per;
2020          for(per = transfers; per; per = per->next) {
2021            if(per->added)
2022              per->abort = TRUE;
2023          }
2024          s->wrapitup_processed = TRUE;
2025        }
2026      }
2027      else if(s->more_transfers) {
2028        s->result = add_parallel_transfers(s->multi, s->share,
2029                                           &s->more_transfers,
2030                                           &s->added_transfers);
2031        if(s->result)
2032          break;
2033      }
2034
2035      s->mcode = curl_multi_poll(s->multi, NULL, 0, 1000, NULL);
2036      if(!s->mcode)
2037        s->mcode = curl_multi_perform(s->multi, &s->still_running);
2038
2039      progress_meter(s->multi, &s->start, FALSE);
2040    }
2041
2042    (void)progress_meter(s->multi, &s->start, TRUE);
2043  }
2044
2045  /* Result is the first failed transfer - if there was one. */
2046  result = s->result;
2047
2048  /* Make sure to return some kind of error if there was a multi problem */
2049  if(s->mcode) {
2050    result = (s->mcode == CURLM_OUT_OF_MEMORY) ? CURLE_OUT_OF_MEMORY :
2051      /* The other multi errors should never happen, so return
2052         something suitably generic */
2053      CURLE_BAD_FUNCTION_ARGUMENT;
2054  }
2055
2056  curl_multi_cleanup(s->multi);
2057
2058  return result;
2059}
2060
2061static CURLcode serial_transfers(CURLSH *share)
2062{
2063  CURLcode returncode = CURLE_OK;
2064  CURLcode result = CURLE_OK;
2065  struct per_transfer *per;
2066  bool added = FALSE;
2067  bool skipped = FALSE;
2068
2069  result = create_transfer(share, &added, &skipped);
2070  if(result)
2071    return result;
2072  if(!added) {
2073    errorf("no transfer performed");
2074    return CURLE_READ_ERROR;
2075  }
2076  for(per = transfers; per;) {
2077    bool retry;
2078    uint32_t delay_ms;
2079    bool bailout = FALSE;
2080    struct curltime start;
2081
2082    start = curlx_now();
2083    if(!per->skip) {
2084      result = pre_transfer(per);
2085      if(result)
2086        break;
2087
2088      if(global->libcurl) {
2089        result = easysrc_perform();
2090        if(result)
2091          break;
2092      }
2093
2094#ifdef DEBUGBUILD
2095      if(getenv("CURL_FORBID_REUSE"))
2096        (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L);
2097
2098      if(global->test_duphandle) {
2099        CURL *dup = curl_easy_duphandle(per->curl);
2100        curl_easy_cleanup(per->curl);
2101        per->curl = dup;
2102        if(!dup) {
2103          result = CURLE_OUT_OF_MEMORY;
2104          break;
2105        }
2106        /* a duplicate needs the share re-added */
2107        (void)curl_easy_setopt(per->curl, CURLOPT_SHARE, share);
2108      }
2109      if(global->test_event_based)
2110        result = curl_easy_perform_ev(per->curl);
2111      else
2112#endif
2113        result = curl_easy_perform(per->curl);
2114    }
2115
2116    returncode = post_per_transfer(per, result, &retry, &delay_ms);
2117    if(retry) {
2118      curlx_wait_ms(delay_ms);
2119      continue;
2120    }
2121
2122    /* Bail out upon critical errors or --fail-early */
2123    if(is_fatal_error(returncode) || (returncode && global->fail_early))
2124      bailout = TRUE;
2125    else {
2126      do {
2127        /* setup the next one before we delete this */
2128        result = create_transfer(share, &added, &skipped);
2129        if(result) {
2130          returncode = result;
2131          bailout = TRUE;
2132          break;
2133        }
2134      } while(skipped);
2135    }
2136
2137    per = del_per_transfer(per);
2138
2139    if(bailout)
2140      break;
2141
2142    if(per && global->ms_per_transfer) {
2143      /* how long time did the most recent transfer take in number of
2144         milliseconds */
2145      timediff_t milli = curlx_timediff_ms(curlx_now(), start);
2146      if(milli < global->ms_per_transfer) {
2147        notef("Transfer took %" CURL_FORMAT_CURL_OFF_T " ms, "
2148              "waits %ldms as set by --rate",
2149              milli, (long)(global->ms_per_transfer - milli));
2150        /* The transfer took less time than wanted. Wait a little. */
2151        curlx_wait_ms((long)(global->ms_per_transfer - milli));
2152      }
2153    }
2154  }
2155  if(returncode)
2156    /* returncode errors have priority */
2157    result = returncode;
2158
2159  if(result)
2160    single_transfer_cleanup();
2161
2162  return result;
2163}
2164
2165static CURLcode is_using_schannel(int *pusing)
2166{
2167  CURLcode result = CURLE_OK;
2168  static int using_schannel = -1; /* -1 = not checked
2169                                     0 = nope
2170                                     1 = yes */
2171  if(using_schannel == -1) {
2172    CURL *curltls = curl_easy_init();
2173    /* The TLS backend remains, so keep the info */
2174    const struct curl_tlssessioninfo *tls_backend_info = NULL;
2175
2176    if(!curltls)
2177      result = CURLE_OUT_OF_MEMORY;
2178    else {
2179      result = curl_easy_getinfo(curltls, CURLINFO_TLS_SSL_PTR,
2180                                 &tls_backend_info);
2181      if(!result)
2182        using_schannel =
2183          (tls_backend_info->backend == CURLSSLBACKEND_SCHANNEL);
2184    }
2185    curl_easy_cleanup(curltls);
2186    if(result)
2187      return result;
2188  }
2189  *pusing = using_schannel;
2190  return result;
2191}
2192
2193/* Set the CA cert locations specified in the environment. For Windows if no
2194 * environment-specified filename is found then check for CA bundle default
2195 * filename curl-ca-bundle.crt in the user's PATH.
2196 *
2197 * If the user has set a CA cert/path or disabled peer verification (including
2198 * for DoH, so completely disabled) then these locations are ignored.
2199 *
2200 * If Schannel is the selected SSL backend then these locations are ignored.
2201 * We allow setting CA location for Schannel only when explicitly specified by
2202 * the user via CURLOPT_CAINFO / --cacert.
2203 */
2204static CURLcode cacertpaths(struct OperationConfig *config)
2205{
2206  char *env;
2207  CURLcode result;
2208  int using_schannel;
2209
2210  if(!feature_ssl || config->cacert || config->capath ||
2211     (config->insecure_ok && (!config->doh_url || config->doh_insecure_ok)))
2212    return CURLE_OK;
2213
2214  result = is_using_schannel(&using_schannel);
2215  if(result || using_schannel)
2216    return result;
2217
2218  env = curl_getenv("CURL_CA_BUNDLE");
2219  if(env) {
2220    config->cacert = curlx_strdup(env);
2221    curl_free(env);
2222    if(!config->cacert) {
2223      result = CURLE_OUT_OF_MEMORY;
2224      goto fail;
2225    }
2226  }
2227  else {
2228    env = curl_getenv("SSL_CERT_DIR");
2229    if(env) {
2230      config->capath = curlx_strdup(env);
2231      curl_free(env);
2232      if(!config->capath) {
2233        result = CURLE_OUT_OF_MEMORY;
2234        goto fail;
2235      }
2236    }
2237    env = curl_getenv("SSL_CERT_FILE");
2238    if(env) {
2239      config->cacert = curlx_strdup(env);
2240      curl_free(env);
2241      if(!config->cacert) {
2242        result = CURLE_OUT_OF_MEMORY;
2243        goto fail;
2244      }
2245    }
2246  }
2247
2248#ifdef _WIN32
2249  if(!config->capath && !config->cacert) {
2250#ifdef CURL_CA_SEARCH_SAFE
2251    char *cacert = NULL;
2252    FILE *cafile = tool_execpath("curl-ca-bundle.crt", &cacert);
2253    if(cafile) {
2254      curlx_fclose(cafile);
2255      config->cacert = curlx_strdup(cacert);
2256      if(!config->cacert) {
2257        result = CURLE_OUT_OF_MEMORY;
2258        goto fail;
2259      }
2260    }
2261#elif !defined(CURL_WINDOWS_UWP) && !defined(CURL_DISABLE_CA_SEARCH)
2262    result = FindWin32CACert(config, TEXT("curl-ca-bundle.crt"));
2263    if(result)
2264      goto fail;
2265#endif
2266  }
2267#endif
2268  return CURLE_OK;
2269fail:
2270  curlx_safefree(config->capath);
2271  return result;
2272}
2273
2274/* setup a transfer for the given config */
2275static CURLcode transfer_per_config(struct OperationConfig *config,
2276                                    CURLSH *share,
2277                                    bool *added,
2278                                    bool *skipped)
2279{
2280  CURLcode result;
2281  *added = FALSE;
2282
2283  /* Check we have a URL */
2284  if(!config->url_list || !config->url_list->url) {
2285    helpf("(%d) no URL specified", CURLE_FAILED_INIT);
2286    result = CURLE_FAILED_INIT;
2287  }
2288  else {
2289    result = cacertpaths(config);
2290    if(!result) {
2291      result = single_transfer(config, share, added, skipped);
2292      if(!*added || result)
2293        single_transfer_cleanup();
2294    }
2295  }
2296
2297  return result;
2298}
2299
2300/*
2301 * 'create_transfer' gets the details and sets up a new transfer if 'added'
2302 * returns TRUE.
2303 */
2304static CURLcode create_transfer(CURLSH *share,
2305                                bool *added,
2306                                bool *skipped)
2307{
2308  CURLcode result = CURLE_OK;
2309  *added = FALSE;
2310  while(global->current) {
2311    result = transfer_per_config(global->current, share, added, skipped);
2312    if(!result && !*added) {
2313      /* when one set is drained, continue to next */
2314      global->current = global->current->next;
2315      continue;
2316    }
2317    break;
2318  }
2319  return result;
2320}
2321
2322static CURLcode run_all_transfers(CURLSH *share,
2323                                  CURLcode result)
2324{
2325  /* Save the values of noprogress and isatty to restore them later on */
2326  bool orig_noprogress = (bool)global->noprogress;
2327  bool orig_isatty = (bool)global->isatty;
2328  struct per_transfer *per;
2329
2330  /* Time to actually do the transfers */
2331  if(!result) {
2332    if(global->parallel)
2333      result = parallel_transfers(share);
2334    else
2335      result = serial_transfers(share);
2336  }
2337
2338  /* cleanup if there are any left */
2339  for(per = transfers; per;) {
2340    bool retry;
2341    uint32_t delay;
2342    CURLcode result2 = post_per_transfer(per, result, &retry, &delay);
2343    if(!result)
2344      /* do not overwrite the original error */
2345      result = result2;
2346
2347    /* Free list of given URLs */
2348    clean_getout(per->config);
2349
2350    per = del_per_transfer(per);
2351  }
2352
2353  /* Reset the global config variables */
2354  global->noprogress = orig_noprogress;
2355  global->isatty = orig_isatty;
2356
2357  return result;
2358}
2359
2360static CURLcode share_setopt(CURLSH *share, int opt)
2361{
2362  CURLSHcode shres = curl_share_setopt(share, CURLSHOPT_SHARE, opt);
2363  if(!shres || (shres == CURLSHE_NOT_BUILT_IN))
2364    return CURLE_OK;
2365  return CURLE_FAILED_INIT;
2366}
2367
2368static CURLcode share_setup(CURLSH *share)
2369{
2370  CURLcode result = CURLE_OK;
2371  int i;
2372  static int options[7] = {
2373    CURL_LOCK_DATA_COOKIE,
2374    CURL_LOCK_DATA_DNS,
2375    CURL_LOCK_DATA_SSL_SESSION,
2376    CURL_LOCK_DATA_PSL,
2377    CURL_LOCK_DATA_HSTS,
2378    0, /* 5 might be set below */
2379    0
2380  };
2381  /* Running parallel, use the multi connection cache */
2382  if(!global->parallel)
2383    options[5] = CURL_LOCK_DATA_CONNECT;
2384  for(i = 0; !result && options[i]; i++)
2385    result = share_setopt(share, options[i]);
2386  return result;
2387}
2388
2389CURLcode operate(int argc, argv_item_t argv[])
2390{
2391  CURLcode result = CURLE_OK;
2392  const char *first_arg;
2393  char *curlrc_path = NULL;
2394  bool found_curlrc = FALSE;
2395
2396  first_arg = argc > 1 ? convert_tchar_to_UTF8(argv[1]) : NULL;
2397
2398#ifdef HAVE_SETLOCALE
2399  /* Override locale for number parsing (only) */
2400  setlocale(LC_ALL, "");
2401  setlocale(LC_NUMERIC, "C");
2402#endif
2403
2404  /* Parse .curlrc if necessary */
2405  if((argc == 1) ||
2406     (first_arg && strncmp(first_arg, "-q", 2) &&
2407      strcmp(first_arg, "--disable"))) {
2408    if(!parseconfig(NULL, CONFIG_MAX_LEVELS, &curlrc_path))
2409      found_curlrc = TRUE;
2410
2411    /* If we had no arguments then make sure a URL was specified in .curlrc */
2412    if((argc < 2) && (!global->first->url_list)) {
2413      helpf(NULL);
2414      result = CURLE_FAILED_INIT;
2415    }
2416  }
2417
2418  unicodefree(CURL_UNCONST(first_arg));
2419
2420  if(!result) {
2421    /* Parse the command line arguments */
2422    ParameterError err = parse_args(argc, argv);
2423    if(found_curlrc) {
2424      /* After parse_args so notef knows the verbosity */
2425      notef("Read config file from '%s'", curlrc_path);
2426    }
2427    if(err) {
2428      result = CURLE_OK;
2429
2430      /* Check if we were asked for the help */
2431      if(err == PARAM_HELP_REQUESTED)
2432        ; /* already done */
2433      /* Check if we were asked for the manual */
2434      else if(err == PARAM_MANUAL_REQUESTED) {
2435#ifdef USE_MANUAL
2436        hugehelp();
2437#else
2438        warnf("built-in manual was disabled at build-time");
2439#endif
2440      }
2441      /* Check if we were asked for the version information */
2442      else if(err == PARAM_VERSION_INFO_REQUESTED)
2443        tool_version_info();
2444      /* Check if we were asked to list the SSL engines */
2445      else if(err == PARAM_ENGINES_REQUESTED)
2446        tool_list_engines();
2447      /* Check if we were asked to dump the embedded CA bundle */
2448      else if(err == PARAM_CA_EMBED_REQUESTED) {
2449#ifdef CURL_CA_EMBED
2450        curl_mprintf("%s", curl_ca_embed);
2451#endif
2452      }
2453      else if(err == PARAM_LIBCURL_UNSUPPORTED_PROTOCOL)
2454        result = CURLE_UNSUPPORTED_PROTOCOL;
2455      else if(err == PARAM_READ_ERROR)
2456        result = CURLE_READ_ERROR;
2457      else
2458        result = CURLE_FAILED_INIT;
2459    }
2460    else {
2461      if(global->libcurl) {
2462        /* Initialize the libcurl source output */
2463        result = easysrc_init();
2464      }
2465
2466      /* Perform the main operations */
2467      if(!result) {
2468        size_t count = 0;
2469        struct OperationConfig *operation = global->first;
2470        CURLSH *share = curl_share_init();
2471        if(!share) {
2472          if(global->libcurl) {
2473            /* Cleanup the libcurl source output */
2474            easysrc_cleanup();
2475          }
2476          result = CURLE_OUT_OF_MEMORY;
2477        }
2478
2479        if(!result)
2480          result = share_setup(share);
2481
2482        if(!result && global->ssl_sessions && feature_ssls_export)
2483          result = tool_ssls_load(global->first, share,
2484                                  global->ssl_sessions);
2485
2486        if(!result) {
2487          /* Get the required arguments for each operation */
2488          do {
2489            result = get_args(operation, count++);
2490
2491            operation = operation->next;
2492          } while(!result && operation);
2493
2494          if(!result) {
2495            /* Set the current operation pointer */
2496            global->current = global->first;
2497
2498            /* now run! */
2499            result = run_all_transfers(share, result);
2500
2501            if(global->ssl_sessions && feature_ssls_export) {
2502              CURLcode r2 = tool_ssls_save(global->first, share,
2503                                           global->ssl_sessions);
2504              if(r2 && !result)
2505                result = r2;
2506            }
2507          }
2508        }
2509
2510        curl_share_cleanup(share);
2511        if(global->libcurl) {
2512          /* Cleanup the libcurl source output */
2513          easysrc_cleanup();
2514
2515          /* Dump the libcurl code if previously enabled */
2516          dumpeasysrc();
2517        }
2518      }
2519      else
2520        errorf("out of memory");
2521    }
2522  }
2523  curlx_free(curlrc_path);
2524  varcleanup();
2525
2526  return result;
2527}