curl: add --parallel-max-host to limit concurrent connections per host

Where 'host' is protocol + hostname + portnumber.

Closes #18052

docs/cmdline-opts/Makefile.inc

@@ -184,6 +184,7 @@ DPAGES = \
   out-null.md \
   output.md \
   parallel-immediate.md \
+  parallel-max-host.md \
   parallel-max.md \
   parallel.md \
   pass.md \

docs/cmdline-opts/parallel-max-host.md

@@ -0,0 +1,28 @@
+---
+c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+SPDX-License-Identifier: curl
+Long: parallel-max-host
+Arg: <num>
+Help: Maximum connections to a single host
+Added: 8.16.0
+Category: connection curl global
+Multi: single
+Scope: global
+See-also:
+  - parallel
+  - parallel-max
+Example:
+  - --parallel-max-host 5 -Z $URL ftp://example.com/
+---
+
+# `--parallel-max-host`
+
+When asked to do parallel transfers, using --parallel, this option controls
+the maximum amount of concurrent connections curl is allowed to do to the same
+protocol + hostname + port number target.
+
+The limit is enforced by libcurl and queued "internally", which means that
+transfers that are waiting for an available connection still look like started
+transfers in the progress meter.
+
+The default is 0 (unlimited). 65535 is the largest supported value.

docs/cmdline-opts/parallel-max.md

@@ -10,6 +10,7 @@ Multi: single
 Scope: global
 See-also:
   - parallel
+  - parallel-max-host
 Example:
   - --parallel-max 100 -Z $URL ftp://example.com/
 ---

docs/options-in-versions

@@ -151,6 +151,7 @@
 --parallel (-Z)                      7.66.0
 --parallel-immediate                 7.68.0
 --parallel-max                       7.66.0
+--parallel-max-host                  8.16.0
 --pass                               7.9.3
 --path-as-is                         7.42.0
 --pinnedpubkey                       7.39.0

src/tool_cfgable.h

@@ -357,6 +357,7 @@ struct GlobalConfig {
                                      many milliseconds */
   trace tracetype;
   int progressmode;               /* CURL_PROGRESS_BAR / CURL_PROGRESS_STATS */
+  unsigned short parallel_host; /* MAX_PARALLEL_HOST is the maximum */
   unsigned short parallel_max; /* MAX_PARALLEL is the maximum */
   unsigned char verbosity;        /* How verbose we should be */
 #ifdef DEBUGBUILD

src/tool_getparam.c

@@ -230,6 +230,7 @@ static const struct LongShort aliases[]= {
   {"parallel",                   ARG_BOOL, 'Z', C_PARALLEL},
   {"parallel-immediate",         ARG_BOOL, ' ', C_PARALLEL_IMMEDIATE},
   {"parallel-max",               ARG_STRG, ' ', C_PARALLEL_MAX},
+  {"parallel-max-host",          ARG_STRG, ' ', C_PARALLEL_HOST},
   {"pass",                       ARG_STRG|ARG_CLEAR, ' ', C_PASS},
   {"path-as-is",                 ARG_BOOL, ' ', C_PATH_AS_IS},
   {"pinnedpubkey",               ARG_STRG|ARG_TLS, ' ', C_PINNEDPUBKEY},
@@ -2179,6 +2180,7 @@ static ParameterError opt_filestring(struct OperationConfig *config,
 {
   ParameterError err = PARAM_OK;
   curl_off_t value;
+  long val;
   struct GlobalConfig *global = config->global;
   static const char *redir_protos[] = {
     "http",
@@ -2788,8 +2790,19 @@ static ParameterError opt_filestring(struct OperationConfig *config,
     if(!err && !config->low_speed_time)
       config->low_speed_time = 30;
     break;
-  case C_PARALLEL_MAX: {  /* --parallel-max */
-    long val;
+  case C_PARALLEL_HOST: /* --parallel-max-host */
+    err = str2unum(&val, nextarg);
+    if(err)
+      break;
+    if(val > MAX_PARALLEL_HOST)
+      global->parallel_host = MAX_PARALLEL_HOST;
+    else if(val < 1)
+      global->parallel_host = PARALLEL_HOST_DEFAULT;
+    else
+      global->parallel_host = (unsigned short)val;
+    break;
+    break;
+  case C_PARALLEL_MAX:  /* --parallel-max */
     err = str2unum(&val, nextarg);
     if(err)
       break;
@@ -2800,7 +2813,6 @@ static ParameterError opt_filestring(struct OperationConfig *config,
     else
       global->parallel_max = (unsigned short)val;
     break;
-  }
   case C_TIME_COND: /* --time-cond */
     err = parse_time_cond(config, nextarg);
     break;

src/tool_getparam.h

@@ -171,6 +171,7 @@ typedef enum {
   C_OUTPUT,
   C_OUTPUT_DIR,
   C_PARALLEL,
+  C_PARALLEL_HOST,
   C_PARALLEL_IMMEDIATE,
   C_PARALLEL_MAX,
   C_PASS,

src/tool_listhelp.c

@@ -460,6 +460,9 @@ const struct helptxt helptext[] = {
   {"    --parallel-max <num>",
    "Maximum concurrency for parallel transfers",
    CURLHELP_CONNECTION | CURLHELP_CURL | CURLHELP_GLOBAL},
+  {"    --parallel-max-host <num>",
+   "Maximum connections to a single host",
+   CURLHELP_CONNECTION | CURLHELP_CURL | CURLHELP_GLOBAL},
   {"    --pass <phrase>",
    "Passphrase for the private key",
    CURLHELP_SSH | CURLHELP_TLS | CURLHELP_AUTH},

src/tool_main.h

@@ -33,4 +33,7 @@
 #define MAX_PARALLEL 65535
 #define PARALLEL_DEFAULT 50
 
+#define MAX_PARALLEL_HOST 65535
+#define PARALLEL_HOST_DEFAULT 0 /* means not used */
+
 #endif /* HEADER_CURL_TOOL_MAIN_H */

src/tool_operate.c

@@ -1689,6 +1689,8 @@ static CURLcode parallel_event(struct parastate *s)
   curl_multi_setopt(s->multi, CURLMOPT_SOCKETDATA, &uv);
   curl_multi_setopt(s->multi, CURLMOPT_TIMERFUNCTION, cb_timeout);
   curl_multi_setopt(s->multi, CURLMOPT_TIMERDATA, &uv);
+  curl_multi_setopt(s->multi, CURLMOPT_MAX_HOST_CONNECTIONS,
+                    s->global->parallel_host);
 
   /* kickstart the thing */
   curl_multi_socket_action(s->multi, CURL_SOCKET_TIMEOUT, 0,